diff --git a/Documentation/config/format.txt b/Documentation/config/format.txt
index 513fcd88d5..45c7bd5a8f 100644
--- a/Documentation/config/format.txt
+++ b/Documentation/config/format.txt
@@ -106,4 +106,20 @@ If one wishes to use the ref `ref/notes/true`, please use that literal
 instead.
 +
 This configuration can be specified multiple times in order to allow
-multiple notes refs to be included.
+multiple notes refs to be included. In that case, it will behave
+similarly to multiple `--[no-]notes[=]` options passed in. That is, a
+value of `true` will show the default notes, a value of `<ref>` will
+also show notes from that notes ref and a value of `false` will negate
+previous configurations and not show notes.
++
+For example,
++
+------------
+[format]
+	notes = true
+	notes = foo
+	notes = false
+	notes = bar
+------------
++
+will only show notes from `refs/notes/bar`.
diff --git a/builtin/log.c b/builtin/log.c
index 8c664067ca..83a4a6188e 100644
--- a/builtin/log.c
+++ b/builtin/log.c
@@ -208,7 +208,7 @@ static void cmd_log_init_finish(int argc, const char **argv, const char *prefix,
 	if (!rev->show_notes_given && (!rev->pretty_given || w.notes))
 		rev->show_notes = 1;
 	if (rev->show_notes)
-		init_display_notes(&rev->notes_opt);
+		load_display_notes(&rev->notes_opt);
 
 	if ((rev->diffopt.pickaxe_opts & DIFF_PICKAXE_KINDS_MASK) ||
 	    rev->diffopt.filter || rev->diffopt.flags.follow_renames)
@@ -795,6 +795,8 @@ static const char *signature_file;
 static enum cover_setting config_cover_letter;
 static const char *config_output_directory;
 static enum cover_from_description cover_from_description_mode = COVER_FROM_MESSAGE;
+static int show_notes;
+static struct display_notes_opt notes_opt;
 
 static enum cover_from_description parse_cover_from_description(const char *arg)
 {
@@ -814,8 +816,6 @@ static enum cover_from_description parse_cover_from_description(const char *arg)
 
 static int git_format_config(const char *var, const char *value, void *cb)
 {
-	struct rev_info *rev = cb;
-
 	if (!strcmp(var, "format.headers")) {
 		if (!value)
 			die(_("format.headers without value"));
@@ -902,19 +902,13 @@ static int git_format_config(const char *var, const char *value, void *cb)
 		return 0;
 	}
 	if (!strcmp(var, "format.notes")) {
-		struct strbuf buf = STRBUF_INIT;
 		int b = git_parse_maybe_bool(value);
-		if (!b)
-			return 0;
-		rev->show_notes = 1;
-		if (b < 0) {
-			strbuf_addstr(&buf, value);
-			expand_notes_ref(&buf);
-			string_list_append(&rev->notes_opt.extra_notes_refs,
-					strbuf_detach(&buf, NULL));
-		} else {
-			rev->notes_opt.use_default_notes = 1;
-		}
+		if (b < 0)
+			enable_ref_display_notes(&notes_opt, &show_notes, value);
+		else if (b)
+			enable_default_display_notes(&notes_opt, &show_notes);
+		else
+			disable_display_notes(&notes_opt, &show_notes);
 		return 0;
 	}
 	if (!strcmp(var, "format.coverfromdescription")) {
@@ -1719,8 +1713,11 @@ int cmd_format_patch(int argc, const char **argv, const char *prefix)
 	extra_to.strdup_strings = 1;
 	extra_cc.strdup_strings = 1;
 	init_log_defaults();
+	init_display_notes(&notes_opt);
+	git_config(git_format_config, NULL);
 	repo_init_revisions(the_repository, &rev, prefix);
-	git_config(git_format_config, &rev);
+	rev.show_notes = show_notes;
+	memcpy(&rev.notes_opt, &notes_opt, sizeof(notes_opt));
 	rev.commit_format = CMIT_FMT_EMAIL;
 	rev.expand_tabs_in_log_default = 0;
 	rev.verbose_header = 1;
@@ -1839,7 +1836,7 @@ int cmd_format_patch(int argc, const char **argv, const char *prefix)
 		rev.diffopt.flags.binary = 1;
 
 	if (rev.show_notes)
-		init_display_notes(&rev.notes_opt);
+		load_display_notes(&rev.notes_opt);
 
 	if (!output_directory && !use_stdout)
 		output_directory = config_output_directory;
diff --git a/notes.c b/notes.c
index 03e7d0cd2d..0c79964c26 100644
--- a/notes.c
+++ b/notes.c
@@ -1042,6 +1042,39 @@ struct notes_tree **load_notes_trees(struct string_list *refs, int flags)
 }
 
 void init_display_notes(struct display_notes_opt *opt)
+{
+	memset(opt, 0, sizeof(*opt));
+	opt->use_default_notes = -1;
+}
+
+void enable_default_display_notes(struct display_notes_opt *opt, int *show_notes)
+{
+	opt->use_default_notes = 1;
+	*show_notes = 1;
+}
+
+void enable_ref_display_notes(struct display_notes_opt *opt, int *show_notes,
+		const char *ref) {
+	struct strbuf buf = STRBUF_INIT;
+	strbuf_addstr(&buf, ref);
+	expand_notes_ref(&buf);
+	string_list_append(&opt->extra_notes_refs,
+			strbuf_detach(&buf, NULL));
+	*show_notes = 1;
+}
+
+void disable_display_notes(struct display_notes_opt *opt, int *show_notes)
+{
+	opt->use_default_notes = -1;
+	/* we have been strdup'ing ourselves, so trick
+	 * string_list into free()ing strings */
+	opt->extra_notes_refs.strdup_strings = 1;
+	string_list_clear(&opt->extra_notes_refs, 0);
+	opt->extra_notes_refs.strdup_strings = 0;
+	*show_notes = 0;
+}
+
+void load_display_notes(struct display_notes_opt *opt)
 {
 	char *display_ref_env;
 	int load_config_refs = 0;
diff --git a/notes.h b/notes.h
index 76337f2384..c1682c39a9 100644
--- a/notes.h
+++ b/notes.h
@@ -260,6 +260,26 @@ struct display_notes_opt {
 	struct string_list extra_notes_refs;
 };
 
+/*
+ * Initialize a display_notes_opt to its default value.
+ */
+void init_display_notes(struct display_notes_opt *opt);
+
+/*
+ * This family of functions enables or disables the display of notes. In
+ * particular, 'enable_default_display_notes' will display the default notes,
+ * 'enable_ref_display_notes' will display the notes ref 'ref' and
+ * 'disable_display_notes' will disable notes, including those added by previous
+ * invocations of the 'enable_*_display_notes' functions.
+ *
+ * 'show_notes' is a pointer to a boolean which will be set to 1 if notes are
+ * displayed, else 0. It must not be NULL.
+ */
+void enable_default_display_notes(struct display_notes_opt *opt, int *show_notes);
+void enable_ref_display_notes(struct display_notes_opt *opt, int *show_notes,
+		const char *ref);
+void disable_display_notes(struct display_notes_opt *opt, int *show_notes);
+
 /*
  * Load the notes machinery for displaying several notes trees.
  *
@@ -272,16 +292,16 @@ struct display_notes_opt {
  * - extra_notes_refs may contain a list of globs (in the same style
  *   as notes.displayRef) where notes should be loaded from.
  */
-void init_display_notes(struct display_notes_opt *opt);
+void load_display_notes(struct display_notes_opt *opt);
 
 /*
  * Append notes for the given 'object_sha1' from all trees set up by
- * init_display_notes() to 'sb'.
+ * load_display_notes() to 'sb'.
  *
  * If 'raw' is false the note will be indented by 4 places and
  * a 'Notes (refname):' header added.
  *
- * You *must* call init_display_notes() before using this function.
+ * You *must* call load_display_notes() before using this function.
  */
 void format_display_notes(const struct object_id *object_oid,
 			  struct strbuf *sb, const char *output_encoding, int raw);
diff --git a/revision.c b/revision.c
index 39a25e7a5d..8136929e23 100644
--- a/revision.c
+++ b/revision.c
@@ -1668,7 +1668,7 @@ void repo_init_revisions(struct repository *r,
 		revs->diffopt.prefix_length = strlen(prefix);
 	}
 
-	revs->notes_opt.use_default_notes = -1;
+	init_display_notes(&revs->notes_opt);
 }
 
 static void add_pending_commit_list(struct rev_info *revs,
@@ -2203,9 +2203,8 @@ static int handle_revision_opt(struct rev_info *revs, int argc, const char **arg
 			die("'%s': not a non-negative integer", arg);
 		revs->expand_tabs_in_log = val;
 	} else if (!strcmp(arg, "--show-notes") || !strcmp(arg, "--notes")) {
-		revs->show_notes = 1;
+		enable_default_display_notes(&revs->notes_opt, &revs->show_notes);
 		revs->show_notes_given = 1;
-		revs->notes_opt.use_default_notes = 1;
 	} else if (!strcmp(arg, "--show-signature")) {
 		revs->show_signature = 1;
 	} else if (!strcmp(arg, "--no-show-signature")) {
@@ -2220,25 +2219,14 @@ static int handle_revision_opt(struct rev_info *revs, int argc, const char **arg
 		revs->track_first_time = 1;
 	} else if (skip_prefix(arg, "--show-notes=", &optarg) ||
 		   skip_prefix(arg, "--notes=", &optarg)) {
-		struct strbuf buf = STRBUF_INIT;
-		revs->show_notes = 1;
-		revs->show_notes_given = 1;
 		if (starts_with(arg, "--show-notes=") &&
 		    revs->notes_opt.use_default_notes < 0)
 			revs->notes_opt.use_default_notes = 1;
-		strbuf_addstr(&buf, optarg);
-		expand_notes_ref(&buf);
-		string_list_append(&revs->notes_opt.extra_notes_refs,
-				   strbuf_detach(&buf, NULL));
-	} else if (!strcmp(arg, "--no-notes")) {
-		revs->show_notes = 0;
+		enable_ref_display_notes(&revs->notes_opt, &revs->show_notes, optarg);
+		revs->show_notes_given = 1;
+	} else if (!strcmp(arg, "--no-notes")) {
+		disable_display_notes(&revs->notes_opt, &revs->show_notes);
 		revs->show_notes_given = 1;
-		revs->notes_opt.use_default_notes = -1;
-		/* we have been strdup'ing ourselves, so trick
-		 * string_list into free()ing strings */
-		revs->notes_opt.extra_notes_refs.strdup_strings = 1;
-		string_list_clear(&revs->notes_opt.extra_notes_refs, 0);
-		revs->notes_opt.extra_notes_refs.strdup_strings = 0;
 	} else if (!strcmp(arg, "--standard-notes")) {
 		revs->show_notes_given = 1;
 		revs->notes_opt.use_default_notes = 1;
diff --git a/revision.h b/revision.h
index a1a804bd3d..475f048fb6 100644
--- a/revision.h
+++ b/revision.h
@@ -190,10 +190,10 @@ struct rev_info {
 			always_show_header:1;
 
 	/* Format info */
+	int		show_notes;
 	unsigned int	shown_one:1,
 			shown_dashes:1,
 			show_merge:1,
-			show_notes:1,
 			show_notes_given:1,
 			show_signature:1,
 			pretty_given:1,
diff --git a/t/t4014-format-patch.sh b/t/t4014-format-patch.sh
index a5b6302a1c..b653dd7d44 100755
--- a/t/t4014-format-patch.sh
+++ b/t/t4014-format-patch.sh
@@ -793,6 +793,38 @@ test_expect_success 'format-patch with multiple notes refs' '
 	! grep "this is note 2" out
 '
 
+test_expect_success 'format-patch with multiple notes refs in config' '
+	test_when_finished "test_unconfig format.notes" &&
+
+	git notes --ref note1 add -m "this is note 1" HEAD &&
+	test_when_finished git notes --ref note1 remove HEAD &&
+	git notes --ref note2 add -m "this is note 2" HEAD &&
+	test_when_finished git notes --ref note2 remove HEAD &&
+
+	git config format.notes note1 &&
+	git format-patch -1 --stdout >out &&
+	grep "this is note 1" out &&
+	! grep "this is note 2" out &&
+	git config format.notes note2 &&
+	git format-patch -1 --stdout >out &&
+	! grep "this is note 1" out &&
+	grep "this is note 2" out &&
+	git config --add format.notes note1 &&
+	git format-patch -1 --stdout >out &&
+	grep "this is note 1" out &&
+	grep "this is note 2" out &&
+
+	git config --replace-all format.notes note1 &&
+	git config --add format.notes false &&
+	git format-patch -1 --stdout >out &&
+	! grep "this is note 1" out &&
+	! grep "this is note 2" out &&
+	git config --add format.notes note2 &&
+	git format-patch -1 --stdout >out &&
+	! grep "this is note 1" out &&
+	grep "this is note 2" out
+'
+
 echo "fatal: --name-only does not make sense" >expect.name-only
 echo "fatal: --name-status does not make sense" >expect.name-status
 echo "fatal: --check does not make sense" >expect.check