diff --git a/builtin/worktree.c b/builtin/worktree.c
index 654d27c3e1ce99..cc46c1b415bbc1 100644
--- a/builtin/worktree.c
+++ b/builtin/worktree.c
@@ -898,16 +898,18 @@ static int add(int ac, const char **av, const char *prefix,
 		/* DWIM: Infer --orphan when repo has no refs. */
 		opts.orphan = (!s) && dwim_orphan(&opts, !!opt_track, 1);
 	} else if (ac == 2) {
-		struct object_id oid;
-		struct commit *commit;
-		char *remote;
-
-		commit = lookup_commit_reference_by_name(branch);
-		if (!commit) {
-			remote = unique_tracking_name(branch, &oid, NULL);
-			if (remote) {
-				new_branch = branch;
-				branch = new_branch_to_free = remote;
+		if (!new_branch) {
+			struct object_id oid;
+			struct commit *commit;
+			char *remote;
+
+			commit = lookup_commit_reference_by_name(branch);
+			if (!commit) {
+				remote = unique_tracking_name(branch, &oid, NULL);
+				if (remote) {
+					new_branch = branch;
+					branch = new_branch_to_free = remote;
+				}
 			}
 		}
 
diff --git a/t/t2400-worktree-add.sh b/t/t2400-worktree-add.sh
index 87b926728ad8cf..ba3bec078fcdcc 100755
--- a/t/t2400-worktree-add.sh
+++ b/t/t2400-worktree-add.sh
@@ -621,6 +621,16 @@ test_expect_success '"add" <path> <branch> dwims' '
 	)
 '
 
+test_expect_success '"add" <path> <branch> does not dwim with -b' '
+	test_when_finished rm -rf repo_upstream repo_dwim wt &&
+	setup_remote_repo repo_upstream repo_dwim &&
+	(
+		cd repo_dwim &&
+		test_must_fail git worktree add -b branch ../wt foo 2>actual &&
+		test_grep "^fatal: invalid reference: foo" actual
+	)
+'
+
 test_expect_success '"add" <path> <branch> dwims with checkout.defaultRemote' '
 	test_when_finished rm -rf repo_upstream repo_dwim foo &&
 	setup_remote_repo repo_upstream repo_dwim &&