diff --git a/builtin/worktree.c b/builtin/worktree.c
index a95fe67da6..c4abbde2b8 100644
--- a/builtin/worktree.c
+++ b/builtin/worktree.c
@@ -62,6 +62,11 @@ static int delete_git_dir(const char *id)
 	return ret;
 }
 
+static void delete_worktrees_dir_if_empty(void)
+{
+	rmdir(git_path("worktrees")); /* ignore failed removal */
+}
+
 static int prune_worktree(const char *id, struct strbuf *reason)
 {
 	struct stat st;
@@ -149,7 +154,7 @@ static void prune_worktrees(void)
 	}
 	closedir(dir);
 	if (!show_only)
-		rmdir(git_path("worktrees"));
+		delete_worktrees_dir_if_empty();
 	strbuf_release(&reason);
 }
 
@@ -918,6 +923,7 @@ static int remove_worktree(int ac, const char **av, const char *prefix)
 	 * from here.
 	 */
 	ret |= delete_git_dir(wt->id);
+	delete_worktrees_dir_if_empty();
 
 	free_worktrees(worktrees);
 	return ret;
diff --git a/t/t2028-worktree-move.sh b/t/t2028-worktree-move.sh
index 1b5079e8fa..33c0337733 100755
--- a/t/t2028-worktree-move.sh
+++ b/t/t2028-worktree-move.sh
@@ -173,4 +173,16 @@ test_expect_success 'remove locked worktree (force)' '
 	git worktree remove --force --force gumby
 '
 
+test_expect_success 'remove cleans up .git/worktrees when empty' '
+	git init moog &&
+	(
+		cd moog &&
+		test_commit bim &&
+		git worktree add --detach goom &&
+		test_path_exists .git/worktrees &&
+		git worktree remove goom &&
+		test_path_is_missing .git/worktrees
+	)
+'
+
 test_done