diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh
index a0cc99ab9f..1a4dc5f893 100755
--- a/t/t7400-submodule-basic.sh
+++ b/t/t7400-submodule-basic.sh
@@ -299,6 +299,15 @@ test_expect_success 'ls-files gracefully handles trailing slash' '
 
 '
 
+test_expect_success 'moving to a commit without submodule does not leave empty dir' '
+	rm -rf init &&
+	mkdir init &&
+	git reset --hard &&
+	git checkout initial &&
+	test ! -d init &&
+	git checkout second
+'
+
 test_expect_success 'submodule <invalid-path> warns' '
 
 	git submodule no-such-submodule 2> output.err &&
diff --git a/unpack-trees.c b/unpack-trees.c
index dd5999c356..b69847d6c9 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -61,8 +61,16 @@ static void unlink_entry(struct cache_entry *ce)
 {
 	if (has_symlink_or_noent_leading_path(ce->name, ce_namelen(ce)))
 		return;
-	if (unlink_or_warn(ce->name))
-		return;
+	if (S_ISGITLINK(ce->ce_mode)) {
+		if (rmdir(ce->name)) {
+			warning("unable to rmdir %s: %s",
+				ce->name, strerror(errno));
+			return;
+		}
+	}
+	else
+		if (unlink_or_warn(ce->name))
+			return;
 	schedule_dir_for_removal(ce->name, ce_namelen(ce));
 }