diff --git a/refs.c b/refs.c
index af5da5f1c1..c9f48735fc 100644
--- a/refs.c
+++ b/refs.c
@@ -259,8 +259,13 @@ static void clear_ref_dir(struct ref_dir *dir);
 
 static void free_ref_entry(struct ref_entry *entry)
 {
-	if (entry->flag & REF_DIR)
-		clear_ref_dir(get_ref_dir(entry));
+	if (entry->flag & REF_DIR) {
+		/*
+		 * Do not use get_ref_dir() here, as that might
+		 * trigger the reading of loose refs.
+		 */
+		clear_ref_dir(&entry->u.subdir);
+	}
 	free(entry);
 }