Merge branch 'lt/gitlink'

* lt/gitlink:
  Tests for core subproject support
  Expose subprojects as special files to "git diff" machinery
  Fix some "git ls-files -o" fallout from gitlinks
  Teach "git-read-tree -u" to check out submodules as a directory
  Teach git list-objects logic to not follow gitlinks
  Fix gitlink index entry filesystem matching
  Teach "git-read-tree -u" to check out submodules as a directory
  Teach git list-objects logic not to follow gitlinks
  Don't show gitlink directories when we want "other" files
  Teach git-update-index about gitlinks
  Teach directory traversal about subprojects
  Fix thinko in subproject entry sorting
  Teach core object handling functions about gitlinks
  Teach "fsck" not to follow subproject links
  Add "S_IFDIRLNK" file mode infrastructure for git links
  Add 'resolve_gitlink_ref()' helper function
  Avoid overflowing name buffer in deep directory structures
  diff-lib: use ce_mode_from_stat() rather than messing with modes manually

builtin-fsck.c

@@ -253,6 +253,7 @@ static int fsck_tree(struct tree *item)
 		case S_IFREG | 0644:
 		case S_IFLNK:
 		case S_IFDIR:
+		case S_IFDIRLNK:
 			break;
 		/*
 		 * This is nonstandard, but we had a few of these
@@ -703,8 +704,14 @@ int cmd_fsck(int argc, char **argv, const char *prefix)
 		int i;
 		read_cache();
 		for (i = 0; i < active_nr; i++) {
-			struct blob *blob = lookup_blob(active_cache[i]->sha1);
+			unsigned int mode;
+			struct blob *blob;
 			struct object *obj;
+
+			mode = ntohl(active_cache[i]->ce_mode);
+			if (S_ISDIRLNK(mode))
+				continue;
+			blob = lookup_blob(active_cache[i]->sha1);
 			if (!blob)
 				continue;
 			obj = &blob->object;

builtin-ls-files.c

@@ -89,20 +89,38 @@ static void show_dir_entry(const char *tag, struct dir_entry *ent)
 static void show_other_files(struct dir_struct *dir)
 {
 	int i;
+
+
+	/*
+	 * Skip matching and unmerged entries for the paths,
+	 * since we want just "others".
+	 *
+	 * (Matching entries are normally pruned during
+	 * the directory tree walk, but will show up for
+	 * gitlinks because we don't necessarily have
+	 * dir->show_other_directories set to suppress
+	 * them).
+	 */
 	for (i = 0; i < dir->nr; i++) {
-		/* We should not have a matching entry, but we
-		 * may have an unmerged entry for this path.
-		 */
 		struct dir_entry *ent = dir->entries[i];
-		int pos = cache_name_pos(ent->name, ent->len);
+		int len, pos;
 		struct cache_entry *ce;
+
+		/*
+		 * Remove the '/' at the end that directory
+		 * walking adds for directory entries.
+		 */
+		len = ent->len;
+		if (len && ent->name[len-1] == '/')
+			len--;
+		pos = cache_name_pos(ent->name, len);
 		if (0 <= pos)
-			die("bug in show-other-files");
+			continue;	/* exact match */
 		pos = -pos - 1;
 		if (pos < active_nr) { 
 			ce = active_cache[pos];
-			if (ce_namelen(ce) == ent->len &&
+			if (ce_namelen(ce) == len &&
-			    !memcmp(ce->name, ent->name, ent->len))
+			    !memcmp(ce->name, ent->name, len))
 				continue; /* Yup, this one exists unmerged */
 		}
 		show_dir_entry(tag_other, ent);

builtin-ls-tree.c

@@ -6,6 +6,7 @@
 #include "cache.h"
 #include "blob.h"
 #include "tree.h"
+#include "commit.h"
 #include "quote.h"
 #include "builtin.h"
 
@@ -59,7 +60,24 @@ static int show_tree(const unsigned char *sha1, const char *base, int baselen,
 	int retval = 0;
 	const char *type = blob_type;
 
-	if (S_ISDIR(mode)) {
+	if (S_ISDIRLNK(mode)) {
+		/*
+		 * Maybe we want to have some recursive version here?
+		 *
+		 * Something like:
+		 *
+		if (show_subprojects(base, baselen, pathname)) {
+			if (fork()) {
+				chdir(base);
+				exec ls-tree;
+			}
+			waitpid();
+		}
+		 *
+		 * ..or similar..
+		 */
+		type = commit_type;
+	} else if (S_ISDIR(mode)) {
 		if (show_recursive(base, baselen, pathname)) {
 			retval = READ_TREE_RECURSIVE;
 			if (!(ls_options & LS_SHOW_TREES))

builtin-update-index.c

@@ -9,6 +9,7 @@
 #include "cache-tree.h"
 #include "tree-walk.h"
 #include "builtin.h"
+#include "refs.h"
 
 /*
  * Default to not allowing changes to the list of files. The
@@ -60,76 +61,151 @@ static int mark_valid(const char *path)
 	return -1;
 }
 
-static int process_file(const char *path)
+static int remove_one_path(const char *path)
 {
-	int size, namelen, option, status;
+	if (!allow_remove)
-	struct cache_entry *ce;
+		return error("%s: does not exist and --remove not passed", path);
-	struct stat st;
+	if (remove_file_from_cache(path))
+		return error("%s: cannot remove from the index", path);
+	return 0;
+}
 
-	status = lstat(path, &st);
+/*
+ * Handle a path that couldn't be lstat'ed. It's either:
+ *  - missing file (ENOENT or ENOTDIR). That's ok if we're
+ *    supposed to be removing it and the removal actually
+ *    succeeds.
+ *  - permission error. That's never ok.
+ */
+static int process_lstat_error(const char *path, int err)
+{
+	if (err == ENOENT || err == ENOTDIR)
+		return remove_one_path(path);
+	return error("lstat(\"%s\"): %s", path, strerror(errno));
+}
+
+static int add_one_path(struct cache_entry *old, const char *path, int len, struct stat *st)
+{
+	int option, size = cache_entry_size(len);
+	struct cache_entry *ce = xcalloc(1, size);
+
+	memcpy(ce->name, path, len);
+	ce->ce_flags = htons(len);
+	fill_stat_cache_info(ce, st);
+	ce->ce_mode = ce_mode_from_stat(old, st->st_mode);
+
+	if (index_path(ce->sha1, path, st, !info_only))
+		return -1;
+	option = allow_add ? ADD_CACHE_OK_TO_ADD : 0;
+	option |= allow_replace ? ADD_CACHE_OK_TO_REPLACE : 0;
+	if (add_cache_entry(ce, option))
+		return error("%s: cannot add to the index - missing --add option?", path);
+	return 0;
+}
+
+/*
+ * Handle a path that was a directory. Four cases:
+ *
+ *  - it's already a gitlink in the index, and we keep it that
+ *    way, and update it if we can (if we cannot find the HEAD,
+ *    we're going to keep it unchanged in the index!)
+ *
+ *  - it's a *file* in the index, in which case it should be
+ *    removed as a file if removal is allowed, since it doesn't
+ *    exist as such any more. If removal isn't allowed, it's
+ *    an error.
+ *
+ *    (NOTE! This is old and arguably fairly strange behaviour.
+ *    We might want to make this an error unconditionally, and
+ *    use "--force-remove" if you actually want to force removal).
+ *
+ *  - it used to exist as a subdirectory (ie multiple files with
+ *    this particular prefix) in the index, in which case it's wrong
+ *    to try to update it as a directory.
+ *
+ *  - it doesn't exist at all in the index, but it is a valid
+ *    git directory, and it should be *added* as a gitlink.
+ */
+static int process_directory(const char *path, int len, struct stat *st)
+{
+	unsigned char sha1[20];
+	int pos = cache_name_pos(path, len);
+
+	/* Exact match: file or existing gitlink */
+	if (pos >= 0) {
+		struct cache_entry *ce = active_cache[pos];
+		if (S_ISDIRLNK(ntohl(ce->ce_mode))) {
+
+			/* Do nothing to the index if there is no HEAD! */
+			if (resolve_gitlink_ref(path, "HEAD", sha1) < 0)
+				return 0;
+
+			return add_one_path(ce, path, len, st);
+		}
+		/* Should this be an unconditional error? */
+		return remove_one_path(path);
+	}
+
+	/* Inexact match: is there perhaps a subdirectory match? */
+	pos = -pos-1;
+	while (pos < active_nr) {
+		struct cache_entry *ce = active_cache[pos++];
+
+		if (strncmp(ce->name, path, len))
+			break;
+		if (ce->name[len] > '/')
+			break;
+		if (ce->name[len] < '/')
+			continue;
+
+		/* Subdirectory match - error out */
+		return error("%s: is a directory - add individual files instead", path);
+	}
+
+	/* No match - should we add it as a gitlink? */
+	if (!resolve_gitlink_ref(path, "HEAD", sha1))
+		return add_one_path(NULL, path, len, st);
+
+	/* Error out. */
+	return error("%s: is a directory - add files inside instead", path);
+}
+
+/*
+ * Process a regular file
+ */
+static int process_file(const char *path, int len, struct stat *st)
+{
+	int pos = cache_name_pos(path, len);
+	struct cache_entry *ce = pos < 0 ? NULL : active_cache[pos];
+
+	if (ce && S_ISDIRLNK(ntohl(ce->ce_mode)))
+		return error("%s is already a gitlink, not replacing", path);
+
+	return add_one_path(ce, path, len, st);
+}
+
+static int process_path(const char *path)
+{
+	int len;
+	struct stat st;
 
 	/* We probably want to do this in remove_file_from_cache() and
 	 * add_cache_entry() instead...
 	 */
 	cache_tree_invalidate_path(active_cache_tree, path);
 
-	if (status < 0 || S_ISDIR(st.st_mode)) {
+	/*
-		/* When we used to have "path" and now we want to add
+	 * First things first: get the stat information, to decide
-		 * "path/file", we need a way to remove "path" before
+	 * what to do about the pathname!
-		 * being able to add "path/file".  However,
+	 */
-		 * "git-update-index --remove path" would not work.
+	if (lstat(path, &st) < 0)
-		 * --force-remove can be used but this is more user
+		return process_lstat_error(path, errno);
-		 * friendly, especially since we can do the opposite
-		 * case just fine without --force-remove.
-		 */
-		if (status == 0 || (errno == ENOENT || errno == ENOTDIR)) {
-			if (allow_remove) {
-				if (remove_file_from_cache(path))
-					return error("%s: cannot remove from the index",
-					             path);
-				else
-					return 0;
-			} else if (status < 0) {
-				return error("%s: does not exist and --remove not passed",
-				             path);
-			}
-		}
-		if (0 == status)
-			return error("%s: is a directory - add files inside instead",
-			             path);
-		else
-			return error("lstat(\"%s\"): %s", path,
-				     strerror(errno));
-	}
 
-	namelen = strlen(path);
+	len = strlen(path);
-	size = cache_entry_size(namelen);
+	if (S_ISDIR(st.st_mode))
-	ce = xcalloc(1, size);
+		return process_directory(path, len, &st);
-	memcpy(ce->name, path, namelen);
-	ce->ce_flags = htons(namelen);
-	fill_stat_cache_info(ce, &st);
 
-	if (trust_executable_bit && has_symlinks)
+	return process_file(path, len, &st);
-		ce->ce_mode = create_ce_mode(st.st_mode);
-	else {
-		/* If there is an existing entry, pick the mode bits and type
-		 * from it, otherwise assume unexecutable regular file.
-		 */
-		struct cache_entry *ent;
-		int pos = cache_name_pos(path, namelen);
-
-		ent = (0 <= pos) ? active_cache[pos] : NULL;
-		ce->ce_mode = ce_mode_from_stat(ent, st.st_mode);
-	}
-
-	if (index_path(ce->sha1, path, &st, !info_only))
-		return -1;
-	option = allow_add ? ADD_CACHE_OK_TO_ADD : 0;
-	option |= allow_replace ? ADD_CACHE_OK_TO_REPLACE : 0;
-	if (add_cache_entry(ce, option))
-		return error("%s: cannot add to the index - missing --add option?",
-			     path);
-	return 0;
 }
 
 static int add_cacheinfo(unsigned int mode, const unsigned char *sha1,
@@ -210,8 +286,8 @@ static void update_one(const char *path, const char *prefix, int prefix_length)
 		report("remove '%s'", path);
 		goto free_return;
 	}
-	if (process_file(p))
+	if (process_path(p))
-		die("Unable to process file %s", path);
+		die("Unable to process path %s", path);
 	report("add '%s'", path);
  free_return:
 	if (p < path || p > path + strlen(path))

cache-tree.c

@@ -326,7 +326,7 @@ static int update_one(struct cache_tree *it,
 			mode = ntohl(ce->ce_mode);
 			entlen = pathlen - baselen;
 		}
-		if (!missing_ok && !has_sha1_file(sha1))
+		if (mode != S_IFDIRLNK && !missing_ok && !has_sha1_file(sha1))
 			return error("invalid object %s", sha1_to_hex(sha1));
 
 		if (!ce->ce_mode)

cache.h

@@ -24,6 +24,22 @@
 #define DTYPE(de)	DT_UNKNOWN
 #endif
 
+/*
+ * A "directory link" is a link to another git directory.
+ *
+ * The value 0160000 is not normally a valid mode, and
+ * also just happens to be S_IFDIR + S_IFLNK
+ *
+ * NOTE! We *really* shouldn't depend on the S_IFxxx macros
+ * always having the same values everywhere. We should use
+ * our internal git values for these things, and then we can
+ * translate that to the OS-specific value. It just so
+ * happens that everybody shares the same bit representation
+ * in the UNIX world (and apparently wider too..)
+ */
+#define S_IFDIRLNK	0160000
+#define S_ISDIRLNK(m)	(((m) & S_IFMT) == S_IFDIRLNK)
+
 /*
  * Intensive research over the course of many years has shown that
  * port 9418 is totally unused by anything else. Or
@@ -104,6 +120,8 @@ static inline unsigned int create_ce_mode(unsigned int mode)
 {
 	if (S_ISLNK(mode))
 		return htonl(S_IFLNK);
+	if (S_ISDIR(mode) || S_ISDIRLNK(mode))
+		return htonl(S_IFDIRLNK);
 	return htonl(S_IFREG | ce_permissions(mode));
 }
 static inline unsigned int ce_mode_from_stat(struct cache_entry *ce, unsigned int mode)
@@ -121,7 +139,7 @@ static inline unsigned int ce_mode_from_stat(struct cache_entry *ce, unsigned in
 }
 #define canon_mode(mode) \
 	(S_ISREG(mode) ? (S_IFREG | ce_permissions(mode)) : \
-	S_ISLNK(mode) ? S_IFLNK : S_IFDIR)
+	S_ISLNK(mode) ? S_IFLNK : S_ISDIR(mode) ? S_IFDIR : S_IFDIRLNK)
 
 #define cache_entry_size(len) ((offsetof(struct cache_entry,name) + (len) + 8) & ~7)
 

diff-lib.c

@@ -373,7 +373,7 @@ int run_diff_files(struct rev_info *revs, int silent_on_removed)
 					continue;
 			}
 			else
-				dpath->mode = canon_mode(st.st_mode);
+				dpath->mode = ntohl(ce_mode_from_stat(ce, st.st_mode));
 
 			while (i < entries) {
 				struct cache_entry *nce = active_cache[i];
@@ -390,8 +390,7 @@ int run_diff_files(struct rev_info *revs, int silent_on_removed)
 					int mode = ntohl(nce->ce_mode);
 					num_compare_stages++;
 					hashcpy(dpath->parent[stage-2].sha1, nce->sha1);
-					dpath->parent[stage-2].mode =
+					dpath->parent[stage-2].mode = ntohl(ce_mode_from_stat(nce, mode));
-						canon_mode(mode);
 					dpath->parent[stage-2].status =
 						DIFF_STATUS_MODIFIED;
 				}
@@ -440,15 +439,7 @@ int run_diff_files(struct rev_info *revs, int silent_on_removed)
 		if (!changed && !revs->diffopt.find_copies_harder)
 			continue;
 		oldmode = ntohl(ce->ce_mode);
-
+		newmode = ntohl(ce_mode_from_stat(ce, st.st_mode));
-		newmode = canon_mode(st.st_mode);
-		if (!trust_executable_bit &&
-		    S_ISREG(newmode) && S_ISREG(oldmode) &&
-		    ((newmode ^ oldmode) == 0111))
-			newmode = oldmode;
-		else if (!has_symlinks &&
-		    S_ISREG(newmode) && S_ISLNK(oldmode))
-			newmode = oldmode;
 		diff_change(&revs->diffopt, oldmode, newmode,
 			    ce->sha1, (changed ? null_sha1 : ce->sha1),
 			    ce->name, NULL);

diff.c

@@ -1397,6 +1397,22 @@ static int populate_from_stdin(struct diff_filespec *s)
 	return 0;
 }
 
+static int diff_populate_gitlink(struct diff_filespec *s, int size_only)
+{
+	int len;
+	char *data = xmalloc(100);
+	len = snprintf(data, 100,
+		"Subproject commit %s\n", sha1_to_hex(s->sha1));
+	s->data = data;
+	s->size = len;
+	s->should_free = 1;
+	if (size_only) {
+		s->data = NULL;
+		free(data);
+	}
+	return 0;
+}
+
 /*
  * While doing rename detection and pickaxe operation, we may need to
  * grab the data for the blob (or file) for our own in-core comparison.
@@ -1415,6 +1431,10 @@ int diff_populate_filespec(struct diff_filespec *s, int size_only)
 
 	if (s->data)
 		return err;
+
+	if (S_ISDIRLNK(s->mode))
+		return diff_populate_gitlink(s, size_only);
+
 	if (!s->sha1_valid ||
 	    reuse_worktree_file(s->path, s->sha1, 0)) {
 		struct stat st;

dir.c

@@ -7,12 +7,17 @@
  */
 #include "cache.h"
 #include "dir.h"
+#include "refs.h"
 
 struct path_simplify {
 	int len;
 	const char *path;
 };
 
+static int read_directory_recursive(struct dir_struct *dir,
+	const char *path, const char *base, int baselen,
+	int check_only, const struct path_simplify *simplify);
+
 int common_prefix(const char **pathspec)
 {
 	const char *path, *slash, *next;
@@ -286,15 +291,111 @@ struct dir_entry *dir_add_name(struct dir_struct *dir, const char *pathname, int
 	return ent;
 }
 
-static int dir_exists(const char *dirname, int len)
+enum exist_status {
+	index_nonexistent = 0,
+	index_directory,
+	index_gitdir,
+};
+
+/*
+ * The index sorts alphabetically by entry name, which
+ * means that a gitlink sorts as '\0' at the end, while
+ * a directory (which is defined not as an entry, but as
+ * the files it contains) will sort with the '/' at the
+ * end.
+ */
+static enum exist_status directory_exists_in_index(const char *dirname, int len)
 {
 	int pos = cache_name_pos(dirname, len);
-	if (pos >= 0)
+	if (pos < 0)
-		return 1;
+		pos = -pos-1;
-	pos = -pos-1;
+	while (pos < active_nr) {
-	if (pos >= active_nr) /* can't */
+		struct cache_entry *ce = active_cache[pos++];
-		return 0;
+		unsigned char endchar;
-	return !strncmp(active_cache[pos]->name, dirname, len);
+
+		if (strncmp(ce->name, dirname, len))
+			break;
+		endchar = ce->name[len];
+		if (endchar > '/')
+			break;
+		if (endchar == '/')
+			return index_directory;
+		if (!endchar && S_ISDIRLNK(ntohl(ce->ce_mode)))
+			return index_gitdir;
+	}
+	return index_nonexistent;
+}
+
+/*
+ * When we find a directory when traversing the filesystem, we
+ * have three distinct cases:
+ *
+ *  - ignore it
+ *  - see it as a directory
+ *  - recurse into it
+ *
+ * and which one we choose depends on a combination of existing
+ * git index contents and the flags passed into the directory
+ * traversal routine.
+ *
+ * Case 1: If we *already* have entries in the index under that
+ * directory name, we always recurse into the directory to see
+ * all the files.
+ *
+ * Case 2: If we *already* have that directory name as a gitlink,
+ * we always continue to see it as a gitlink, regardless of whether
+ * there is an actual git directory there or not (it might not
+ * be checked out as a subproject!)
+ *
+ * Case 3: if we didn't have it in the index previously, we
+ * have a few sub-cases:
+ *
+ *  (a) if "show_other_directories" is true, we show it as
+ *      just a directory, unless "hide_empty_directories" is
+ *      also true and the directory is empty, in which case
+ *      we just ignore it entirely.
+ *  (b) if it looks like a git directory, and we don't have
+ *      'no_dirlinks' set we treat it as a gitlink, and show it
+ *      as a directory.
+ *  (c) otherwise, we recurse into it.
+ */
+enum directory_treatment {
+	show_directory,
+	ignore_directory,
+	recurse_into_directory,
+};
+
+static enum directory_treatment treat_directory(struct dir_struct *dir,
+	const char *dirname, int len,
+	const struct path_simplify *simplify)
+{
+	/* The "len-1" is to strip the final '/' */
+	switch (directory_exists_in_index(dirname, len-1)) {
+	case index_directory:
+		return recurse_into_directory;
+
+	case index_gitdir:
+		if (dir->show_other_directories)
+			return ignore_directory;
+		return show_directory;
+
+	case index_nonexistent:
+		if (dir->show_other_directories)
+			break;
+		if (!dir->no_dirlinks) {
+			unsigned char sha1[20];
+			if (resolve_gitlink_ref(dirname, "HEAD", sha1) == 0)
+				return show_directory;
+		}
+		return recurse_into_directory;
+	}
+
+	/* This is the "show_other_directories" case */
+	if (!dir->hide_empty_directories)
+		return show_directory;
+	if (!read_directory_recursive(dir, dirname, dirname, len, 1, simplify))
+		return ignore_directory;
+	return show_directory;
 }
 
 /*
@@ -353,6 +454,9 @@ static int read_directory_recursive(struct dir_struct *dir, const char *path, co
 			     !strcmp(de->d_name + 1, "git")))
 				continue;
 			len = strlen(de->d_name);
+			/* Ignore overly long pathnames! */
+			if (len + baselen + 8 > sizeof(fullname))
+				continue;
 			memcpy(fullname + baselen, de->d_name, len+1);
 			if (simplify_away(fullname, baselen + len, simplify))
 				continue;
@@ -377,19 +481,17 @@ static int read_directory_recursive(struct dir_struct *dir, const char *path, co
 			case DT_DIR:
 				memcpy(fullname + baselen + len, "/", 2);
 				len++;
-				if (dir->show_other_directories &&
+				switch (treat_directory(dir, fullname, baselen + len, simplify)) {
-				    !dir_exists(fullname, baselen + len)) {
+				case show_directory:
-					if (dir->hide_empty_directories &&
-					    !read_directory_recursive(dir,
-						    fullname, fullname,
-						    baselen + len, 1, simplify))
-						continue;
 					break;
+				case recurse_into_directory:
+					contents += read_directory_recursive(dir,
+						fullname, fullname, baselen + len, 0, simplify);
+					continue;
+				case ignore_directory:
+					continue;
 				}
-
+				break;
-				contents += read_directory_recursive(dir,
-					fullname, fullname, baselen + len, 0, simplify);
-				continue;
 			case DT_REG:
 			case DT_LNK:
 				break;

dir.h

@@ -33,7 +33,8 @@ struct dir_struct {
 	int nr, alloc;
 	unsigned int show_ignored:1,
 		     show_other_directories:1,
-		     hide_empty_directories:1;
+		     hide_empty_directories:1,
+		     no_dirlinks:1;
 	struct dir_entry **entries;
 
 	/* Exclude info */

entry.c

@@ -62,26 +62,33 @@ static int create_file(const char *path, unsigned int mode)
 	return open(path, O_WRONLY | O_CREAT | O_EXCL, mode);
 }
 
+static void *read_blob_entry(struct cache_entry *ce, const char *path, unsigned long *size)
+{
+	enum object_type type;
+	void *new = read_sha1_file(ce->sha1, &type, size);
+
+	if (new) {
+		if (type == OBJ_BLOB)
+			return new;
+		free(new);
+	}
+	return NULL;
+}
+
 static int write_entry(struct cache_entry *ce, char *path, struct checkout *state, int to_tempfile)
 {
 	int fd;
-	void *new;
-	unsigned long size;
 	long wrote;
-	enum object_type type;
 
-	new = read_sha1_file(ce->sha1, &type, &size);
-	if (!new || type != OBJ_BLOB) {
-		if (new)
-			free(new);
-		return error("git-checkout-index: unable to read sha1 file of %s (%s)",
-			path, sha1_to_hex(ce->sha1));
-	}
 	switch (ntohl(ce->ce_mode) & S_IFMT) {
-		char *buf;
+		char *buf, *new;
-		unsigned long nsize;
+		unsigned long size, nsize;
 
 	case S_IFREG:
+		new = read_blob_entry(ce, path, &size);
+		if (!new)
+			return error("git-checkout-index: unable to read sha1 file of %s (%s)",
+				path, sha1_to_hex(ce->sha1));
 		if (to_tempfile) {
 			strcpy(path, ".merge_file_XXXXXX");
 			fd = mkstemp(path);
@@ -111,6 +118,10 @@ static int write_entry(struct cache_entry *ce, char *path, struct checkout *stat
 			return error("git-checkout-index: unable to write file %s", path);
 		break;
 	case S_IFLNK:
+		new = read_blob_entry(ce, path, &size);
+		if (!new)
+			return error("git-checkout-index: unable to read sha1 file of %s (%s)",
+				path, sha1_to_hex(ce->sha1));
 		if (to_tempfile || !has_symlinks) {
 			if (to_tempfile) {
 				strcpy(path, ".merge_link_XXXXXX");
@@ -136,8 +147,13 @@ static int write_entry(struct cache_entry *ce, char *path, struct checkout *stat
 						 "symlink %s (%s)", path, strerror(errno));
 		}
 		break;
+	case S_IFDIRLNK:
+		if (to_tempfile)
+			return error("git-checkout-index: cannot create temporary subproject %s", path);
+		if (mkdir(path, 0777) < 0)
+			return error("git-checkout-index: cannot create subproject directory %s", path);
+		break;
 	default:
-		free(new);
 		return error("git-checkout-index: unknown file mode for %s", path);
 	}
 
@@ -179,6 +195,9 @@ int checkout_entry(struct cache_entry *ce, struct checkout *state, char *topath)
 		 */
 		unlink(path);
 		if (S_ISDIR(st.st_mode)) {
+			/* If it is a gitlink, leave it alone! */
+			if (S_ISDIRLNK(ntohl(ce->ce_mode)))
+				return 0;
 			if (!state->force)
 				return error("%s is a directory", path);
 			remove_subtree(path);

list-objects.c

@@ -25,6 +25,37 @@ static void process_blob(struct rev_info *revs,
 	add_object(obj, p, path, name);
 }
 
+/*
+ * Processing a gitlink entry currently does nothing, since
+ * we do not recurse into the subproject.
+ *
+ * We *could* eventually add a flag that actually does that,
+ * which would involve:
+ *  - is the subproject actually checked out?
+ *  - if so, see if the subproject has already been added
+ *    to the alternates list, and add it if not.
+ *  - process the commit (or tag) the gitlink points to
+ *    recursively.
+ *
+ * However, it's unclear whether there is really ever any
+ * reason to see superprojects and subprojects as such a
+ * "unified" object pool (potentially resulting in a totally
+ * humongous pack - avoiding which was the whole point of
+ * having gitlinks in the first place!).
+ *
+ * So for now, there is just a note that we *could* follow
+ * the link, and how to do it. Whether it necessarily makes
+ * any sense what-so-ever to ever do that is another issue.
+ */
+static void process_gitlink(struct rev_info *revs,
+			    const unsigned char *sha1,
+			    struct object_array *p,
+			    struct name_path *path,
+			    const char *name)
+{
+	/* Nothing to do */
+}
+
 static void process_tree(struct rev_info *revs,
 			 struct tree *tree,
 			 struct object_array *p,
@@ -56,6 +87,9 @@ static void process_tree(struct rev_info *revs,
 			process_tree(revs,
 				     lookup_tree(entry.sha1),
 				     p, &me, entry.path);
+		else if (S_ISDIRLNK(entry.mode))
+			process_gitlink(revs, entry.sha1,
+					p, &me, entry.path);
 		else
 			process_blob(revs,
 				     lookup_blob(entry.sha1),

read-cache.c

@@ -5,6 +5,7 @@
  */
 #include "cache.h"
 #include "cache-tree.h"
+#include "refs.h"
 
 /* Index extensions.
  *
@@ -91,6 +92,23 @@ static int ce_compare_link(struct cache_entry *ce, size_t expected_size)
 	return match;
 }
 
+static int ce_compare_gitlink(struct cache_entry *ce)
+{
+	unsigned char sha1[20];
+
+	/*
+	 * We don't actually require that the .git directory
+	 * under DIRLNK directory be a valid git directory. It
+	 * might even be missing (in case nobody populated that
+	 * sub-project).
+	 *
+	 * If so, we consider it always to match.
+	 */
+	if (resolve_gitlink_ref(ce->name, "HEAD", sha1) < 0)
+		return 0;
+	return hashcmp(sha1, ce->sha1);
+}
+
 static int ce_modified_check_fs(struct cache_entry *ce, struct stat *st)
 {
 	switch (st->st_mode & S_IFMT) {
@@ -102,6 +120,9 @@ static int ce_modified_check_fs(struct cache_entry *ce, struct stat *st)
 		if (ce_compare_link(ce, xsize_t(st->st_size)))
 			return DATA_CHANGED;
 		break;
+	case S_IFDIR:
+		if (S_ISDIRLNK(ntohl(ce->ce_mode)))
+			return 0;
 	default:
 		return TYPE_CHANGED;
 	}
@@ -127,6 +148,12 @@ static int ce_match_stat_basic(struct cache_entry *ce, struct stat *st)
 		    (has_symlinks || !S_ISREG(st->st_mode)))
 			changed |= TYPE_CHANGED;
 		break;
+	case S_IFDIRLNK:
+		if (!S_ISDIR(st->st_mode))
+			changed |= TYPE_CHANGED;
+		else if (ce_compare_gitlink(ce))
+			changed |= DATA_CHANGED;
+		return changed;
 	default:
 		die("internal error: ce_mode is %o", ntohl(ce->ce_mode));
 	}
@@ -334,10 +361,14 @@ int add_file_to_cache(const char *path, int verbose)
 	if (lstat(path, &st))
 		die("%s: unable to stat (%s)", path, strerror(errno));
 
-	if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode))
+	if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode) && !S_ISDIR(st.st_mode))
-		die("%s: can only add regular files or symbolic links", path);
+		die("%s: can only add regular files, symbolic links or git-directories", path);
 
 	namelen = strlen(path);
+	if (S_ISDIR(st.st_mode)) {
+		while (namelen && path[namelen-1] == '/')
+			namelen--;
+	}
 	size = cache_entry_size(namelen);
 	ce = xcalloc(1, size);
 	memcpy(ce->name, path, namelen);

refs.c

@@ -283,6 +283,86 @@ static struct ref_list *get_loose_refs(void)
 
 /* We allow "recursive" symbolic refs. Only within reason, though */
 #define MAXDEPTH 5
+#define MAXREFLEN (1024)
+
+static int resolve_gitlink_packed_ref(char *name, int pathlen, const char *refname, unsigned char *result)
+{
+	FILE *f;
+	struct cached_refs refs;
+	struct ref_list *ref;
+	int retval;
+
+	strcpy(name + pathlen, "packed-refs");
+	f = fopen(name, "r");
+	if (!f)
+		return -1;
+	read_packed_refs(f, &refs);
+	fclose(f);
+	ref = refs.packed;
+	retval = -1;
+	while (ref) {
+		if (!strcmp(ref->name, refname)) {
+			retval = 0;
+			memcpy(result, ref->sha1, 20);
+			break;
+		}
+		ref = ref->next;
+	}
+	free_ref_list(refs.packed);
+	return retval;
+}
+
+static int resolve_gitlink_ref_recursive(char *name, int pathlen, const char *refname, unsigned char *result, int recursion)
+{
+	int fd, len = strlen(refname);
+	char buffer[128], *p;
+
+	if (recursion > MAXDEPTH || len > MAXREFLEN)
+		return -1;
+	memcpy(name + pathlen, refname, len+1);
+	fd = open(name, O_RDONLY);
+	if (fd < 0)
+		return resolve_gitlink_packed_ref(name, pathlen, refname, result);
+
+	len = read(fd, buffer, sizeof(buffer)-1);
+	close(fd);
+	if (len < 0)
+		return -1;
+	while (len && isspace(buffer[len-1]))
+		len--;
+	buffer[len] = 0;
+
+	/* Was it a detached head or an old-fashioned symlink? */
+	if (!get_sha1_hex(buffer, result))
+		return 0;
+
+	/* Symref? */
+	if (strncmp(buffer, "ref:", 4))
+		return -1;
+	p = buffer + 4;
+	while (isspace(*p))
+		p++;
+
+	return resolve_gitlink_ref_recursive(name, pathlen, p, result, recursion+1);
+}
+
+int resolve_gitlink_ref(const char *path, const char *refname, unsigned char *result)
+{
+	int len = strlen(path), retval;
+	char *gitdir;
+
+	while (len && path[len-1] == '/')
+		len--;
+	if (!len)
+		return -1;
+	gitdir = xmalloc(len + MAXREFLEN + 8);
+	memcpy(gitdir, path, len);
+	memcpy(gitdir + len, "/.git/", 7);
+
+	retval = resolve_gitlink_ref_recursive(gitdir, len+6, refname, result, 0);
+	free(gitdir);
+	return retval;
+}
 
 const char *resolve_ref(const char *ref, unsigned char *sha1, int reading, int *flag)
 {

refs.h

@@ -60,4 +60,7 @@ extern int check_ref_format(const char *target);
 /** rename ref, return 0 on success **/
 extern int rename_ref(const char *oldref, const char *newref, const char *logmsg);
 
+/** resolve ref in nested "gitlink" repository */
+extern int resolve_gitlink_ref(const char *name, const char *refname, unsigned char *result);
+
 #endif /* REFS_H */

sha1_file.c

@@ -13,6 +13,7 @@
 #include "commit.h"
 #include "tag.h"
 #include "tree.h"
+#include "refs.h"
 
 #ifndef O_NOATIME
 #if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
@@ -2392,6 +2393,8 @@ int index_path(unsigned char *sha1, const char *path, struct stat *st, int write
 				     path);
 		free(target);
 		break;
+	case S_IFDIR:
+		return resolve_gitlink_ref(path, "HEAD", sha1);
 	default:
 		return error("%s: unsupported file type", path);
 	}

t/t3040-subprojects-basic.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+
+test_description='Basic subproject functionality'
+. ./test-lib.sh
+
+test_expect_success 'Super project creation' \
+    ': >Makefile &&
+    git add Makefile &&
+    git commit -m "Superproject created"'
+
+
+cat >expected <<EOF
+:000000 160000 00000... A	sub1
+:000000 160000 00000... A	sub2
+EOF
+test_expect_success 'create subprojects' \
+    'mkdir sub1 &&
+    ( cd sub1 && git init && : >Makefile && git add * &&
+    git commit -q -m "subproject 1" ) &&
+    mkdir sub2 &&
+    ( cd sub2 && git init && : >Makefile && git add * &&
+    git commit -q -m "subproject 2" ) &&
+    git update-index --add sub1 &&
+    git add sub2 &&
+    git commit -q -m "subprojects added" &&
+    git diff-tree --abbrev=5 HEAD^ HEAD |cut -d" " -f-3,5- >current &&
+    git diff expected current'
+
+git branch save HEAD
+
+test_expect_success 'check if fsck ignores the subprojects' \
+    'git fsck --full'
+
+test_expect_success 'check if commit in a subproject detected' \
+    '( cd sub1 &&
+    echo "all:" >>Makefile &&
+    echo "	true" >>Makefile &&
+    git commit -q -a -m "make all" ) && {
+        git diff-files --exit-code
+	test $? = 1
+    }'
+
+test_expect_success 'check if a changed subproject HEAD can be committed' \
+    'git commit -q -a -m "sub1 changed" && {
+	git diff-tree --exit-code HEAD^ HEAD
+	test $? = 1
+    }'
+
+test_expect_success 'check if diff-index works for subproject elements' \
+    'git diff-index --exit-code --cached save -- sub1
+    test $? = 1'
+
+test_expect_success 'check if diff-tree works for subproject elements' \
+    'git diff-tree --exit-code HEAD^ HEAD -- sub1
+    test $? = 1'
+
+test_expect_success 'check if git diff works for subproject elements' \
+    'git diff --exit-code HEAD^ HEAD
+    test $? = 1'
+
+test_expect_success 'check if clone works' \
+    'git ls-files -s >expected &&
+    git clone -l -s . cloned &&
+    ( cd cloned && git ls-files -s ) >current &&
+    git diff expected current'
+
+test_expect_success 'removing and adding subproject' \
+    'git update-index --force-remove -- sub2 &&
+    mv sub2 sub3 &&
+    git add sub3 &&
+    git commit -q -m "renaming a subproject" && {
+	git diff -M --name-status --exit-code HEAD^ HEAD
+	test $? = 1
+    }'
+
+# the index must contain the object name the HEAD of the
+# subproject sub1 was at the point "save"
+test_expect_success 'checkout in superproject' \
+    'git checkout save &&
+    git diff-index --exit-code --raw --cached save -- sub1'
+
+# just interesting what happened...
+# git diff --name-status -M save master
+
+test_done

tree.c

@@ -143,6 +143,14 @@ struct tree *lookup_tree(const unsigned char *sha1)
 	return (struct tree *) obj;
 }
 
+/*
+ * NOTE! Tree refs to external git repositories
+ * (ie gitlinks) do not count as real references.
+ *
+ * You don't have to have those repositories
+ * available at all, much less have the objects
+ * accessible from the current repository.
+ */
 static void track_tree_refs(struct tree *item)
 {
 	int n_refs = 0, i;
@@ -152,8 +160,11 @@ static void track_tree_refs(struct tree *item)
 
 	/* Count how many entries there are.. */
 	init_tree_desc(&desc, item->buffer, item->size);
-	while (tree_entry(&desc, &entry))
+	while (tree_entry(&desc, &entry)) {
+		if (S_ISDIRLNK(entry.mode))
+			continue;
 		n_refs++;
+	}
 
 	/* Allocate object refs and walk it again.. */
 	i = 0;
@@ -162,6 +173,8 @@ static void track_tree_refs(struct tree *item)
 	while (tree_entry(&desc, &entry)) {
 		struct object *obj;
 
+		if (S_ISDIRLNK(entry.mode))
+			continue;
 		if (S_ISDIR(entry.mode))
 			obj = &lookup_tree(entry.sha1)->object;
 		else