Make trailing LF optional for all fast-import commands

For the same reasons as the prior change we want to allow frontends
to omit the trailing LF that usually delimits commands.  In some
cases these just make the input stream more verbose looking than
it needs to be, and its just simpler for the frontend developer to
get started if our parser is slightly more lenient about where an
LF is required and where it isn't.

To make this optional LF feature work we now have to buffer up to one
line of input in command_buf.  This buffering can happen if we look
at the current input command but don't recognize it at this point
in the code.  In such a case we need to "unget" the entire line,
but we cannot depend upon the stdio library to let us do ungetc()
for that many characters at once.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Documentation/git-fast-import.txt

@@ -312,7 +312,7 @@ change to the project.
 	('from' SP <committish> LF)?
 	('merge' SP <committish> LF)?
 	(filemodify | filedelete | filecopy | filerename | filedeleteall)*
-	LF
+	LF?
 ....
 
 where `<ref>` is the name of the branch to make the commit on.
@@ -343,6 +343,8 @@ all `filemodify`, `filecopy` and `filerename` commands in the same
 commit, as `filedeleteall`
 wipes the branch clean (see below).
 
+The `LF` after the command is optional (it used to be required).
+
 `author`
 ^^^^^^^^
 An `author` command may optionally appear, if the author information
@@ -654,12 +656,14 @@ branch from an existing commit without creating a new commit.
 ....
 	'reset' SP <ref> LF
 	('from' SP <committish> LF)?
-	LF
+	LF?
 ....
 
 For a detailed description of `<ref>` and `<committish>` see above
 under `commit` and `from`.
 
+The `LF` after the command is optional (it used to be required).
+
 The `reset` command can also be used to create lightweight
 (non-annotated) tags.  For example:
 
@@ -750,7 +754,7 @@ save out all current branch refs, tags and marks.
 
 ....
 	'checkpoint' LF
-	LF
+	LF?
 ....
 
 Note that fast-import automatically switches packfiles when the current
@@ -769,6 +773,7 @@ process access to a branch.  However given that a 30 GiB Subversion
 repository can be loaded into Git through fast-import in about 3 hours,
 explicit checkpointing may not be necessary.
 
+The `LF` after the command is optional (it used to be required).
 
 Tips and Tricks
 ---------------

fast-import.c

@@ -23,7 +23,7 @@ Format of STDIN stream:
     ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
     ('merge' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)*
     file_change*
-    lf;
+    lf?;
   commit_msg ::= data;
 
   file_change ::= file_clr
@@ -48,10 +48,10 @@ Format of STDIN stream:
 
   reset_branch ::= 'reset' sp ref_str lf
     ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
-    lf;
+    lf?;
 
   checkpoint ::= 'checkpoint' lf
-    lf;
+    lf?;
 
      # note: the first idnum in a stream should be 1 and subsequent
      # idnums should not have gaps between values as this will cause
@@ -330,6 +330,7 @@ static struct tag *last_tag;
 /* Input stream parsing */
 static whenspec_type whenspec = WHENSPEC_RAW;
 static struct strbuf command_buf;
+static int unread_command_buf;
 static uintmax_t next_mark;
 static struct dbuf new_data;
 
@@ -1466,7 +1467,10 @@ static void dump_marks(void)
 static void read_next_command(void)
 {
 	do {
-		read_line(&command_buf, stdin, '\n');
+		if (unread_command_buf)
+			unread_command_buf = 0;
+		else
+			read_line(&command_buf, stdin, '\n');
 	} while (!command_buf.eof && command_buf.buf[0] == '#');
 }
 
@@ -1825,13 +1829,13 @@ static void cmd_from_existing(struct branch *b)
 	}
 }
 
-static void cmd_from(struct branch *b)
+static int cmd_from(struct branch *b)
 {
 	const char *from;
 	struct branch *s;
 
 	if (prefixcmp(command_buf.buf, "from "))
-		return;
+		return 0;
 
 	if (b->branch_tree.tree) {
 		release_tree_content_recursive(b->branch_tree.tree);
@@ -1866,6 +1870,7 @@ static void cmd_from(struct branch *b)
 		die("Invalid ref name or SHA1 expression: %s", from);
 
 	read_next_command();
+	return 1;
 }
 
 static struct hash_list *cmd_merge(unsigned int *count)
@@ -1950,10 +1955,8 @@ static void cmd_new_commit(void)
 	}
 
 	/* file_change* */
-	for (;;) {
+	while (!command_buf.eof && command_buf.len > 1) {
-		if (1 == command_buf.len)
+		if (!prefixcmp(command_buf.buf, "M "))
-			break;
-		else if (!prefixcmp(command_buf.buf, "M "))
 			file_change_m(b);
 		else if (!prefixcmp(command_buf.buf, "D "))
 			file_change_d(b);
@@ -1963,8 +1966,10 @@ static void cmd_new_commit(void)
 			file_change_cr(b, 0);
 		else if (!strcmp("deleteall", command_buf.buf))
 			file_change_deleteall(b);
-		else
+		else {
-			die("Unsupported file_change: %s", command_buf.buf);
+			unread_command_buf = 1;
+			break;
+		}
 		read_next_command();
 	}
 
@@ -2105,7 +2110,8 @@ static void cmd_reset_branch(void)
 	else
 		b = new_branch(sp);
 	read_next_command();
-	cmd_from(b);
+	if (!cmd_from(b) && command_buf.len > 1)
+		unread_command_buf = 1;
 }
 
 static void cmd_checkpoint(void)
@@ -2116,7 +2122,7 @@ static void cmd_checkpoint(void)
 		dump_tags();
 		dump_marks();
 	}
-	read_next_command();
+	skip_optional_lf();
 }
 
 static void import_marks(const char *input_file)

t/t9300-fast-import.sh

@@ -839,4 +839,50 @@ test_expect_success \
 	'git-fast-import <input &&
 	 test `git-rev-parse N3` = `git-rev-parse O2`'
 
+test_expect_success \
+	'O: repack before next test' \
+	'git repack -a -d'
+
+cat >input <<INPUT_END
+commit refs/heads/O3
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zstring
+COMMIT
+commit refs/heads/O3
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zof
+COMMIT
+checkpoint
+commit refs/heads/O3
+mark :5
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zempty
+COMMIT
+checkpoint
+commit refs/heads/O3
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zcommits
+COMMIT
+reset refs/tags/O3-2nd
+from :5
+INPUT_END
+
+cat >expect <<INPUT_END
+string
+of
+empty
+commits
+INPUT_END
+test_expect_success \
+	'O: blank lines not necessary after other commands' \
+	'git-fast-import <input &&
+	 test 8 = `find .git/objects/pack -type f | wc -l` &&
+	 test `git rev-parse refs/tags/O3-2nd` = `git rev-parse O3^` &&
+	 git log --reverse --pretty=oneline O3 | sed s/^.*z// >actual &&
+	 git diff expect actual'
+
 test_done