Start preparing the API documents.

Most of them are still stubs, but the procedure to build the HTML
documentation, maintaining the index and installing the end product are
there.

I placed names of people who are likely to know the most about the topic
in the stub files, so that volunteers will know whom to ask questions as
needed.

Signed-off-by: Junio C Hamano <gitster@pobox.com>

Documentation/.gitignore

@@ -2,6 +2,7 @@
 *.html
 *.[1-8]
 *.made
+git.info
 howto-index.txt
 doc.dep
 cmds-*.txt

Documentation/Makefile

@@ -24,6 +24,9 @@ ARTICLES += git-tools
 ARTICLES += glossary
 # with their own formatting rules.
 SP_ARTICLES = howto/revert-branch-rebase user-manual
+API_DOCS = $(patsubst %.txt,%,$(filter-out technical/api-index-skel.txt technical/api-index.txt, $(wildcard technical/api-*.txt)))
+SP_ARTICLES += $(API_DOCS)
+SP_ARTICLES += technical/api-index
 
 DOC_HTML += $(patsubst %,%.html,$(ARTICLES) $(SP_ARTICLES))
 
@@ -142,10 +145,12 @@ cmd-list.made: cmd-list.perl ../command-list.txt $(MAN1_TXT)
 git.7 git.html: git.txt
 
 clean:
-	$(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 *.texi *.texi+ howto-index.txt howto/*.html doc.dep
+	$(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 *.texi *.texi+ git.info
+	$(RM) howto-index.txt howto/*.html doc.dep
+	$(RM) technical/api-*.html technical/api-index.txt
 	$(RM) $(cmds_txt) *.made
 
-%.html : %.txt
+$(MAN_HTML): %.html : %.txt
 	$(RM) $@+ $@
 	$(ASCIIDOC) -b xhtml11 -d manpage -f asciidoc.conf \
 		$(ASCIIDOC_EXTRA) -agit_version=$(GIT_VERSION) -o $@+ $<
@@ -164,6 +169,14 @@ clean:
 user-manual.xml: user-manual.txt user-manual.conf
 	$(ASCIIDOC) -b docbook -d book $<
 
+technical/api-index.txt: technical/api-index-skel.txt \
+	technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS))
+	cd technical && sh ./api-index.sh
+
+$(patsubst %,%.html,$(API_DOCS) technical/api-index): %.html : %.txt
+	$(ASCIIDOC) -b xhtml11 -f asciidoc.conf \
+		$(ASCIIDOC_EXTRA) -agit_version=$(GIT_VERSION) $*.txt
+
 XSLT = docbook.xsl
 XSLTOPTS = --xinclude --stringparam html.stylesheet docbook-xsl.css
 

Documentation/git.txt

@@ -153,6 +153,8 @@ introductions to the underlying git architecture.
 See also the link:howto-index.html[howto] documents for some useful
 examples.
 
+The internals are documented link:technical/api-index.html[here].
+
 GIT COMMANDS
 ------------
 

Documentation/install-webdoc.sh

@@ -2,9 +2,16 @@
 
 T="$1"
 
-for h in *.html *.txt howto/*.txt howto/*.html RelNotes-*.txt *.css
+for h in \
+	*.txt *.html \
+	howto/*.txt howto/*.html \
+	technical/*.txt technical/*.html \
+	RelNotes-*.txt *.css
 do
-	if test -f "$T/$h" &&
+	if test ! -f "$h"
+	then
+		: did not match
+	elif test -f "$T/$h" &&
 	   diff -u -I'Last updated [0-9][0-9]-[A-Z][a-z][a-z]-' "$T/$h" "$h"
 	then
 		:; # up to date
@@ -16,7 +23,10 @@ do
 	fi
 done
 strip_leading=`echo "$T/" | sed -e 's|.|.|g'`
-for th in "$T"/*.html "$T"/*.txt "$T"/howto/*.txt "$T"/howto/*.html
+for th in \
+	"$T"/*.html "$T"/*.txt \
+	"$T"/howto/*.txt "$T"/howto/*.html \
+	"$T"/technical/*.txt "$T"/technical/*.html
 do
 	h=`expr "$th" : "$strip_leading"'\(.*\)'`
 	case "$h" in

Documentation/technical/.gitignore

@@ -0,0 +1 @@
+api-index.txt

Documentation/technical/api-allocation-growing.txt

@@ -0,0 +1,34 @@
+allocation growing API
+======================
+
+Dynamically growing an array using realloc() is error prone and boring.
+
+Define your array with:
+
+* a pointer (`ary`) that points at the array, initialized to `NULL`;
+
+* an integer variable (`alloc`) that keeps track of how big the current
+  allocation is, initialized to `0`;
+
+* another integer variable (`nr`) to keep track of how many elements the
+  array currently has, initialized to `0`.
+
+Then before adding `n`th element to the array, call `ALLOC_GROW(ary, n,
+alloc)`.  This ensures that the array can hold at least `n` elements by
+calling `realloc(3)` and adjusting `alloc` variable.
+
+------------
+sometype *ary;
+size_t nr;
+size_t alloc
+
+for (i = 0; i < nr; i++)
+	if (we like ary[i] already)
+		return;
+
+/* we did not like any existing one, so add one */
+ALLOC_GROW(ary, nr + 1, alloc);
+ary[nr++] = value you like;
+------------
+
+You are responsible for updating the `nr` variable.

Documentation/technical/api-builtin.txt

@@ -0,0 +1,63 @@
+builtin API
+===========
+
+Adding a new built-in
+---------------------
+
+There are 4 things to do to add a bulit-in command implementation to
+git:
+
+. Define the implementation of the built-in command `foo` with
+  signature:
+
+	int cmd_foo(int argc, const char **argv, const char *prefix);
+
+. Add the external declaration for the function to `builtin.h`.
+
+. Add the command to `commands[]` table in `handle_internal_command()`,
+  defined in `git.c`.  The entry should look like:
+
+	{ "foo", cmd_foo, <options> },
+
+  where options is the bitwise-or of:
+
+`RUN_SETUP`::
+
+	Make sure there is a git directory to work on, and if there is a
+	work tree, chdir to the top of it if the command was invoked
+	in a subdirectory.  If there is no work tree, no chdir() is
+	done.
+
+`USE_PAGER`::
+
+	If the standard output is connected to a tty, spawn a pager and
+	feed our output to it.
+
+. Add `builtin-foo.o` to `BUILTIN_OBJS` in `Makefile`.
+
+Additionally, if `foo` is a new command, there are 3 more things to do:
+
+. Add tests to `t/` directory.
+
+. Write documentation in `Documentation/git-foo.txt`.
+
+. Add an entry for `git-foo` to the list at the end of
+  `Documentation/cmd-list.perl`.
+
+
+How a built-in is called
+------------------------
+
+The implementation `cmd_foo()` takes three parameters, `argc`, `argv,
+and `prefix`.  The first two are similar to what `main()` of a
+standalone command would be called with.
+
+When `RUN_SETUP` is specified in the `commands[]` table, and when you
+were started from a subdirectory of the work tree, `cmd_foo()` is called
+after chdir(2) to the top of the work tree, and `prefix` gets the path
+to the subdirectory the command started from.  This allows you to
+convert a user-supplied pathname (typically relative to that directory)
+to a pathname relative to the top of the work tree.
+
+The return value from `cmd_foo()` becomes the exit status of the
+command.

Documentation/technical/api-decorate.txt

@@ -0,0 +1,6 @@
+decorate API
+============
+
+Talk about <decorate.h>
+
+(Linus)

Documentation/technical/api-diff.txt

@@ -0,0 +1,166 @@
+diff API
+========
+
+The diff API is for programs that compare two sets of files (e.g. two
+trees, one tree and the index) and present the found difference in
+various ways.  The calling program is responsible for feeding the API
+pairs of files, one from the "old" set and the corresponding one from
+"new" set, that are different.  The library called through this API is
+called diffcore, and is responsible for two things.
+
+* finding total rewrites (`-B`), renames (`-M`) and copies (`-C`), and
+  changes that touch a string (`-S`), as specified by the caller.
+
+* outputting the differences in various formats, as specified by the
+  caller.
+
+Calling sequence
+----------------
+
+* Prepare `struct diff_options` to record the set of diff options, and
+  then call `diff_setup()` to initialize this structure.  This sets up
+  the vanilla default.
+
+* Fill in the options structure to specify desired output format, rename
+  detection, etc.  `diff_opt_parse()` can be used to parse options given
+  from the command line in a way consistent with existing git-diff
+  family of programs.
+
+* Call `diff_setup_done()`; this inspects the options set up so far for
+  internal consistency and make necessary tweaking to it (e.g. if
+  textual patch output was asked, recursive behaviour is turned on).
+
+* As you find different pairs of files, call `diff_change()` to feed
+  modified files, `diff_addremove()` to feed created or deleted files,
+  or `diff_unmerged()` to feed a file whose state is 'unmerged' to the
+  API.  These are thin wrappers to a lower-level `diff_queue()` function
+  that is flexible enough to record any of these kinds of changes.
+
+* Once you finish feeding the pairs of files, call `diffcore_std()`.
+  This will tell the diffcore library to go ahead and do its work.
+
+* Calling `diffcore_flush()` will produce the output.
+
+
+Data structures
+---------------
+
+* `struct diff_filespec`
+
+This is the internal representation for a single file (blob).  It
+records the blob object name (if known -- for a work tree file it
+typically is a NUL SHA-1), filemode and pathname.  This is what the
+`diff_addremove()`, `diff_change()` and `diff_unmerged()` synthesize and
+feed `diff_queue()` function with.
+
+* `struct diff_filepair`
+
+This records a pair of `struct diff_filespec`; the filespec for a file
+in the "old" set (i.e. preimage) is called `one`, and the filespec for a
+file in the "new" set (i.e. postimage) is called `two`.  A change that
+represents file creation has NULL in `one`, and file deletion has NULL
+in `two`.
+
+A `filepair` starts pointing at `one` and `two` that are from the same
+filename, but `diffcore_std()` can break pairs and match component
+filespecs with other filespecs from a different filepair to form new
+filepair.  This is called 'rename detection'.
+
+* `struct diff_queue`
+
+This is a collection of filepairs.  Notable members are:
+
+`queue`::
+
+	An array of pointers to `struct diff_filepair`.  This
+	dynamically grows as you add filepairs;
+
+`alloc`::
+
+	The allocated size of the `queue` array;
+
+`nr`::
+
+	The number of elements in the `queue` array.
+
+
+* `struct diff_options`
+
+This describes the set of options the calling program wants to affect
+the operation of diffcore library with.
+
+Notable members are:
+
+`output_format`::
+	The output format used when `diff_flush()` is run.
+
+`context`::
+	Number of context lines to generate in patch output.
+
+`break_opt`, `detect_rename`, `rename-score`, `rename_limit`::
+	Affects the way detection logic for complete rewrites, renames
+	and copies.
+
+`abbrev`::
+	Number of hexdigits to abbrevate raw format output to.
+
+`pickaxe`::
+	A constant string (can and typically does contain newlines to
+	look for a block of text, not just a single line) to filter out
+	the filepairs that do not change the number of strings contained
+	in its preimage and postmage of the diff_queue.
+
+`flags`::
+	This is mostly a collection of boolean options that affects the
+	operation, but some do not have anything to do with the diffcore
+	library.
+
+BINARY, TEXT;;
+	Affects the way how a file that is seemingly binary is treated.
+
+FULL_INDEX;;
+	Tells the patch output format not to use abbreviated object
+	names on the "index" lines.
+
+FIND_COPIES_HARDER;;
+	Tells the diffcore library that the caller is feeding unchanged
+	filepairs to allow copies from unmodified files be detected.
+
+COLOR_DIFF;;
+	Output should be colored.
+
+COLOR_DIFF_WORDS;;
+	Output is a colored word-diff.
+
+NO_INDEX;;
+	Tells diff-files that the input is not tracked files but files
+	in random locations on the filesystem.
+
+ALLOW_EXTERNAL;;
+	Tells output routine that it is Ok to call user specified patch
+	output routine.  Plumbing disables this to ensure stable output.
+
+QUIET;;
+	Do not show any output.
+
+REVERSE_DIFF;;
+	Tells the library that the calling program is feeding the
+	filepairs reversed; `one` is two, and `two` is one.
+
+EXIT_WITH_STATUS;;
+	For communication between the calling program and the options
+	parser; tell the calling program to signal the presense of
+	difference using program exit code.
+
+HAS_CHANGES;;
+	Internal; used for optimization to see if there is any change.
+
+SILENT_ON_REMOVE;;
+	Affects if diff-files shows removed files.
+
+RECURSIVE, TREE_IN_RECURSIVE;;
+	Tells if tree traversal done by tree-diff should recursively
+	descend into a tree object pair that are different in preimage
+	and postimage set.
+
+(JC)

Documentation/technical/api-directory-listing.txt

@@ -0,0 +1,76 @@
+directory listing API
+=====================
+
+The directory listing API is used to enumerate paths in the work tree,
+optionally taking `.git/info/exclude` and `.gitignore` files per
+directory into account.
+
+Data structure
+--------------
+
+`struct dir_struct` structure is used to pass directory traversal
+options to the library and to record the paths discovered.  The notable
+options are:
+
+`exclude_per_dir`::
+
+	The name of the file to be read in each directory for excluded
+	files (typically `.gitignore`).
+
+`collect_ignored`::
+
+	Include paths that are to be excluded in the result.
+
+`show_ignored`::
+
+	The traversal is for finding just ignored files, not unignored
+	files.
+
+`show_other_directories`::
+
+	Include a directory that is not tracked.
+
+`hide_empty_directories`::
+
+	Do not include a directory that is not tracked and is empty.
+
+`no_gitlinks`::
+
+	If set, recurse into a directory that looks like a git
+	directory.  Otherwise it is shown as a directory.
+
+The result of the enumeration is left in these fields::
+
+`entries[]`::
+
+	An array of `struct dir_entry`, each element of which describes
+	a path.
+
+`nr`::
+
+	The number of members in `entries[]` array.
+
+`alloc`::
+
+	Internal use; keeps track of allocation of `entries[]` array.
+
+
+Calling sequence
+----------------
+
+* Prepare `struct dir_struct dir` and clear it with `memset(&dir, 0,
+  sizeof(dir))`.
+
+* Call `add_exclude()` to add single exclude pattern,
+  `add_excludes_from_file()` to add patterns from a file
+  (e.g. `.git/info/exclude`), and/or set `dir.exclude_per_dir`.  A
+  short-hand function `setup_standard_excludes()` can be used to set up
+  the standard set of exclude settings.
+
+* Set options described in the Data Structure section above.
+
+* Call `read_directory()`.
+
+* Use `dir.entries[]`.
+
+(JC)

Documentation/technical/api-gitattributes.txt

@@ -0,0 +1,111 @@
+gitattributes API
+=================
+
+gitattributes mechanism gives a uniform way to associate various
+attributes to set of paths.
+
+
+Data Structure
+--------------
+
+`struct git_attr`::
+
+	An attribute is an opaque object that is identified by its name.
+	Pass the name and its length to `git_attr()` function to obtain
+	the object of this type.  The internal representation of this
+	structure is of no interest to the calling programs.
+
+`struct git_attr_check`::
+
+	This structure represents a set of attributes to check in a call
+	to `git_checkattr()` function, and receives the results.
+
+
+Calling Sequence
+----------------
+
+* Prepare an array of `struct git_attr_check` to define the list of
+  attributes you would want to check.  To populate this array, you would
+  need to define necessary attributes by calling `git_attr()` function.
+
+* Call git_checkattr() to check the attributes for the path.
+
+* Inspect `git_attr_check` structure to see how each of the attribute in
+  the array is defined for the path.
+
+
+Attribute Values
+----------------
+
+An attribute for a path can be in one of four states: Set, Unset,
+Unspecified or set to a string, and `.value` member of `struct
+git_attr_check` records it.  There are three macros to check these:
+
+`ATTR_TRUE()`::
+
+	Returns true if the attribute is Set for the path.
+
+`ATTR_FALSE()`::
+
+	Returns true if the attribute is Unset for the path.
+
+`ATTR_UNSET()`::
+
+	Returns true if the attribute is Unspecified for the path.
+
+If none of the above returns true, `.value` member points at a string
+value of the attribute for the path.
+
+
+Example
+-------
+
+To see how attributes "crlf" and "indent" are set for different paths.
+
+. Prepare an array of `struct git_attr_check` with two elements (because
+  we are checking two attributes).  Initialize their `attr` member with
+  pointers to `struct git_attr` obtained by calling `git_attr()`:
+
+------------
+static struct git_attr_check check[2];
+static void setup_check(void)
+{
+	if (check[0].attr)
+		return; /* already done */
+	check[0].attr = git_attr("crlf", 4);
+	check[1].attr = git_attr("ident", 5);
+}
+------------
+
+. Call `git_checkattr()` with the prepared array of `struct git_attr_check`:
+
+------------
+	const char *path;
+
+	setup_check();
+	git_checkattr(path, ARRAY_SIZE(check), check);
+------------
+
+. Act on `.value` member of the result, left in `check[]`:
+
+------------
+	const char *value = check[0].value;
+
+	if (ATTR_TRUE(value)) {
+		The attribute is Set, by listing only the name of the
+		attribute in the gitattributes file for the path.
+	} else if (ATTR_FALSE(value)) {
+		The attribute is Unset, by listing the name of the
+		attribute prefixed with a dash - for the path.
+	} else if (ATTR_UNSET(value)) {
+		The attribute is not set nor unset for the path.
+	} else if (!strcmp(value, "input")) {
+		If none of ATTR_TRUE(), ATTR_FALSE(), or ATTR_UNSET() is
+		true, the value is a string set in the gitattributes
+		file for the path by saying "attr=value".
+	} else if (... other check using value as string ...) {
+		...
+	}
+------------
+
+(JC)

Documentation/technical/api-grep.txt

@@ -0,0 +1,8 @@
+grep API
+========
+
+Talk about <grep.h>, things like:
+
+* grep_buffer()
+
+(JC)

Documentation/technical/api-hash.txt

@@ -0,0 +1,6 @@
+hash API
+========
+
+Talk about <hash.h>
+
+(Linus)

Documentation/technical/api-in-core-index.txt

@@ -0,0 +1,21 @@
+in-core index API
+=================
+
+Talk about <read-cache.c> and <cache-tree.c>, things like:
+
+* cache -> the_index macros
+* read_index()
+* write_index()
+* ie_match_stat() and ie_modified(); how they are different and when to
+  use which.
+* index_name_pos()
+* remove_index_entry_at()
+* remove_file_from_index()
+* add_file_to_index()
+* add_index_entry()
+* refresh_index()
+* discard_index()
+* cache_tree_invalidate_path()
+* cache_tree_update()
+
+(JC, Linus)

Documentation/technical/api-index-skel.txt

@@ -0,0 +1,15 @@
+GIT API Documents
+=================
+
+GIT has grown a set of internal API over time.  This collection
+documents them.
+
+////////////////////////////////////////////////////////////////
+// table of contents begin
+////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////
+// table of contents end
+////////////////////////////////////////////////////////////////
+
+2007-11-24

Documentation/technical/api-index.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+(
+	c=////////////////////////////////////////////////////////////////
+	skel=api-index-skel.txt
+	sed -e '/^\/\/ table of contents begin/q' "$skel"
+	echo "$c"
+
+	ls api-*.txt |
+	while read filename
+	do
+		case "$filename" in
+		api-index-skel.txt | api-index.txt) continue ;;
+		esac
+		title=$(sed -e 1q "$filename")
+		html=${filename%.txt}.html
+		echo "* link:$html[$title]"
+	done
+	echo "$c"
+	sed -n -e '/^\/\/ table of contents end/,$p' "$skel"
+) >api-index.txt+
+
+if test -f api-index.txt && cmp api-index.txt api-index.txt+ >/dev/null
+then
+	rm -f api-index.txt+
+else
+	mv api-index.txt+ api-index.txt
+fi

Documentation/technical/api-lockfile.txt

@@ -0,0 +1,12 @@
+lockfile API
+============
+
+Talk about <lockfile.c>, things like:
+
+* lockfile lifetime -- atexit(3) looks at them, do not put them on the
+  stack;
+* hold_lock_file_for_update()
+* commit_lock_file()
+* rollback_rock_file()
+
+(JC, Dscho, Shawn)

Documentation/technical/api-object-access.txt

@@ -0,0 +1,15 @@
+object access API
+=================
+
+Talk about <sha1_file.c> and <object.h> family, things like
+
+* read_sha1_file()
+* read_object_with_reference()
+* has_sha1_file()
+* write_sha1_file()
+* pretend_sha1_file()
+* lookup_{object,commit,tag,blob,tree}
+* parse_{object,commit,tag,blob,tree}
+* Use of object flags
+
+(JC, Shawn, Daniel, Dscho, Linus)

Documentation/technical/api-parse-options.txt

@@ -0,0 +1,6 @@
+parse-options API
+=================
+
+Talk about <parse-options.h>
+
+(Pierre)

Documentation/technical/api-path-list.txt

@@ -0,0 +1,9 @@
+path-list API
+=============
+
+Talk about <path-list.h>, things like
+
+* it is not just paths but strings in general;
+* the calling sequence.
+
+(Dscho)

Documentation/technical/api-quote.txt

@@ -0,0 +1,10 @@
+quote API
+=========
+
+Talk about <quote.h>, things like
+
+* sq_quote and unquote
+* c_style quote and unquote
+* quoting for foreign languages
+
+(JC)

Documentation/technical/api-revision-walking.txt

@@ -0,0 +1,9 @@
+revision walking API
+====================
+
+Talk about <revision.h>, things like:
+
+* two diff_options, one for path limiting, another for output;
+* calling sequence: init_revisions(), setup_revsions(), get_revision();
+
+(Linus, JC, Dscho)

Documentation/technical/api-run-command.txt

@@ -0,0 +1,10 @@
+run-command API
+===============
+
+Talk about <run-command.h>, and things like:
+
+* Environment the command runs with (e.g. GIT_DIR);
+* File descriptors and pipes;
+* Exit status;
+
+(Hannes, Dscho, Shawn)

Documentation/technical/api-setup.txt

@@ -0,0 +1,13 @@
+setup API
+=========
+
+Talk about
+
+* setup_git_directory()
+* setup_git_directory_gently()
+* is_inside_git_dir()
+* is_inside_work_tree()
+* setup_work_tree()
+* get_pathspec()
+
+(Dscho)

Documentation/technical/api-strbuf.txt

@@ -0,0 +1,6 @@
+strbuf API
+==========
+
+Talk about <strbuf.h>
+
+(Pierre, JC)

Documentation/technical/api-tree-walking.txt

@@ -0,0 +1,12 @@
+tree walking API
+================
+
+Talk about <tree-walk.h>, things like
+
+* struct tree_desc
+* init_tree_desc
+* tree_entry_extract
+* update_tree_entry
+* get_tree_entry
+
+(JC, Linus)

Documentation/technical/api-xdiff-interface.txt

@@ -0,0 +1,7 @@
+xdiff interface API
+===================
+
+Talk about our calling convention to xdiff library, including
+xdiff_emit_consume_fn.
+
+(Dscho, JC)