Add public API for pathspec matching

This adds a new public API for compiling pathspecs and matching
them against the working directory, the index, or a tree from the
repository.  This also reworks the pathspec internals to allow the
sharing of code between the existing internal usage of pathspec
matching and the new external API.

While this is working and the new API is ready for discussion, I
think there is still an incorrect behavior in which patterns are
always matched against the full path of an entry without taking
the subdirectories into account (so "s*" will match "subdir/file"
even though it wouldn't with core Git).  Further enhancements are
coming, but this was a good place to take a functional snapshot.

include/git2/index.h

@@ -138,6 +138,14 @@ typedef enum {
 	GIT_INDEX_ADD_CHECK_PATHSPEC = (1u << 2),
 } git_index_add_option_t;
 
+/**
+ * Match any index stage.
+ *
+ * Some index APIs take a stage to match; pass this value to match
+ * any entry matching the path regardless of stage.
+ */
+#define GIT_INDEX_STAGE_ANY -1
+
 /** @name Index File Functions
  *
  * These functions work on the index file itself.

include/git2/pathspec.h

+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_git_pathspec_h__
+#define INCLUDE_git_pathspec_h__
+
+#include "common.h"
+#include "types.h"
+#include "strarray.h"
+
+/**
+ * Compiled pathspec
+ */
+typedef struct git_pathspec git_pathspec;
+
+/**
+ * List of filenames matching a pathspec
+ */
+typedef struct git_pathspec_match_list git_pathspec_match_list;
+
+/**
+ * Options controlling how pathspec match should be executed
+ *
+ * - GIT_PATHSPEC_IGNORE_CASE forces match to ignore case; otherwise
+ *   match will use native case sensitivity of platform
+ * - GIT_PATHSPEC_USE_CASE forces case sensitive match; otherwise
+ *   match will use native case sensitivity of platform
+ * - GIT_PATHSPEC_NO_GLOB disables glob patterns and just uses simple
+ *   string comparison for matching
+ * - GIT_PATHSPEC_NO_MATCH_ERROR means the match function will return
+ *   GIT_ENOTFOUND if no matches are found; otherwise it will return 0
+ *   for success and `git_pathspec_match_list_entrycount` will be 0.
+ * - GIT_PATHSPEC_FIND_FAILURES only applies to a git_pathspec_match_list;
+ *   it means to check file names against all unmatched patterns so that
+ *   at the end of a match we can identify patterns that did not match any
+ *   files.
+ * - GIT_PATHSPEC_FAILURES_ONLY only applies to a git_pathspec_match_list;
+ *   it means to only check for mismatches and not record matched paths.
+ */
+typedef enum {
+	GIT_PATHSPEC_DEFAULT        = 0,
+	GIT_PATHSPEC_IGNORE_CASE    = (1u << 0),
+	GIT_PATHSPEC_USE_CASE       = (1u << 1),
+	GIT_PATHSPEC_NO_GLOB        = (1u << 2),
+	GIT_PATHSPEC_NO_MATCH_ERROR = (1u << 3),
+	GIT_PATHSPEC_FIND_FAILURES  = (1u << 4),
+	GIT_PATHSPEC_FAILURES_ONLY  = (1u << 5),
+} git_pathspec_flag_t;
+
+/**
+ * Compile a pathspec
+ *
+ * @param out Output of the compiled pathspec
+ * @param flags Combination of git_pathspec_flag_t values
+ * @param pathspec A git_strarray of the paths to match
+ * @return 0 on success, <0 on failure
+ */
+GIT_EXTERN(int) git_pathspec_new(
+	git_pathspec **out, const git_strarray *pathspec);
+
+/**
+ * Free a pathspec
+ *
+ * @param ps The compiled pathspec
+ */
+GIT_EXTERN(void) git_pathspec_free(git_pathspec *ps);
+
+/**
+ * Try to match a path against a pathspec
+ *
+ * Unlike most of the other pathspec matching functions, this will not
+ * fall back on the native case-sensitivity for your platform.  You must
+ * explicitly pass flags to control case sensitivity or else this will
+ * fall back on being case sensitive.
+ *
+ * @param ps The compiled pathspec
+ * @param flags Match flags to influence matching behavior
+ * @param path The pathname to attempt to match
+ * @return 1 is path matches spec, 0 if it does not
+ */
+GIT_EXTERN(int) git_pathspec_matches_path(
+	const git_pathspec *ps, uint32_t flags, const char *path);
+
+/**
+ * Match a pathspec against the working directory of a repository.
+ *
+ * This returns a `git_patchspec_match` object that contains the list of
+ * all files matching the given pathspec in the working directory of the
+ * repository.  This handles git ignores (i.e. ignored files will not be
+ * considered to match the `pathspec` unless the file is tracked in the
+ * index).
+ *
+ * @param out Object with list of matching items
+ * @param repo The repository in which to match; bare repo is an error
+ * @param flags Options to control matching behavior
+ * @param ps Pathspec to be matched
+ * @return 0 on success, -1 on error, GIT_ENOTFOUND if no matches and
+ *         the GIT_PATHSPEC_NO_MATCH_ERROR flag is used
+ */
+GIT_EXTERN(int) git_pathspec_match_workdir(
+	git_pathspec_match_list **out,
+	git_repository *repo,
+	uint32_t flags,
+	git_pathspec *ps);
+
+/**
+ * Match a pathspec against entries in an index.
+ *
+ * This returns a `git_patchspec_match` object that contains the list of
+ * all files matching the given pathspec in the index.
+ *
+ * NOTE: At the moment, the case sensitivity of this match is controlled
+ * by the current case-sensitivity of the index object itself and the
+ * USE_CASE and IGNORE_CASE flags will have no effect.  This behavior will
+ * be corrected in a future release.
+ *
+ * @param out Object with list of matching items
+ * @param inex The index in which to match
+ * @param flags Options to control matching behavior
+ * @param ps Pathspec to be matched
+ * @return 0 on success, -1 on error, GIT_ENOTFOUND if no matches and
+ *         the GIT_PATHSPEC_NO_MATCH_ERROR flag is used
+ */
+GIT_EXTERN(int) git_pathspec_match_index(
+	git_pathspec_match_list **out,
+	git_index *index,
+	uint32_t flags,
+	git_pathspec *ps);
+
+/**
+ * Match a pathspec against files in a tree.
+ *
+ * This returns a `git_patchspec_match` object that contains the list of
+ * all files matching the given pathspec in the given tree.
+ *
+ * @param out Object with list of matching items
+ * @param inex The index in which to match
+ * @param flags Options to control matching behavior
+ * @param ps Pathspec to be matched
+ * @return 0 on success, -1 on error, GIT_ENOTFOUND if no matches and
+ *         the GIT_PATHSPEC_NO_MATCH_ERROR flag is used
+ */
+GIT_EXTERN(int) git_pathspec_match_tree(
+	git_pathspec_match_list **out,
+	git_tree *tree,
+	uint32_t flags,
+	git_pathspec *ps);
+
+/**
+ * Free memory associates with a git_pathspec_match_list
+ *
+ * @param m The git_pathspec_match_list to be freed
+ */
+GIT_EXTERN(void) git_pathspec_match_list_free(git_pathspec_match_list *m);
+
+/**
+ * Get the number of items in a match list.
+ *
+ * @param m The git_pathspec_match_list object
+ * @return Number of items in match list
+ */
+GIT_EXTERN(size_t) git_pathspec_match_list_entrycount(
+	const git_pathspec_match_list *m);
+
+/**
+ * Get a matching filename by position.
+ *
+ * @param m The git_pathspec_match_list object
+ * @param pos The index into the list
+ * @return The filename of the match
+ */
+GIT_EXTERN(const char *) git_pathspec_match_list_entry(
+	const git_pathspec_match_list *m, size_t pos);
+
+/**
+ * Get the number of pathspec items that did not match.
+ *
+ * This will be zero unless you passed GIT_PATHSPEC_FIND_FAILURES when
+ * generating the git_pathspec_match_list.
+ *
+ * @param m The git_pathspec_match_list object
+ * @return Number of items in original pathspec that had no matches
+ */
+GIT_EXTERN(size_t) git_pathspec_match_list_failed_entrycount(
+	const git_pathspec_match_list *m);
+
+/**
+ * Get an original pathspec string that had no matches.
+ *
+ * This will be return NULL for positions out of range.
+ *
+ * @param m The git_pathspec_match_list object
+ * @param pos The index into the failed items
+ * @return The pathspec pattern that didn't match anything
+ */
+GIT_EXTERN(const char *) git_pathspec_match_list_failed_entry(
+	const git_pathspec_match_list *m, size_t pos);
+
+#endif

src/checkout.c

@@ -246,10 +246,10 @@ static int checkout_action_wd_only(
 	bool remove = false;
 	git_checkout_notify_t notify = GIT_CHECKOUT_NOTIFY_NONE;
 
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			pathspec, wd->path,
 			(data->strategy & GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH) != 0,
-			git_iterator_ignore_case(workdir), NULL))
+			git_iterator_ignore_case(workdir), NULL, NULL))
 		return 0;
 
 	/* check if item is tracked in the index but not in the checkout diff */
@@ -607,7 +607,7 @@ static int checkout_get_actions(
 	uint32_t *actions = NULL;
 
 	if (data->opts.paths.count > 0 &&
-		git_pathspec_init(&pathspec, &data->opts.paths, &pathpool) < 0)
+		git_pathspec__vinit(&pathspec, &data->opts.paths, &pathpool) < 0)
 		return -1;
 
 	if ((error = git_iterator_current(&wditem, workdir)) < 0 &&
@@ -659,7 +659,7 @@ static int checkout_get_actions(
 		goto fail;
 	}
 
-	git_pathspec_free(&pathspec);
+	git_pathspec__vfree(&pathspec);
 	git_pool_clear(&pathpool);
 
 	return 0;
@@ -670,7 +670,7 @@ fail:
 	*actions_ptr = NULL;
 	git__free(actions);
 
-	git_pathspec_free(&pathspec);
+	git_pathspec__vfree(&pathspec);
 	git_pool_clear(&pathpool);
 
 	return error;

src/diff.c

@@ -81,11 +81,11 @@ static int diff_delta__from_one(
 		DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES))
 		return 0;
 
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			&diff->pathspec, entry->path,
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
-			&matched_pathspec))
+			&matched_pathspec, NULL))
 		return 0;
 
 	delta = diff_delta__alloc(diff, status, entry->path);
@@ -387,7 +387,7 @@ static int diff_list_apply_options(
 		DIFF_FLAG_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE, icase);
 
 		/* initialize pathspec from options */
-		if (git_pathspec_init(&diff->pathspec, &opts->pathspec, pool) < 0)
+		if (git_pathspec__vinit(&diff->pathspec, &opts->pathspec, pool) < 0)
 			return -1;
 	}
 
@@ -473,7 +473,7 @@ static void diff_list_free(git_diff_list *diff)
 	}
 	git_vector_free(&diff->deltas);
 
-	git_pathspec_free(&diff->pathspec);
+	git_pathspec__vfree(&diff->pathspec);
 	git_pool_clear(&diff->pool);
 
 	git__memzero(diff, sizeof(*diff));
@@ -634,11 +634,11 @@ static int maybe_modified(
 	bool new_is_workdir = (info->new_iter->type == GIT_ITERATOR_TYPE_WORKDIR);
 	const char *matched_pathspec;
 
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			&diff->pathspec, oitem->path,
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
-			&matched_pathspec))
+			&matched_pathspec, NULL))
 		return 0;
 
 	memset(&noid, 0, sizeof(noid));

src/index.c

@@ -101,8 +101,6 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 static bool is_index_extended(git_index *index);
 static int write_index(git_index *index, git_filebuf *file);
 
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage);
-
 static void index_entry_free(git_index_entry *entry);
 static void index_entry_reuc_free(git_index_reuc_entry *reuc);
 
@@ -114,7 +112,7 @@ static int index_srch(const void *key, const void *array_member)
 
 	ret = strcmp(srch_key->path, entry->path);
 
-	if (ret == 0)
+	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
 		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
 
 	return ret;
@@ -128,7 +126,7 @@ static int index_isrch(const void *key, const void *array_member)
 
 	ret = strcasecmp(srch_key->path, entry->path);
 
-	if (ret == 0)
+	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
 		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
 
 	return ret;
@@ -562,7 +560,7 @@ const git_index_entry *git_index_get_bypath(
 
 	git_vector_sort(&index->entries);
 
-	if (index_find(&pos, index, path, stage) < 0) {
+	if (git_index__find(&pos, index, path, stage) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
 		return NULL;
 	}
@@ -719,7 +717,8 @@ static int index_insert(git_index *index, git_index_entry *entry, int replace)
 		entry->flags |= GIT_IDXENTRY_NAMEMASK;
 
 	/* look if an entry with this path already exists */
-	if (!index_find(&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
+	if (!git_index__find(
+			&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
 		existing = (git_index_entry **)&index->entries.contents[position];
 
 		/* update filemode to existing values if stat is not trusted */
@@ -831,7 +830,7 @@ int git_index_remove(git_index *index, const char *path, int stage)
 
 	git_vector_sort(&index->entries);
 
-	if (index_find(&position, index, path, stage) < 0) {
+	if (git_index__find(&position, index, path, stage) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d",
 			path, stage);
 		return GIT_ENOTFOUND;
@@ -887,7 +886,8 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 	return error;
 }
 
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage)
+int git_index__find(
+	size_t *at_pos, git_index *index, const char *path, int stage)
 {
 	struct entry_srch_key srch_key;
 
@@ -896,7 +896,8 @@ static int index_find(size_t *at_pos, git_index *index, const char *path, int st
 	srch_key.path = path;
 	srch_key.stage = stage;
 
-	return git_vector_bsearch2(at_pos, &index->entries, index->entries_search, &srch_key);
+	return git_vector_bsearch2(
+		at_pos, &index->entries, index->entries_search, &srch_key);
 }
 
 int git_index_find(size_t *at_pos, git_index *index, const char *path)
@@ -2053,7 +2054,7 @@ int git_index_add_all(
 	git_iterator *wditer = NULL;
 	const git_index_entry *wd = NULL;
 	git_index_entry *entry;
-	git_pathspec_context ps;
+	git_pathspec ps;
 	const char *match;
 	size_t existing;
 	bool no_fnmatch = (flags & GIT_INDEX_ADD_DISABLE_PATHSPEC_MATCH) != 0;
@@ -2074,7 +2075,7 @@ int git_index_add_all(
 	if (git_repository__cvar(&ignorecase, repo, GIT_CVAR_IGNORECASE) < 0)
 		return -1;
 
-	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
 		return error;
 
 	/* optionally check that pathspec doesn't mention any ignored files */
@@ -2091,14 +2092,14 @@ int git_index_add_all(
 	while (!(error = git_iterator_advance(&wd, wditer))) {
 
 		/* check if path actually matches */
-		if (!git_pathspec_match_path(
-				&ps.pathspec, wd->path, no_fnmatch, ignorecase, &match))
+		if (!git_pathspec__match(
+				&ps.pathspec, wd->path, no_fnmatch, ignorecase, &match, NULL))
 			continue;
 
 		/* skip ignored items that are not already in the index */
 		if ((flags & GIT_INDEX_ADD_FORCE) == 0 &&
 			git_iterator_current_is_ignored(wditer) &&
-			index_find(&existing, index, wd->path, 0) < 0)
+			git_index__find(&existing, index, wd->path, 0) < 0)
 			continue;
 
 		/* issue notification callback if requested */
@@ -2148,7 +2149,7 @@ int git_index_add_all(
 
 cleanup:
 	git_iterator_free(wditer);
-	git_pathspec_context_free(&ps);
+	git_pathspec__clear(&ps);
 
 	return error;
 }
@@ -2168,13 +2169,13 @@ static int index_apply_to_all(
 {
 	int error = 0;
 	size_t i;
-	git_pathspec_context ps;
+	git_pathspec ps;
 	const char *match;
 	git_buf path = GIT_BUF_INIT;
 
 	assert(index);
 
-	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
 		return error;
 
 	git_vector_sort(&index->entries);
@@ -2183,8 +2184,9 @@ static int index_apply_to_all(
 		git_index_entry *entry = git_vector_get(&index->entries, i);
 
 		/* check if path actually matches */
-		if (!git_pathspec_match_path(
-				&ps.pathspec, entry->path, false, index->ignore_case, &match))
+		if (!git_pathspec__match(
+				&ps.pathspec, entry->path, false, index->ignore_case,
+				&match, NULL))
 			continue;
 
 		/* issue notification callback if requested */
@@ -2231,7 +2233,7 @@ static int index_apply_to_all(
 	}
 
 	git_buf_free(&path);
-	git_pathspec_context_free(&ps);
+	git_pathspec__clear(&ps);
 
 	return error;
 }

src/index.h

@@ -47,13 +47,17 @@ struct git_index_conflict_iterator {
 	size_t cur;
 };
 
-extern void git_index_entry__init_from_stat(git_index_entry *entry, struct stat *st);
+extern void git_index_entry__init_from_stat(
+	git_index_entry *entry, struct stat *st);
 
 extern size_t git_index__prefix_position(git_index *index, const char *path);
 
 extern int git_index_entry__cmp(const void *a, const void *b);
 extern int git_index_entry__cmp_icase(const void *a, const void *b);
 
+extern int git_index__find(
+	size_t *at_pos, git_index *index, const char *path, int stage);
+
 extern void git_index__set_ignore_case(git_index *index, bool ignore_case);
 
 #endif

src/pathspec.h

@@ -8,9 +8,27 @@
 #define INCLUDE_pathspec_h__
 
 #include "common.h"
+#include <git2/pathspec.h>
 #include "buffer.h"
 #include "vector.h"
 #include "pool.h"
+#include "array.h"
+
+/* public compiled pathspec */
+struct git_pathspec {
+	git_refcount rc;
+	char *prefix;
+	git_vector pathspec;
+	git_pool pool;
+};
+
+/* public interface to pathspec matching */
+struct git_pathspec_match_list {
+	git_pathspec *pathspec;
+	git_array_t(char *) matches;
+	git_array_t(char *) failures;
+	git_pool pool;
+};
 
 /* what is the common non-wildcard prefix for all items in the pathspec */
 extern char *git_pathspec_prefix(const git_strarray *pathspec);
@@ -19,36 +37,31 @@ extern char *git_pathspec_prefix(const git_strarray *pathspec);
 extern bool git_pathspec_is_empty(const git_strarray *pathspec);
 
 /* build a vector of fnmatch patterns to evaluate efficiently */
-extern int git_pathspec_init(
+extern int git_pathspec__vinit(
 	git_vector *vspec, const git_strarray *strspec, git_pool *strpool);
 
 /* free data from the pathspec vector */
-extern void git_pathspec_free(git_vector *vspec);
+extern void git_pathspec__vfree(git_vector *vspec);
+
+#define GIT_PATHSPEC_NOMATCH ((size_t)-1)
 
 /*
  * Match a path against the vectorized pathspec.
  * The matched pathspec is passed back into the `matched_pathspec` parameter,
  * unless it is passed as NULL by the caller.
  */
-extern bool git_pathspec_match_path(
-	git_vector *vspec,
+extern bool git_pathspec__match(
+	const git_vector *vspec,
 	const char *path,
 	bool disable_fnmatch,
 	bool casefold,
-	const char **matched_pathspec);
+	const char **matched_pathspec,
+	size_t *matched_at);
 
 /* easy pathspec setup */
 
-typedef struct {
-	char *prefix;
-	git_vector pathspec;
-	git_pool pool;
-} git_pathspec_context;
-
-extern int git_pathspec_context_init(
-	git_pathspec_context *ctxt, const git_strarray *paths);
+extern int git_pathspec__init(git_pathspec *ps, const git_strarray *paths);
 
-extern void git_pathspec_context_free(
-	git_pathspec_context *ctxt);
+extern void git_pathspec__clear(git_pathspec *ps);
 
 #endif