odb: add git_odb_foreach()

Go through each backend and list every objects that exists in
them. This allows fsck-like uses.

include/git2/odb.h

@@ -173,6 +173,20 @@ GIT_EXTERN(int) git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *d
 GIT_EXTERN(int) git_odb_exists(git_odb *db, const git_oid *id);
 
 /**
+ * List all objects available in the database
+ *
+ * The callback will be called for each object available in the
+ * database. Note that the objects are likely to be returned in the
+ * index order, which would make accessing the objects in that order
+ * inefficient.
+ *
+ * @param db database to use
+ * @param cb the callback to call for each object
+ * @param data data to pass to the callback
+ */
+GIT_EXTERN(int) git_odb_foreach(git_odb *db, int (*cb)(git_oid *oid, void *data), void *data);
+
+/**
  * Write an object directly into the ODB
  *
  * This method writes a full object straight into the ODB.

include/git2/odb_backend.h

@@ -71,6 +71,12 @@ struct git_odb_backend {
 			struct git_odb_backend *,
 			const git_oid *);
 
+	int (*foreach)(
+		       struct git_odb_backend *,
+		       int (*cb)(git_oid *oid, void *data),
+		       void *data
+		       );
+
 	void (* free)(struct git_odb_backend *);
 };
 

src/odb.c

@@ -605,6 +605,18 @@ int git_odb_read_prefix(
 	return 0;
 }
 
+int git_odb_foreach(git_odb *db, int (*cb)(git_oid *oid, void *data), void *data)
+{
+	unsigned int i;
+	backend_internal *internal;
+	git_vector_foreach(&db->backends, i, internal) {
+		git_odb_backend *b = internal->backend;
+		b->foreach(b, cb, data);
+	}
+
+	return 0;
+}
+
 int git_odb_write(
 	git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
 {

src/odb_loose.c

@@ -676,6 +676,89 @@ static int loose_backend__exists(git_odb_backend *backend, const git_oid *oid)
 	return !error;
 }
 
+struct foreach_state {
+	size_t dir_len;
+	int (*cb)(git_oid *oid, void *data);
+	void *data;
+};
+
+static inline int filename_to_oid(git_oid *oid, const char *ptr)
+{
+	int v, i = 0;
+	if (strlen(ptr) != 41)
+		return -1;
+
+	if (ptr[2] != '/') {
+		return -1;
+	}
+
+	v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i+1]);
+	if (v < 0)
+		return -1;
+
+	oid->id[0] = (unsigned char) v;
+
+	ptr += 3;
+	for (i = 0; i < 38; i += 2) {
+		v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i + 1]);
+		if (v < 0)
+			return -1;
+
+		oid->id[1 + i/2] = (unsigned char) v;
+	}
+
+	return 0;
+}
+
+static int foreach_object_dir_cb(void *_state, git_buf *path)
+{
+	git_oid oid;
+	struct foreach_state *state = (struct foreach_state *) _state;
+
+	if (filename_to_oid(&oid, path->ptr + state->dir_len) < 0)
+		return 0;
+
+	if (state->cb(&oid, state->data) < 0)
+		return -1;
+
+	return 0;
+}
+
+static int foreach_cb(void *_state, git_buf *path)
+{
+	struct foreach_state *state = (struct foreach_state *) _state;
+
+	if (git_path_direach(path, foreach_object_dir_cb, state) < 0)
+		return -1;
+
+	return 0;
+}
+
+static int loose_backend__foreach(git_odb_backend *_backend, int (*cb)(git_oid *oid, void *data), void *data)
+{
+	char *objects_dir;
+	int error;
+	git_buf buf = GIT_BUF_INIT;
+	struct foreach_state state;
+	loose_backend *backend = (loose_backend *) _backend;
+
+	assert(backend && cb);
+
+	objects_dir = backend->objects_dir;
+
+	git_buf_sets(&buf, objects_dir);
+	git_path_to_dir(&buf);
+
+	state.cb = cb;
+	state.data = data;
+	state.dir_len = git_buf_len(&buf);
+
+	error = git_path_direach(&buf, foreach_cb, &state);
+	git_buf_free(&buf);
+
+	return error;
+}
+
 static int loose_backend__stream_fwrite(git_oid *oid, git_odb_stream *_stream)
 {
 	loose_writestream *stream = (loose_writestream *)_stream;
@@ -845,6 +928,7 @@ int git_odb_backend_loose(
 	backend->parent.read_header = &loose_backend__read_header;
 	backend->parent.writestream = &loose_backend__stream;
 	backend->parent.exists = &loose_backend__exists;
+	backend->parent.foreach = &loose_backend__foreach;
 	backend->parent.free = &loose_backend__free;
 
 	*backend_out = (git_odb_backend *)backend;

src/odb_pack.c

@@ -420,6 +420,25 @@ static int pack_backend__exists(git_odb_backend *backend, const git_oid *oid)
 	return pack_entry_find(&e, (struct pack_backend *)backend, oid) == 0;
 }
 
+static int pack_backend__foreach(git_odb_backend *_backend, int (*cb)(git_oid *oid, void *data), void *data)
+{
+	struct git_pack_file *p;
+	struct pack_backend *backend;
+	unsigned int i;
+
+	assert(_backend && cb);
+	backend = (struct pack_backend *)_backend;
+
+	/* Make sure we know about the packfiles */
+	if (packfile_refresh_all(backend) < 0)
+		return -1;
+
+	git_vector_foreach(&backend->packs, i, p) {
+		git_pack_foreach_entry(p, cb, &data);
+	}
+	return 0;
+}
+
 static void pack_backend__free(git_odb_backend *_backend)
 {
 	struct pack_backend *backend;
@@ -463,6 +482,7 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
 	backend->parent.read_prefix = &pack_backend__read_prefix;
 	backend->parent.read_header = NULL;
 	backend->parent.exists = &pack_backend__exists;
+	backend->parent.foreach = &pack_backend__foreach;
 	backend->parent.free = &pack_backend__free;
 
 	*backend_out = (git_odb_backend *)backend;

src/pack.c

@@ -686,6 +686,49 @@ static git_off_t nth_packed_object_offset(const struct git_pack_file *p, uint32_
 	}
 }
 
+int git_pack_foreach_entry(
+		struct git_pack_file *p,
+		int (*cb)(git_oid *oid, void *data),
+		void *data)
+
+{
+	const unsigned char *index = p->index_map.data, *current;
+	unsigned stride;
+	uint32_t i;
+
+	if (index == NULL) {
+		int error;
+
+		if ((error = pack_index_open(p)) < 0)
+			return error;
+
+		assert(p->index_map.data);
+
+		index = p->index_map.data;
+	}
+
+	if (p->index_version > 1) {
+		index += 8;
+	}
+
+	index += 4 * 256;
+
+	if (p->index_version > 1) {
+		stride = 20;
+	} else {
+		stride = 24;
+		index += 4;
+	}
+
+	current = index;
+	for (i = 0; i < p->num_objects; i++) {
+		cb((git_oid *)current, data);
+		current += stride;
+	}
+
+	return 0;
+}
+
 static int pack_entry_find_offset(
 	git_off_t *offset_out,
 	git_oid *found_oid,

src/pack.h

@@ -102,5 +102,9 @@ int git_pack_entry_find(
 		struct git_pack_file *p,
 		const git_oid *short_oid,
 		unsigned int len);
+int git_pack_foreach_entry(
+		struct git_pack_file *p,
+		int (*cb)(git_oid *oid, void *data),
+		void *data);
 
 #endif

tests-clar/odb/foreach.c

+#include "clar_libgit2.h"
+#include "odb.h"
+#include "git2/odb_backend.h"
+#include "pack.h"
+
+static git_odb *_odb;
+static git_repository *_repo;
+static int nobj;
+
+void test_odb_foreach__initialize(void)
+{
+	cl_git_pass(git_repository_open(&_repo, cl_fixture("testrepo.git")));
+	git_repository_odb(&_odb, _repo);
+}
+
+void test_odb_foreach__cleanup(void)
+{
+	git_odb_free(_odb);
+	git_repository_free(_repo);
+}
+
+static int foreach_cb(git_oid *oid, void *data)
+{
+	GIT_UNUSED(data);
+	GIT_UNUSED(oid);
+
+	nobj++;
+
+	return 0;
+}
+
+void test_odb_foreach__foreach(void)
+{
+	cl_git_pass(git_odb_foreach(_odb, foreach_cb, NULL));
+	cl_assert(nobj == 1681);
+}