Skip to content

G
git2
Commit fe372740 by Russell Belfer
Options

Rewrite refdb_fs using git_sortedcache object 

This adds thread safety to the refdb_fs by using the new
git_sortedcache object and also by relaxing the handling of some
filesystem errors where the fs may be changed out from under us.

This also adds some new threading tests that hammer on the refdb.
parent 24c71f14
Showing with 514 additions and 496 deletions
src/refdb_fs.c
......@@ -15,6 +15,7 @@
#include "refdb.h"
#include "refdb_fs.h"
#include "iterator.h"
#include "sortedcache.h"
#include <git2/tag.h>
#include <git2/object.h>
......@@ -53,243 +54,149 @@ typedef struct refdb_fs_backend {
git_repository *repo;
char *path;
git_refcache refcache;
git_sortedcache *refcache;
int peeling_mode;
} refdb_fs_backend;
static int reference_read(
git_buf *file_content,
time_t *mtime,
const char *repo_path,
const char *ref_name,
int *updated)
static int packref_cmp(const void *a_, const void *b_)
{
git_buf path = GIT_BUF_INIT;
int result;
assert(file_content && repo_path && ref_name);
/* Determine the full path of the file */
if (git_buf_joinpath(&path, repo_path, ref_name) < 0)
return -1;
result = git_futils_readbuffer_updated(file_content, path.ptr, mtime, NULL, updated);
git_buf_free(&path);
return result;
const struct packref *a = a_, *b = b_;
return strcmp(a->name, b->name);
}
static int packed_parse_oid(
struct packref **ref_out,
const char **buffer_out,
const char *buffer_end)
static int packed_reload(refdb_fs_backend *backend)
{
struct packref *ref = NULL;
const char *buffer = *buffer_out;
const char *refname_begin, *refname_end;
size_t refname_len;
git_oid id;
refname_begin = (buffer + GIT_OID_HEXSZ + 1);
if (refname_begin >= buffer_end || refname_begin[-1] != ' ')
goto corrupt;
/* Is this a valid object id? */
if (git_oid_fromstr(&id, buffer) < 0)
goto corrupt;
refname_end = memchr(refname_begin, '\n', buffer_end - refname_begin);
if (refname_end == NULL)
refname_end = buffer_end;
if (refname_end[-1] == '\r')
refname_end--;
refname_len = refname_end - refname_begin;
ref = git__calloc(1, sizeof(struct packref) + refname_len + 1);
GITERR_CHECK_ALLOC(ref);
memcpy(ref->name, refname_begin, refname_len);
ref->name[refname_len] = 0;
git_oid_cpy(&ref->oid, &id);
*ref_out = ref;
*buffer_out = refname_end + 1;
return 0;
corrupt:
git__free(ref);
giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
return -1;
}
static int packed_parse_peel(
struct packref *tag_ref,
const char **buffer_out,
const char *buffer_end)
{
const char *buffer = *buffer_out + 1;
assert(buffer[-1] == '^');
/* Ensure it's not the first entry of the file */
if (tag_ref == NULL)
goto corrupt;
if (buffer + GIT_OID_HEXSZ > buffer_end)
goto corrupt;
/* Is this a valid object id? */
if (git_oid_fromstr(&tag_ref->peel, buffer) < 0)
goto corrupt;
buffer = buffer + GIT_OID_HEXSZ;
if (*buffer == '\r')
buffer++;
if (buffer != buffer_end) {
if (*buffer == '\n')
buffer++;
else
goto corrupt;
}
tag_ref->flags |= PACKREF_HAS_PEEL;
*buffer_out = buffer;
return 0;
corrupt:
giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
return -1;
}
static int packed_load(refdb_fs_backend *backend)
{
int result, updated;
git_buf packfile = GIT_BUF_INIT;
const char *buffer_start, *buffer_end;
git_refcache *ref_cache = &backend->refcache;
/* First we make sure we have allocated the hash table */
if (ref_cache->packfile == NULL) {
ref_cache->packfile = git_strmap_alloc();
GITERR_CHECK_ALLOC(ref_cache->packfile);
}
int error;
git_buf packedrefs = GIT_BUF_INIT;
char *scan, *eof, *eol;
if (backend->path == NULL)
if (!backend->path)
return 0;
result = reference_read(&packfile, &ref_cache->packfile_time,
backend->path, GIT_PACKEDREFS_FILE, &updated);
error = git_sortedcache_lockandload(backend->refcache, &packedrefs);
/*
* If we couldn't find the file, we need to clear the table and
* return. On any other error, we return that error. If everything
* went fine and the file wasn't updated, then there's nothing new
* for us here, so just return. Anything else means we need to
* refresh the packed refs.
* If we can't find the packed-refs, clear table and return.
* Any other error just gets passed through.
* If no error, and file wasn't changed, just return.
* Anything else means we need to refresh the packed refs.
*/
if (result == GIT_ENOTFOUND) {
git_strmap_clear(ref_cache->packfile);
return 0;
if (error <= 0) {
if (error == GIT_ENOTFOUND) {
git_sortedcache_clear(backend->refcache, true);
giterr_clear();
error = 0;
}
return error;
}
if (result < 0)
return -1;
if (!updated)
return 0;
/* At this point, refresh the packed refs from the loaded buffer. */
/*
* At this point, we want to refresh the packed refs. We already
* have the contents in our buffer.
*/
git_strmap_clear(ref_cache->packfile);
git_sortedcache_clear(backend->refcache, false);
buffer_start = (const char *)packfile.ptr;
buffer_end = (const char *)(buffer_start) + packfile.size;
scan = (char *)packedrefs.ptr;
eof = scan + packedrefs.size;
backend->peeling_mode = PEELING_NONE;
if (buffer_start[0] == '#') {
if (*scan == '#') {
static const char *traits_header = "# pack-refs with: ";
if (git__prefixcmp(buffer_start, traits_header) == 0) {
char *traits = (char *)buffer_start + strlen(traits_header);
char *traits_end = strchr(traits, '\n');
if (git__prefixcmp(scan, traits_header) == 0) {
scan += strlen(traits_header);
eol = strchr(scan, '\n');
if (traits_end == NULL)
if (!eol)
goto parse_failed;
*eol = '\0';
*traits_end = '\0';
if (strstr(traits, " fully-peeled ") != NULL) {
if (strstr(scan, " fully-peeled ") != NULL) {
backend->peeling_mode = PEELING_FULL;
} else if (strstr(traits, " peeled ") != NULL) {
} else if (strstr(scan, " peeled ") != NULL) {
backend->peeling_mode = PEELING_STANDARD;
}
buffer_start = traits_end + 1;
scan = eol + 1;
}
}
while (buffer_start < buffer_end && buffer_start[0] == '#') {
buffer_start = strchr(buffer_start, '\n');
if (buffer_start == NULL)
while (scan < eof && *scan == '#') {
if (!(eol = strchr(scan, '\n')))
goto parse_failed;
buffer_start++;
scan = eol + 1;
}
while (buffer_start < buffer_end) {
int err;
struct packref *ref = NULL;
while (scan < eof) {
struct packref *ref;
git_oid oid;
/* parse "<OID> <refname>\n" */
if (packed_parse_oid(&ref, &buffer_start, buffer_end) < 0)
if (git_oid_fromstr(&oid, scan) < 0)
goto parse_failed;
scan += GIT_OID_HEXSZ;
if (buffer_start[0] == '^') {
if (packed_parse_peel(ref, &buffer_start, buffer_end) < 0)
goto parse_failed;
} else if (backend->peeling_mode == PEELING_FULL ||
(backend->peeling_mode == PEELING_STANDARD &&
git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0)) {
ref->flags |= PACKREF_CANNOT_PEEL;
}
if (*scan++ != ' ')
goto parse_failed;
if (!(eol = strchr(scan, '\n')))
goto parse_failed;
*eol = '\0';
if (eol[-1] == '\r')
eol[-1] = '\0';
git_strmap_insert(ref_cache->packfile, ref->name, ref, err);
if (err < 0)
if (git_sortedcache_upsert((void **)&ref, backend->refcache, scan) < 0)
goto parse_failed;
scan = eol + 1;
git_oid_cpy(&ref->oid, &oid);
/* look for optional "^<OID>\n" */
if (*scan == '^') {
if (git_oid_fromstr(&oid, scan + 1) < 0)
goto parse_failed;
scan += GIT_OID_HEXSZ + 1;
if (scan < eof) {
if (!(eol = strchr(scan, '\n')))
goto parse_failed;
scan = eol + 1;
}
git_oid_cpy(&ref->peel, &oid);
ref->flags |= PACKREF_HAS_PEEL;
}
else if (backend->peeling_mode == PEELING_FULL ||
(backend->peeling_mode == PEELING_STANDARD &&
git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0))
ref->flags |= PACKREF_CANNOT_PEEL;
}
git_buf_free(&packfile);
git_sortedcache_unlock(backend->refcache);
git_buf_free(&packedrefs);
return 0;
parse_failed:
git_strmap_free(ref_cache->packfile);
ref_cache->packfile = NULL;
git_buf_free(&packfile);
giterr_set(GITERR_REFERENCE, "Corrupted packed references file");
git_sortedcache_clear(backend->refcache, false);
git_sortedcache_unlock(backend->refcache);
git_buf_free(&packedrefs);
return -1;
}
static int loose_parse_oid(git_oid *oid, const char *filename, git_buf *file_content)
static int loose_parse_oid(
git_oid *oid, const char *filename, git_buf *file_content)
{
size_t len;
const char *str;
const char *str = git_buf_cstr(file_content);
len = git_buf_len(file_content);
if (len < GIT_OID_HEXSZ)
if (git_buf_len(file_content) < GIT_OID_HEXSZ)
goto corrupted;
/* str is guranteed to be zero-terminated */
str = git_buf_cstr(file_content);
/* we need to get 40 OID characters from the file */
if (git_oid_fromstr(oid, git_buf_cstr(file_content)) < 0)
if (git_oid_fromstr(oid, str) < 0)
goto corrupted;
/* If the file is longer than 40 chars, the 41st must be a space */
......@@ -302,77 +209,71 @@ corrupted:
return -1;
}
static int loose_lookup_to_packfile(
struct packref **ref_out,
refdb_fs_backend *backend,
const char *name)
static int loose_readbuffer(git_buf *buf, const char *base, const char *path)
{
int error;
/* build full path to file */
if ((error = git_buf_joinpath(buf, base, path)) < 0 ||
(error = git_futils_readbuffer(buf, buf->ptr)) < 0)
git_buf_free(buf);
return error;
}
static int loose_lookup_to_packfile(refdb_fs_backend *backend, const char *name)
{
int error = 0;
git_buf ref_file = GIT_BUF_INIT;
struct packref *ref = NULL;
size_t name_len;
*ref_out = NULL;
git_oid oid;
if (reference_read(&ref_file, NULL, backend->path, name, NULL) < 0)
return -1;
/* if we fail to load the loose reference, assume someone changed
* the filesystem under us and skip it...
*/
if (loose_readbuffer(&ref_file, backend->path, name) < 0) {
giterr_clear();
goto done;
}
/* skip symbolic refs */
if (!git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF)) {
git_buf_free(&ref_file);
return 0;
}
if (!git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF))
goto done;
git_buf_rtrim(&ref_file);
/* parse OID from file */
if ((error = loose_parse_oid(&oid, name, &ref_file)) < 0)
goto done;
name_len = strlen(name);
ref = git__calloc(1, sizeof(struct packref) + name_len + 1);
GITERR_CHECK_ALLOC(ref);
git_sortedcache_lock(backend->refcache);
memcpy(ref->name, name, name_len);
ref->name[name_len] = 0;
if (!(error = git_sortedcache_upsert(
(void **)&ref, backend->refcache, name))) {
if (loose_parse_oid(&ref->oid, name, &ref_file) < 0) {
git_buf_free(&ref_file);
git__free(ref);
return -1;
git_oid_cpy(&ref->oid, &oid);
ref->flags = PACKREF_WAS_LOOSE;
}
ref->flags = PACKREF_WAS_LOOSE;
git_sortedcache_unlock(backend->refcache);
*ref_out = ref;
done:
git_buf_free(&ref_file);
return 0;
return error;
}
static int _dirent_loose_load(void *data, git_buf *full_path)
{
refdb_fs_backend *backend = (refdb_fs_backend *)data;
void *old_ref = NULL;
struct packref *ref;
const char *file_path;
int err;
if (git_path_isdir(full_path->ptr) == true)
if (git__suffixcmp(full_path->ptr, ".lock") == 0)
return 0;
if (git_path_isdir(full_path->ptr))
return git_path_direach(full_path, _dirent_loose_load, backend);
file_path = full_path->ptr + strlen(backend->path);
if (loose_lookup_to_packfile(&ref, backend, file_path) < 0)
return -1;
if (!ref)
return 0;
git_strmap_insert2(
backend->refcache.packfile, ref->name, ref, old_ref, err);
if (err < 0) {
git__free(ref);
return -1;
}
git__free(old_ref);
return 0;
return loose_lookup_to_packfile(backend, file_path);
}
/*
......@@ -383,11 +284,8 @@ static int _dirent_loose_load(void *data, git_buf *full_path)
*/
static int packed_loadloose(refdb_fs_backend *backend)
{
int error;
git_buf refs_path = GIT_BUF_INIT;
int result;
/* the packfile must have been previously loaded! */
assert(backend->refcache.packfile);
if (git_buf_joinpath(&refs_path, backend->path, GIT_REFS_DIR) < 0)
return -1;
......@@ -397,10 +295,11 @@ static int packed_loadloose(refdb_fs_backend *backend)
* This will overwrite any old packed entries with their
* updated loose versions
*/
result = git_path_direach(&refs_path, _dirent_loose_load, backend);
error = git_path_direach(&refs_path, _dirent_loose_load, backend);
git_buf_free(&refs_path);
return result;
return error;
}
static int refdb_fs_backend__exists(
......@@ -408,23 +307,17 @@ static int refdb_fs_backend__exists(
git_refdb_backend *_backend,
const char *ref_name)
{
refdb_fs_backend *backend;
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
git_buf ref_path = GIT_BUF_INIT;
assert(_backend);
backend = (refdb_fs_backend *)_backend;
assert(backend);
if (packed_load(backend) < 0)
if (packed_reload(backend) < 0 ||
git_buf_joinpath(&ref_path, backend->path, ref_name) < 0)
return -1;
if (git_buf_joinpath(&ref_path, backend->path, ref_name) < 0)
return -1;
if (git_path_isfile(ref_path.ptr) == true ||
git_strmap_exists(backend->refcache.packfile, ref_path.ptr))
*exists = 1;
else
*exists = 0;
*exists = git_path_isfile(ref_path.ptr) ||
git_sortedcache_lookup(backend->refcache, ref_name);
git_buf_free(&ref_path);
return 0;
......@@ -456,69 +349,39 @@ static int loose_lookup(
refdb_fs_backend *backend,
const char *ref_name)
{
const char *target;
git_oid oid;
git_buf ref_file = GIT_BUF_INIT;
int error = 0;
if (out)
*out = NULL;
error = reference_read(&ref_file, NULL, backend->path, ref_name, NULL);
if ((error = loose_readbuffer(&ref_file, backend->path, ref_name)) < 0)
/* cannot read loose ref file - gah */;
else if (git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF) == 0) {
const char *target;
if (error < 0)
goto done;
if (git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF) == 0) {
git_buf_rtrim(&ref_file);
if ((target = loose_parse_symbolic(&ref_file)) == NULL) {
if (!(target = loose_parse_symbolic(&ref_file)))
error = -1;
goto done;
}
if (out)
else if (out != NULL)
*out = git_reference__alloc_symbolic(ref_name, target);
} else {
if ((error = loose_parse_oid(&oid, ref_name, &ref_file)) < 0)
goto done;
git_oid oid;
if (out)
if (!(error = loose_parse_oid(&oid, ref_name, &ref_file)) &&
out != NULL)
*out = git_reference__alloc(ref_name, &oid, NULL);
}
if (out && *out == NULL)
error = -1;
done:
git_buf_free(&ref_file);
return error;
}
static int packed_map_entry(
struct packref **entry,
khiter_t *pos,
refdb_fs_backend *backend,
const char *ref_name)
static int ref_error_notfound(const char *name)
{
git_strmap *packfile_refs;
if (packed_load(backend) < 0)
return -1;
/* Look up on the packfile */
packfile_refs = backend->refcache.packfile;
*pos = git_strmap_lookup_index(packfile_refs, ref_name);
if (!git_strmap_valid_index(packfile_refs, *pos)) {
giterr_set(GITERR_REFERENCE, "Reference '%s' not found", ref_name);
return GIT_ENOTFOUND;
}
*entry = git_strmap_value_at(packfile_refs, *pos);
return 0;
giterr_set(GITERR_REFERENCE, "Reference '%s' not found", name);
return GIT_ENOTFOUND;
}
static int packed_lookup(
......@@ -526,18 +389,25 @@ static int packed_lookup(
refdb_fs_backend *backend,
const char *ref_name)
{
struct packref *entry;
khiter_t pos;
int error = 0;
struct packref *entry;
if ((error = packed_map_entry(&entry, &pos, backend, ref_name)) < 0)
return error;
if ((*out = git_reference__alloc(ref_name,
&entry->oid, &entry->peel)) == NULL)
if (packed_reload(backend) < 0)
return -1;
return 0;
git_sortedcache_lock(backend->refcache);
entry = git_sortedcache_lookup(backend->refcache, ref_name);
if (!entry) {
error = ref_error_notfound(ref_name);
} else {
*out = git_reference__alloc(ref_name, &entry->oid, &entry->peel);
if (!*out)
error = -1;
}
git_sortedcache_unlock(backend->refcache);
return error;
}
static int refdb_fs_backend__lookup(
......@@ -545,24 +415,22 @@ static int refdb_fs_backend__lookup(
git_refdb_backend *_backend,
const char *ref_name)
{
refdb_fs_backend *backend;
int result;
assert(_backend);
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
int error;
backend = (refdb_fs_backend *)_backend;
assert(backend);
if ((result = loose_lookup(out, backend, ref_name)) == 0)
if (!(error = loose_lookup(out, backend, ref_name)))
return 0;
/* only try to lookup this reference on the packfile if it
* wasn't found on the loose refs; not if there was a critical error */
if (result == GIT_ENOTFOUND) {
if (error == GIT_ENOTFOUND) {
giterr_clear();
result = packed_lookup(out, backend, ref_name);
error = packed_lookup(out, backend, ref_name);
}
return result;
return error;
}
typedef struct {
......@@ -573,8 +441,8 @@ typedef struct {
git_pool pool;
git_vector loose;
unsigned int loose_pos;
khiter_t packed_pos;
size_t loose_pos;
size_t packed_pos;
} refdb_fs_iter;
static void refdb_fs_backend__iterator_free(git_reference_iterator *_iter)
......@@ -589,7 +457,6 @@ static void refdb_fs_backend__iterator_free(git_reference_iterator *_iter)
static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter)
{
int error = 0;
git_strmap *packfile = backend->refcache.packfile;
git_buf path = GIT_BUF_INIT;
git_iterator *fsit = NULL;
const git_index_entry *entry = NULL;
......@@ -597,18 +464,18 @@ static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter)
if (!backend->path) /* do nothing if no path for loose refs */
return 0;
if (git_buf_printf(&path, "%s/refs", backend->path) < 0)
return -1;
if ((error = git_buf_printf(&path, "%s/refs", backend->path)) < 0 ||
(error = git_iterator_for_filesystem(
&fsit, git_buf_cstr(&path), 0, NULL, NULL)) < 0) {
git_buf_free(&path);
return error;
}
if ((error = git_iterator_for_filesystem(
&fsit, git_buf_cstr(&path), 0, NULL, NULL)) < 0 ||
(error = git_vector_init(&iter->loose, 8, NULL)) < 0 ||
(error = git_buf_sets(&path, GIT_REFS_DIR)) < 0)
goto cleanup;
error = git_buf_sets(&path, GIT_REFS_DIR);
while (!git_iterator_advance(&entry, fsit)) {
while (!error && !git_iterator_advance(&entry, fsit)) {
const char *ref_name;
khiter_t pos;
struct packref *ref;
char *ref_dup;
git_buf_truncate(&path, strlen(GIT_REFS_DIR));
......@@ -619,26 +486,23 @@ static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter)
(iter->glob && p_fnmatch(iter->glob, ref_name, 0) != 0))
continue;
pos = git_strmap_lookup_index(packfile, ref_name);
if (git_strmap_valid_index(packfile, pos)) {
struct packref *ref = git_strmap_value_at(packfile, pos);
git_sortedcache_lock(backend->refcache);
ref = git_sortedcache_lookup(backend->refcache, ref_name);
if (ref)
ref->flags |= PACKREF_SHADOWED;
}
git_sortedcache_unlock(backend->refcache);
if (!(ref_dup = git_pool_strdup(&iter->pool, ref_name))) {
ref_dup = git_pool_strdup(&iter->pool, ref_name);
if (!ref_dup)
error = -1;
goto cleanup;
}
if ((error = git_vector_insert(&iter->loose, ref_dup)) < 0)
goto cleanup;
else
error = git_vector_insert(&iter->loose, ref_dup);
}
cleanup:
git_iterator_free(fsit);
git_buf_free(&path);
return 0;
return error;
}
static int refdb_fs_backend__iterator_next(
......@@ -646,7 +510,7 @@ static int refdb_fs_backend__iterator_next(
{
refdb_fs_iter *iter = (refdb_fs_iter *)_iter;
refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend;
git_strmap *packfile = backend->refcache.packfile;
struct packref *ref;
while (iter->loose_pos < iter->loose.length) {
const char *path = git_vector_get(&iter->loose, iter->loose_pos++);
......@@ -657,17 +521,12 @@ static int refdb_fs_backend__iterator_next(
giterr_clear();
}
while (iter->packed_pos < kh_end(packfile)) {
struct packref *ref = NULL;
git_sortedcache_lock(backend->refcache);
while (!kh_exist(packfile, iter->packed_pos)) {
iter->packed_pos++;
if (iter->packed_pos == kh_end(packfile))
return GIT_ITEROVER;
}
ref = kh_val(packfile, iter->packed_pos);
iter->packed_pos++;
while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++);
if (!ref) /* stop now if another thread deleted refs and we past end */
break;
if (ref->flags & PACKREF_SHADOWED)
continue;
......@@ -676,12 +535,11 @@ static int refdb_fs_backend__iterator_next(
continue;
*out = git_reference__alloc(ref->name, &ref->oid, &ref->peel);
if (*out == NULL)
return -1;
return 0;
git_sortedcache_unlock(backend->refcache);
return (*out != NULL) ? 0 : -1;
}
git_sortedcache_unlock(backend->refcache);
return GIT_ITEROVER;
}
......@@ -690,14 +548,10 @@ static int refdb_fs_backend__iterator_next_name(
{
refdb_fs_iter *iter = (refdb_fs_iter *)_iter;
refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend;
git_strmap *packfile = backend->refcache.packfile;
while (iter->loose_pos < iter->loose.length) {
const char *path = git_vector_get(&iter->loose, iter->loose_pos++);
if (git_strmap_exists(packfile, path))
continue;
if (loose_lookup(NULL, backend, path) != 0) {
giterr_clear();
continue;
......@@ -707,22 +561,25 @@ static int refdb_fs_backend__iterator_next_name(
return 0;
}
while (iter->packed_pos < kh_end(packfile)) {
while (!kh_exist(packfile, iter->packed_pos)) {
iter->packed_pos++;
if (iter->packed_pos == kh_end(packfile))
return GIT_ITEROVER;
}
git_sortedcache_lock(backend->refcache);
while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
struct packref *ref =
git_sortedcache_entry(backend->refcache, iter->packed_pos++);
if (ref->flags & PACKREF_SHADOWED)
continue;
*out = kh_key(packfile, iter->packed_pos);
iter->packed_pos++;
*out = ref->name;
if (iter->glob && p_fnmatch(iter->glob, *out, 0) != 0)
continue;
git_sortedcache_unlock(backend->refcache);
return 0;
}
git_sortedcache_unlock(backend->refcache);
return GIT_ITEROVER;
}
......@@ -730,18 +587,18 @@ static int refdb_fs_backend__iterator(
git_reference_iterator **out, git_refdb_backend *_backend, const char *glob)
{
refdb_fs_iter *iter;
refdb_fs_backend *backend;
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
assert(_backend);
backend = (refdb_fs_backend *)_backend;
assert(backend);
if (packed_load(backend) < 0)
if (packed_reload(backend) < 0)
return -1;
iter = git__calloc(1, sizeof(refdb_fs_iter));
GITERR_CHECK_ALLOC(iter);
if (git_pool_init(&iter->pool, 1, 0) < 0)
if (git_pool_init(&iter->pool, 1, 0) < 0 ||
git_vector_init(&iter->loose, 8, NULL) < 0)
goto fail;
if (glob != NULL &&
......@@ -786,15 +643,16 @@ static int reference_path_available(
const char* old_ref,
int force)
{
struct packref *this_ref;
size_t i;
if (packed_load(backend) < 0)
if (packed_reload(backend) < 0)
return -1;
if (!force) {
int exists;
if (refdb_fs_backend__exists(&exists, (git_refdb_backend *)backend, new_ref) < 0)
if (refdb_fs_backend__exists(
&exists, (git_refdb_backend *)backend, new_ref) < 0)
return -1;
if (exists) {
......@@ -805,14 +663,21 @@ static int reference_path_available(
}
}
git_strmap_foreach_value(backend->refcache.packfile, this_ref, {
git_sortedcache_lock(backend->refcache);
for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) {
struct packref *this_ref =
git_sortedcache_entry(backend->refcache, i);
if (!ref_is_available(old_ref, new_ref, this_ref->name)) {
git_sortedcache_unlock(backend->refcache);
giterr_set(GITERR_REFERENCE,
"The path to reference '%s' collides with an existing one", new_ref);
return -1;
}
});
}
git_sortedcache_unlock(backend->refcache);
return 0;
}
......@@ -839,12 +704,9 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref)
if (ref->type == GIT_REF_OID) {
char oid[GIT_OID_HEXSZ + 1];
git_oid_fmt(oid, &ref->target.oid);
oid[GIT_OID_HEXSZ] = '\0';
git_oid_nfmt(oid, sizeof(oid), &ref->target.oid);
git_filebuf_printf(&file, "%s\n", oid);
} else if (ref->type == GIT_REF_SYMBOLIC) {
git_filebuf_printf(&file, GIT_SYMREF "%s\n", ref->target.symbolic);
} else {
......@@ -854,14 +716,6 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref)
return git_filebuf_commit(&file, GIT_REFS_FILE_MODE);
}
static int packed_sort(const void *a, const void *b)
{
const struct packref *ref_a = (const struct packref *)a;
const struct packref *ref_b = (const struct packref *)b;
return strcmp(ref_a->name, ref_b->name);
}
/*
* Find out what object this reference resolves to.
*
......@@ -914,9 +768,7 @@ static int packed_find_peel(refdb_fs_backend *backend, struct packref *ref)
static int packed_write_ref(struct packref *ref, git_filebuf *file)
{
char oid[GIT_OID_HEXSZ + 1];
git_oid_fmt(oid, &ref->oid);
oid[GIT_OID_HEXSZ] = 0;
git_oid_nfmt(oid, sizeof(oid), &ref->oid);
/*
* For references that peel to an object in the repo, we must
......@@ -930,8 +782,7 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file)
*/
if (ref->flags & PACKREF_HAS_PEEL) {
char peel[GIT_OID_HEXSZ + 1];
git_oid_fmt(peel, &ref->peel);
peel[GIT_OID_HEXSZ] = 0;
git_oid_nfmt(peel, sizeof(peel), &ref->peel);
if (git_filebuf_printf(file, "%s %s\n^%s\n", oid, ref->name, peel) < 0)
return -1;
......@@ -954,16 +805,16 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file)
* is well-written, because we are destructing references
* here otherwise.
*/
static int packed_remove_loose(
refdb_fs_backend *backend,
git_vector *packing_list)
static int packed_remove_loose(refdb_fs_backend *backend)
{
size_t i;
git_buf full_path = GIT_BUF_INIT;
int failed = 0;
for (i = 0; i < packing_list->length; ++i) {
struct packref *ref = git_vector_get(packing_list, i);
/* backend->refcache is already locked when this is called */
for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) {
struct packref *ref = git_sortedcache_entry(backend->refcache, i);
if ((ref->flags & PACKREF_WAS_LOOSE) == 0)
continue;
......@@ -971,14 +822,13 @@ static int packed_remove_loose(
if (git_buf_joinpath(&full_path, backend->path, ref->name) < 0)
return -1; /* critical; do not try to recover on oom */
if (git_path_exists(full_path.ptr) == true && p_unlink(full_path.ptr) < 0) {
if (git_path_exists(full_path.ptr) && p_unlink(full_path.ptr) < 0) {
if (failed)
continue;
giterr_set(GITERR_REFERENCE,
"Failed to remove loose reference '%s' after packing: %s",
full_path.ptr, strerror(errno));
failed = 1;
}
......@@ -1002,33 +852,14 @@ static int packed_write(refdb_fs_backend *backend)
git_filebuf pack_file = GIT_FILEBUF_INIT;
size_t i;
git_buf pack_file_path = GIT_BUF_INIT;
git_vector packing_list;
unsigned int total_refs;
assert(backend && backend->refcache.packfile);
total_refs =
(unsigned int)git_strmap_num_entries(backend->refcache.packfile);
if (git_vector_init(&packing_list, total_refs, packed_sort) < 0)
/* lock the cache to updates while we do this */
if (git_sortedcache_lock(backend->refcache) < 0)
return -1;
/* Load all the packfile into a vector */
{
struct packref *reference;
/* cannot fail: vector already has the right size */
git_strmap_foreach_value(backend->refcache.packfile, reference, {
git_vector_insert(&packing_list, reference);
});
}
/* sort the vector so the entries appear sorted on the packfile */
git_vector_sort(&packing_list);
/* Now we can open the file! */
if (git_buf_joinpath(&pack_file_path,
backend->path, GIT_PACKEDREFS_FILE) < 0)
/* Open the file! */
if (git_buf_joinpath(
&pack_file_path, backend->path, GIT_PACKEDREFS_FILE) < 0)
goto cleanup_memory;
if (git_filebuf_open(&pack_file, pack_file_path.ptr, 0) < 0)
......@@ -1040,8 +871,9 @@ static int packed_write(refdb_fs_backend *backend)
if (git_filebuf_printf(&pack_file, "%s\n", GIT_PACKEDREFS_HEADER) < 0)
goto cleanup_packfile;
for (i = 0; i < packing_list.length; ++i) {
struct packref *ref = (struct packref *)git_vector_get(&packing_list, i);
for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) {
struct packref *ref =
git_sortedcache_entry(backend->refcache, i);
if (packed_find_peel(backend, ref) < 0)
goto cleanup_packfile;
......@@ -1057,16 +889,16 @@ static int packed_write(refdb_fs_backend *backend)
/* when and only when the packfile has been properly written,
* we can go ahead and remove the loose refs */
if (packed_remove_loose(backend, &packing_list) < 0)
goto cleanup_memory;
if (packed_remove_loose(backend) < 0)
goto cleanup_memory;
{
struct stat st;
if (p_stat(pack_file_path.ptr, &st) == 0)
backend->refcache.packfile_time = st.st_mtime;
}
git_sortedcache_unlock(backend->refcache);
/* update filestamp to latest value */
if (git_futils_filestamp_check(
&backend->refcache->stamp, pack_file_path.ptr) < 0)
giterr_clear();
git_vector_free(&packing_list);
git_buf_free(&pack_file_path);
/* we're good now */
......@@ -1076,7 +908,7 @@ cleanup_packfile:
git_filebuf_cleanup(&pack_file);
cleanup_memory:
git_vector_free(&packing_list);
git_sortedcache_unlock(backend->refcache);
git_buf_free(&pack_file_path);
return -1;
......@@ -1087,11 +919,10 @@ static int refdb_fs_backend__write(
const git_reference *ref,
int force)
{
refdb_fs_backend *backend;
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
int error;
assert(_backend);
backend = (refdb_fs_backend *)_backend;
assert(backend);
error = reference_path_available(backend, ref->name, NULL, force);
if (error < 0)
......@@ -1104,17 +935,13 @@ static int refdb_fs_backend__delete(
git_refdb_backend *_backend,
const char *ref_name)
{
refdb_fs_backend *backend;
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
git_buf loose_path = GIT_BUF_INIT;
struct packref *pack_ref;
khiter_t pack_ref_pos;
size_t pack_pos;
int error = 0;
bool loose_deleted = 0;
assert(_backend);
assert(ref_name);
backend = (refdb_fs_backend *)_backend;
assert(backend && ref_name);
/* If a loose reference exists, remove it from the filesystem */
if (git_buf_joinpath(&loose_path, backend->path, ref_name) < 0)
......@@ -1130,19 +957,23 @@ static int refdb_fs_backend__delete(
if (error != 0)
return error;
if (packed_reload(backend) < 0)
return -1;
/* If a packed reference exists, remove it from the packfile and repack */
error = packed_map_entry(&pack_ref, &pack_ref_pos, backend, ref_name);
if (git_sortedcache_lock(backend->refcache) < 0)
return -1;
if (error == GIT_ENOTFOUND)
return loose_deleted ? 0 : GIT_ENOTFOUND;
if (!(error = git_sortedcache_lookup_index(
&pack_pos, backend->refcache, ref_name)))
error = git_sortedcache_remove(backend->refcache, pack_pos, false);
if (error == 0) {
git_strmap_delete_at(backend->refcache.packfile, pack_ref_pos);
git__free(pack_ref);
error = packed_write(backend);
}
git_sortedcache_unlock(backend->refcache);
return error;
if (error == GIT_ENOTFOUND)
return loose_deleted ? 0 : ref_error_notfound(ref_name);
return packed_write(backend);
}
static int refdb_fs_backend__rename(
......@@ -1152,53 +983,44 @@ static int refdb_fs_backend__rename(
const char *new_name,
int force)
{
refdb_fs_backend *backend;
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
git_reference *old, *new;
int error;
assert(_backend);
backend = (refdb_fs_backend *)_backend;
assert(backend);
error = reference_path_available(backend, new_name, old_name, force);
if (error < 0)
return error;
error = refdb_fs_backend__lookup(&old, _backend, old_name);
if (error < 0)
if ((error = reference_path_available(
backend, new_name, old_name, force)) < 0 ||
(error = refdb_fs_backend__lookup(&old, _backend, old_name)) < 0)
return error;
error = refdb_fs_backend__delete(_backend, old_name);
if (error < 0) {
if ((error = refdb_fs_backend__delete(_backend, old_name)) < 0) {
git_reference_free(old);
return error;
}
new = realloc(old, sizeof(git_reference) + strlen(new_name) + 1);
memcpy(new->name, new_name, strlen(new_name) + 1);
error = loose_write(backend, new);
if (error < 0) {
git_reference_free(new);
return error;
new = git_reference__set_name(old, new_name);
if (!new) {
git_reference_free(old);
return -1;
}
if (out) {
*out = new;
} else {
if ((error = loose_write(backend, new)) < 0 || out == NULL) {
git_reference_free(new);
return error;
}
*out = new;
return 0;
}
static int refdb_fs_backend__compress(git_refdb_backend *_backend)
{
refdb_fs_backend *backend;
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
assert(_backend);
backend = (refdb_fs_backend *)_backend;
assert(backend);
if (packed_load(backend) < 0 || /* load the existing packfile */
if (packed_reload(backend) < 0 || /* load the existing packfile */
packed_loadloose(backend) < 0 || /* add all the loose refs */
packed_write(backend) < 0) /* write back to disk */
return -1;
......@@ -1206,29 +1028,13 @@ static int refdb_fs_backend__compress(git_refdb_backend *_backend)
return 0;
}
static void refcache_free(git_refcache *refs)
{
assert(refs);
if (refs->packfile) {
struct packref *reference;
git_strmap_foreach_value(refs->packfile, reference, {
git__free(reference);
});
git_strmap_free(refs->packfile);
}
}
static void refdb_fs_backend__free(git_refdb_backend *_backend)
{
refdb_fs_backend *backend;
refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
assert(_backend);
backend = (refdb_fs_backend *)_backend;
assert(backend);
refcache_free(&backend->refcache);
git_sortedcache_free(backend->refcache);
git__free(backend->path);
git__free(backend);
}
......@@ -1252,7 +1058,7 @@ static int setup_namespace(git_buf *path, git_repository *repo)
if (parts == NULL)
return -1;
/**
/*
* From `man gitnamespaces`:
* namespaces which include a / will expand to a hierarchy
* of namespaces; for example, GIT_NAMESPACE=foo/bar will store
......@@ -1269,7 +1075,7 @@ static int setup_namespace(git_buf *path, git_repository *repo)
if (git_futils_mkdir_r(git_buf_cstr(path), repo->path_repository, 0777) < 0)
return -1;
/* Return the root of the namespaced path, i.e. without the trailing '/refs' */
/* Return root of the namespaced path, i.e. without the trailing '/refs' */
git_buf_rtruncate_at_char(path, '/');
return 0;
}
......@@ -1286,13 +1092,19 @@ int git_refdb_backend_fs(
backend->repo = repository;
if (setup_namespace(&path, repository) < 0) {
git__free(backend);
return -1;
}
if (setup_namespace(&path, repository) < 0)
goto fail;
backend->path = git_buf_detach(&path);
if (git_buf_joinpath(&path, backend->path, GIT_PACKEDREFS_FILE) < 0 ||
git_sortedcache_new(
&backend->refcache, offsetof(struct packref, name),
NULL, NULL, packref_cmp, git_buf_cstr(&path)) < 0)
goto fail;
git_buf_free(&path);
backend->parent.exists = &refdb_fs_backend__exists;
backend->parent.lookup = &refdb_fs_backend__lookup;
backend->parent.iterator = &refdb_fs_backend__iterator;
......@@ -1304,4 +1116,10 @@ int git_refdb_backend_fs(
*backend_out = (git_refdb_backend *)backend;
return 0;
fail:
git_buf_free(&path);
git__free(backend->path);
git__free(backend);
return -1;
}
tests-clar/threads/refdb.c 0 → 100644
#include "clar_libgit2.h"
#include "git2/refdb.h"
#include "refdb.h"
static git_repository *g_repo;
static int g_expected = 0;
void test_threads_refdb__initialize(void)
{
g_repo = NULL;
}
void test_threads_refdb__cleanup(void)
{
cl_git_sandbox_cleanup();
g_repo = NULL;
}
#define REPEAT 20
#define THREADS 20
static void *iterate_refs(void *arg)
{
git_reference_iterator *i;
git_reference *ref;
int count = 0, *id = arg;
usleep(THREADS - *id);
cl_git_pass(git_reference_iterator_new(&i, g_repo));
for (count = 0; !git_reference_next(&ref, i); ++count) {
cl_assert(ref != NULL);
git_reference_free(ref);
}
if (g_expected > 0)
cl_assert_equal_i(g_expected, count);
git_reference_iterator_free(i);
return arg;
}
void test_threads_refdb__iterator(void)
{
int r, t;
git_thread th[THREADS];
int id[THREADS];
git_oid head;
git_reference *ref;
char name[128];
git_refdb *refdb;
g_repo = cl_git_sandbox_init("testrepo2");
cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
/* make a bunch of references */
for (r = 0; r < 200; ++r) {
snprintf(name, sizeof(name), "refs/heads/direct-%03d", r);
cl_git_pass(git_reference_create(&ref, g_repo, name, &head, 0));
git_reference_free(ref);
}
cl_git_pass(git_repository_refdb(&refdb, g_repo));
cl_git_pass(git_refdb_compress(refdb));
git_refdb_free(refdb);
g_expected = 206;
for (r = 0; r < REPEAT; ++r) {
g_repo = cl_git_sandbox_reopen(); /* reopen to flush caches */
for (t = 0; t < THREADS; ++t) {
id[t] = t;
cl_git_pass(git_thread_create(&th[t], NULL, iterate_refs, &id[t]));
}
for (t = 0; t < THREADS; ++t) {
cl_git_pass(git_thread_join(th[t], NULL));
}
memset(th, 0, sizeof(th));
}
}
static void *create_refs(void *arg)
{
int *id = arg, i;
git_oid head;
char name[128];
git_reference *ref[10];
cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
for (i = 0; i < 10; ++i) {
snprintf(name, sizeof(name), "refs/heads/thread-%03d-%02d", *id, i);
cl_git_pass(git_reference_create(&ref[i], g_repo, name, &head, 0));
if (i == 5) {
git_refdb *refdb;
cl_git_pass(git_repository_refdb(&refdb, g_repo));
cl_git_pass(git_refdb_compress(refdb));
git_refdb_free(refdb);
}
}
for (i = 0; i < 10; ++i)
git_reference_free(ref[i]);
return arg;
}
static void *delete_refs(void *arg)
{
int *id = arg, i;
git_reference *ref;
char name[128];
for (i = 0; i < 10; ++i) {
snprintf(
name, sizeof(name), "refs/heads/thread-%03d-%02d", (*id) & ~0x3, i);
if (!git_reference_lookup(&ref, g_repo, name)) {
fprintf(stderr, "deleting %s\n", name);
cl_git_pass(git_reference_delete(ref));
git_reference_delete(ref);
}
if (i == 5) {
git_refdb *refdb;
cl_git_pass(git_repository_refdb(&refdb, g_repo));
cl_git_pass(git_refdb_compress(refdb));
git_refdb_free(refdb);
}
}
return arg;
}
void test_threads_refdb__edit_while_iterate(void)
{
int r, t;
git_thread th[THREADS];
int id[THREADS];
git_oid head;
git_reference *ref;
char name[128];
git_refdb *refdb;
g_repo = cl_git_sandbox_init("testrepo2");
cl_git_pass(git_reference_name_to_id(&head, g_repo, "HEAD"));
/* make a bunch of references */
for (r = 0; r < 50; ++r) {
snprintf(name, sizeof(name), "refs/heads/starter-%03d", r);
cl_git_pass(git_reference_create(&ref, g_repo, name, &head, 0));
git_reference_free(ref);
}
cl_git_pass(git_repository_refdb(&refdb, g_repo));
cl_git_pass(git_refdb_compress(refdb));
git_refdb_free(refdb);
g_expected = -1;
g_repo = cl_git_sandbox_reopen(); /* reopen to flush caches */
for (t = 0; t < THREADS; ++t) {
void *(*fn)(void *arg);
switch (t & 0x3) {
case 0: fn = create_refs; break;
case 1: fn = delete_refs; break;
default: fn = iterate_refs; break;
}
id[t] = t;
cl_git_pass(git_thread_create(&th[t], NULL, fn, &id[t]));
}
for (t = 0; t < THREADS; ++t) {
cl_git_pass(git_thread_join(th[t], NULL));
}
memset(th, 0, sizeof(th));
for (t = 0; t < THREADS; ++t) {
id[t] = t;
cl_git_pass(git_thread_create(&th[t], NULL, iterate_refs, &id[t]));
}
for (t = 0; t < THREADS; ++t) {
cl_git_pass(git_thread_join(th[t], NULL));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment