Add support for SQLite backends

Configure again the build system to look for SQLite3. If the library is
found, the SQLite backend will be automatically compiled.

Enjoy *very* fast reads and writes.

MASTER PROTIP: Initialize the backend with ":memory" as the path to the
SQLite database for fully-hosted in-memory repositories. Rejoice.

Signed-off-by: Vicent Marti <tanoku@gmail.com>

src/backends/sqlite.c

+/*
+ * This file is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * In addition to the permissions in the GNU General Public License,
+ * the authors give you unlimited permission to link the compiled
+ * version of this file into combinations with other programs,
+ * and to distribute those combinations without any restriction
+ * coming from the use of this file.  (The General Public License
+ * restrictions do apply in other respects; for example, they cover
+ * modification of the file, and distribution when not linked into
+ * a combined executable.)
+ *
+ * This file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include "common.h"
+#include "git2/object.h"
+#include "hash.h"
+#include "odb.h"
+
+#include "git2/odb_backend.h"
+
+#ifdef GIT2_SQLITE_BACKEND
+
+#include <sqlite3.h>
+
+#define GIT2_TABLE_NAME "git2_odb"
+
+typedef struct {
+	git_odb_backend parent;
+	sqlite3 *db;
+	sqlite3_stmt *st_read;
+	sqlite3_stmt *st_write;
+	sqlite3_stmt *st_read_header;
+} sqlite_backend;
+
+int sqlite_backend__read_header(git_rawobj *obj, git_odb_backend *_backend, const git_oid *oid)
+{
+	sqlite_backend *backend;
+	int error;
+
+	assert(obj && _backend && oid);
+
+	backend = (sqlite_backend *)_backend;
+	error = GIT_ERROR;
+	obj->data = NULL;
+
+	if (sqlite3_bind_text(backend->st_read_header, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) {
+		if (sqlite3_step(backend->st_read_header) == SQLITE_ROW) {
+			obj->type = sqlite3_column_int(backend->st_read_header, 0);
+			obj->len = sqlite3_column_int(backend->st_read_header, 1);
+			assert(sqlite3_step(backend->st_read_header) == SQLITE_DONE);
+			error = GIT_SUCCESS;
+		} else {
+			error = GIT_ENOTFOUND;
+		}
+	}
+
+	sqlite3_reset(backend->st_read_header);
+	return error;
+}
+
+
+int sqlite_backend__read(git_rawobj *obj, git_odb_backend *_backend, const git_oid *oid)
+{
+	sqlite_backend *backend;
+	int error;
+
+	assert(obj && _backend && oid);
+
+	backend = (sqlite_backend *)_backend;
+	error = GIT_ERROR;
+
+	if (sqlite3_bind_text(backend->st_read, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) {
+		if (sqlite3_step(backend->st_read) == SQLITE_ROW) {
+			obj->type = sqlite3_column_int(backend->st_read, 0);
+			obj->len = sqlite3_column_int(backend->st_read, 1);
+			obj->data = git__malloc(obj->len);
+
+			if (obj->data == NULL) {
+				error = GIT_ENOMEM;
+			} else {
+				memcpy(obj->data, sqlite3_column_blob(backend->st_read, 2), obj->len);
+				error = GIT_SUCCESS;
+			}
+
+			assert(sqlite3_step(backend->st_read) == SQLITE_DONE);
+		} else {
+			error = GIT_ENOTFOUND;
+		}
+	}
+
+	sqlite3_reset(backend->st_read);
+	return error;
+}
+
+int sqlite_backend__exists(git_odb_backend *_backend, const git_oid *oid)
+{
+	sqlite_backend *backend;
+	int found;
+
+	assert(_backend && oid);
+
+	backend = (sqlite_backend *)_backend;
+	found = 0;
+
+	if (sqlite3_bind_text(backend->st_read_header, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) {
+		if (sqlite3_step(backend->st_read_header) == SQLITE_ROW) {
+			found = 1;
+			assert(sqlite3_step(backend->st_read_header) == SQLITE_DONE);
+		}
+	}
+
+	sqlite3_reset(backend->st_read_header);
+	return found;
+}
+
+
+int sqlite_backend__write(git_oid *id, git_odb_backend *_backend, git_rawobj *obj)
+{
+	char hdr[64];
+	int  hdrlen;
+
+	int error;
+	sqlite_backend *backend;
+
+	assert(id && _backend && obj);
+
+	backend = (sqlite_backend *)_backend;
+
+	if ((error = git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, obj)) < 0)
+		return error;
+
+	error = SQLITE_ERROR;
+
+	if (sqlite3_bind_text(backend->st_write, 1, (char *)id->id, 20, SQLITE_TRANSIENT) == SQLITE_OK &&
+		sqlite3_bind_int(backend->st_write, 2, (int)obj->type) == SQLITE_OK &&
+		sqlite3_bind_int(backend->st_write, 3, obj->len) == SQLITE_OK &&
+		sqlite3_bind_blob(backend->st_write, 4, obj->data, obj->len, SQLITE_TRANSIENT) == SQLITE_OK) {
+		error = sqlite3_step(backend->st_write);
+	}
+
+	sqlite3_reset(backend->st_write);
+	return (error == SQLITE_DONE) ? GIT_SUCCESS : GIT_ERROR;
+}
+
+
+void sqlite_backend__free(git_odb_backend *_backend)
+{
+	sqlite_backend *backend;
+	assert(_backend);
+	backend = (sqlite_backend *)_backend;
+
+	sqlite3_finalize(backend->st_read);
+	sqlite3_finalize(backend->st_read_header);
+	sqlite3_finalize(backend->st_write);
+	sqlite3_close(backend->db);
+
+	free(backend);
+}
+
+static int create_table(sqlite3 *db)
+{
+	static const char *sql_creat =
+		"CREATE TABLE '" GIT2_TABLE_NAME "' ("
+		"'oid' CHARACTER(20) PRIMARY KEY NOT NULL,"
+		"'type' INTEGER NOT NULL,"
+		"'size' INTEGER NOT NULL,"
+		"'data' BLOB);";
+
+	if (sqlite3_exec(db, sql_creat, NULL, NULL, NULL) != SQLITE_OK)
+		return GIT_ERROR;
+
+	return GIT_SUCCESS;
+}
+
+static int init_db(sqlite3 *db)
+{
+	static const char *sql_check =
+		"SELECT name FROM sqlite_master WHERE type='table' AND name='" GIT2_TABLE_NAME "';";
+
+	sqlite3_stmt *st_check;
+	int error;
+
+	if (sqlite3_prepare_v2(db, sql_check, -1, &st_check, NULL) != SQLITE_OK)
+		return GIT_ERROR;
+
+	switch (sqlite3_step(st_check)) {
+	case SQLITE_DONE:
+		/* the table was not found */
+		error = create_table(db);
+		break;
+
+	case SQLITE_ROW:
+		/* the table was found */
+		error = GIT_SUCCESS;
+		break;
+
+	default:
+		error = GIT_ERROR;
+		break;
+	}
+
+	sqlite3_finalize(st_check);
+	return error;
+}
+
+static int init_statements(sqlite_backend *backend)
+{
+	static const char *sql_read =
+		"SELECT type, size, data FROM '" GIT2_TABLE_NAME "' WHERE oid = ?;";
+
+	static const char *sql_read_header =
+		"SELECT type, size FROM '" GIT2_TABLE_NAME "' WHERE oid = ?;";
+
+	static const char *sql_write =
+		"INSERT OR IGNORE INTO '" GIT2_TABLE_NAME "' VALUES (?, ?, ?, ?);";
+
+	if (sqlite3_prepare_v2(backend->db, sql_read, -1, &backend->st_read, NULL) != SQLITE_OK)
+		return GIT_ERROR;
+
+	if (sqlite3_prepare_v2(backend->db, sql_read_header, -1, &backend->st_read_header, NULL) != SQLITE_OK)
+		return GIT_ERROR;
+
+	if (sqlite3_prepare_v2(backend->db, sql_write, -1, &backend->st_write, NULL) != SQLITE_OK)
+		return GIT_ERROR;
+
+	return GIT_SUCCESS;
+}
+
+int git_odb_backend_sqlite(git_odb_backend **backend_out, const char *sqlite_db)
+{
+	sqlite_backend *backend;
+	int error;
+
+	backend = git__calloc(1, sizeof(sqlite_backend));
+	if (backend == NULL)
+		return GIT_ENOMEM;
+
+	if (sqlite3_open(sqlite_db, &backend->db) != SQLITE_OK)
+		goto cleanup;
+
+	error = init_db(backend->db);
+	if (error < 0)
+		goto cleanup;
+
+	error = init_statements(backend);
+	if (error < 0)
+		goto cleanup;
+
+	backend->parent.read = &sqlite_backend__read;
+	backend->parent.read_header = &sqlite_backend__read_header;
+	backend->parent.write = &sqlite_backend__write;
+	backend->parent.exists = &sqlite_backend__exists;
+	backend->parent.free = &sqlite_backend__free;
+
+	backend->parent.priority = 0; 
+
+	*backend_out = (git_odb_backend *)backend;
+	return GIT_SUCCESS;
+
+cleanup:
+	sqlite_backend__free((git_odb_backend *)backend);
+	return GIT_ERROR;
+}
+
+#endif /* HAVE_SQLITE3 */

src/git2/odb_backend.h

@@ -66,6 +66,13 @@ struct git_odb_backend {
 	void (* free)(struct git_odb_backend *);
 };
 
+GIT_EXTERN(int) git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir);
+GIT_EXTERN(int) git_odb_backend_loose(git_odb_backend **backend_out, const char *objects_dir);
+
+#ifdef GIT2_SQLITE_BACKEND
+GIT_EXTERN(int) git_odb_backend_sqlite(git_odb_backend **backend_out, const char *sqlite_db);
+#endif
+
 GIT_END_DECL
 
 #endif

src/t03-data.h

 
-static char *odb_dir = "test-objects";
-
 typedef struct object_data {
     char *id;     /* object id (sha1)                          */
     char *dir;    /* object store (fan-out) directory name     */

tests/t03-objwrite.c

@@ -23,10 +23,11 @@
  * Boston, MA 02110-1301, USA.
  */
 #include "test_lib.h"
-#include "t03-data.h"
-
 #include "fileops.h"
 
+static char *odb_dir = "test-objects";
+#include "t03-data.h"
+
 static int make_odb_dir(void)
 {
 	if (gitfo_mkdir(odb_dir, 0755) < 0) {

tests/t11-sqlite.c

+/*
+ * This file is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * In addition to the permissions in the GNU General Public License,
+ * the authors give you unlimited permission to link the compiled
+ * version of this file into combinations with other programs,
+ * and to distribute those combinations without any restriction
+ * coming from the use of this file.  (The General Public License
+ * restrictions do apply in other respects; for example, they cover
+ * modification of the file, and distribution when not linked into
+ * a combined executable.)
+ *
+ * This file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+#include "test_lib.h"
+#include "t03-data.h"
+
+#include "fileops.h"
+#include "git2/odb_backend.h"
+
+static int cmp_objects(git_rawobj *o1, git_rawobj *o2)
+{
+	if (o1->type != o2->type)
+		return -1;
+	if (o1->len != o2->len)
+		return -1;
+	if ((o1->len > 0) && (memcmp(o1->data, o2->data, o1->len) != 0))
+		return -1;
+	return 0;
+}
+
+static git_odb *open_sqlite_odb(void)
+{
+#ifdef GIT2_SQLITE_BACKEND
+	git_odb *odb;
+	git_odb_backend *sqlite;
+
+	if (git_odb_new(&odb) < GIT_SUCCESS)
+		return NULL;
+
+	if (git_odb_backend_sqlite(&sqlite, ":memory") < GIT_SUCCESS)
+		return NULL;
+
+	if (git_odb_add_backend(odb, sqlite) < GIT_SUCCESS)
+		return NULL;
+
+	return odb;
+#else
+	return NULL;
+#endif
+}
+
+#define TEST_WRITE(PTR) {\
+    git_odb *db; \
+	git_oid id1, id2; \
+    git_rawobj obj; \
+	db = open_sqlite_odb(); \
+	must_be_true(db != NULL); \
+    must_pass(git_oid_mkstr(&id1, PTR.id)); \
+    must_pass(git_odb_write(&id2, db, &PTR##_obj)); \
+    must_be_true(git_oid_cmp(&id1, &id2) == 0); \
+    must_pass(git_odb_read(&obj, db, &id1)); \
+    must_pass(cmp_objects(&obj, &PTR##_obj)); \
+    git_rawobj_close(&obj); \
+    git_odb_close(db); \
+}
+
+BEGIN_TEST("sqlite", sql_write_commit)
+	TEST_WRITE(commit);
+END_TEST
+
+BEGIN_TEST("sqlite", sql_write_tree)
+	TEST_WRITE(tree);
+END_TEST
+
+BEGIN_TEST("sqlite", sql_write_tag)
+	TEST_WRITE(tag);
+END_TEST
+
+BEGIN_TEST("sqlite", sql_write_zero)
+	TEST_WRITE(zero);
+END_TEST
+
+BEGIN_TEST("sqlite", sql_write_one)
+	TEST_WRITE(one);
+END_TEST
+
+BEGIN_TEST("sqlite", sql_write_two)
+	TEST_WRITE(two);
+END_TEST
+
+BEGIN_TEST("sqlite", sql_write_some)
+	TEST_WRITE(some);
+END_TEST
+
+
+git_testsuite *libgit2_suite_sqlite(void)
+{
+	git_testsuite *suite = git_testsuite_new("SQLite Backend");
+
+#ifdef GIT2_SQLITE_BACKEND
+	ADD_TEST(suite, "sqlite", sql_write_commit);
+	ADD_TEST(suite, "sqlite", sql_write_tree);
+	ADD_TEST(suite, "sqlite", sql_write_tag);
+	ADD_TEST(suite, "sqlite", sql_write_zero);
+	ADD_TEST(suite, "sqlite", sql_write_one);
+	ADD_TEST(suite, "sqlite", sql_write_two);
+	ADD_TEST(suite, "sqlite", sql_write_some);
+#endif
+
+	return suite;
+}
+

tests/test_main.c

@@ -40,6 +40,7 @@ extern git_testsuite *libgit2_suite_hashtable(void);
 extern git_testsuite *libgit2_suite_tag(void);
 extern git_testsuite *libgit2_suite_tree(void);
 extern git_testsuite *libgit2_suite_refs(void);
+extern git_testsuite *libgit2_suite_sqlite(void);
 
 typedef git_testsuite *(*libgit2_suite)(void);
 
@@ -54,7 +55,8 @@ static libgit2_suite suite_methods[]= {
 	libgit2_suite_hashtable,
 	libgit2_suite_tag,
 	libgit2_suite_tree,
-	libgit2_suite_refs
+	libgit2_suite_refs,
+	libgit2_suite_sqlite,
 };
 
 #define GIT_SUITE_COUNT (ARRAY_SIZE(suite_methods))

wscript

@@ -16,7 +16,7 @@ CFLAGS_WIN32_L = ['/RELEASE']  # used for /both/ debug and release builds.
                                # sets the module's checksum in the header.
 CFLAGS_WIN32_L_DBG = ['/DEBUG']
 
-ALL_LIBS = ['z', 'crypto', 'pthread']
+ALL_LIBS = ['z', 'crypto', 'pthread', 'sqlite3']
 
 def options(opt):
     opt.load('compiler_c')
@@ -58,13 +58,17 @@ def configure(conf):
             zlib_name = 'zlibwapi'
 
         elif conf.env.CC_NAME == 'gcc':
-            conf.check(features='c cprogram', lib='pthread', uselib_store='pthread')
+            conf.check_cc(lib='pthread', uselib_store='pthread')
 
     else:
         conf.env.PLATFORM = 'unix'
 
     # check for Z lib
-    conf.check(features='c cprogram', lib=zlib_name, uselib_store='z', install_path=None)
+    conf.check_cc(lib=zlib_name, uselib_store='z', install_path=None)
+
+    # check for sqlite3
+    if conf.check_cc(lib='sqlite3', uselib_store='sqlite3', install_path=None, mandatory=False):
+        conf.env.DEFINES += ['GIT2_SQLITE_BACKEND']
 
     if conf.options.sha1 not in ['openssl', 'ppc', 'builtin']:
         ctx.fatal('Invalid SHA1 option')
@@ -115,6 +119,7 @@ def build_library(bld, build_type):
     # E.g.  src/unix/*.c
     #       src/win32/*.c
     sources = sources + directory.ant_glob('src/%s/*.c' % bld.env.PLATFORM)
+    sources = sources + directory.ant_glob('src/backends/*.c')
 
     # SHA1 methods source
     if bld.env.sha1 == "ppc":