Merge pull request #5243 from pks-t/pks/config-optimize-mem

Memory optimizations for config entries

src/config_entries.c

@@ -11,6 +11,7 @@ typedef struct config_entry_list {
 	struct config_entry_list *next;
 	struct config_entry_list *last;
 	git_config_entry *entry;
+	bool first;
 } config_entry_list;
 
 typedef struct config_entries_iterator {
@@ -25,31 +26,6 @@ struct git_config_entries {
 	config_entry_list *list;
 };
 
-static void config_entry_list_free(config_entry_list *list)
-{
-	config_entry_list *next;
-
-	while (list != NULL) {
-		next = list->next;
-
-		git__free((char*) list->entry->name);
-		git__free((char *) list->entry->value);
-		git__free(list->entry);
-		git__free(list);
-
-		list = next;
-	};
-}
-
-static void config_entry_list_append(config_entry_list **list, config_entry_list *entry)
-{
-	if (*list)
-		(*list)->last->next = entry;
-	else
-		*list = entry;
-	(*list)->last = entry;
-}
-
 int git_config_entries_new(git_config_entries **out)
 {
 	git_config_entries *entries;
@@ -127,12 +103,15 @@ static void config_entries_free(git_config_entries *entries)
 {
 	config_entry_list *list = NULL, *next;
 
-	git_strmap_foreach_value(entries->map, list, config_entry_list_free(list));
 	git_strmap_free(entries->map);
 
 	list = entries->list;
 	while (list != NULL) {
 		next = list->next;
+		if (list->first)
+			git__free((char *) list->entry->name);
+		git__free((char *) list->entry->value);
+		git__free(list->entry);
 		git__free(list);
 		list = next;
 	}
@@ -148,46 +127,33 @@ void git_config_entries_free(git_config_entries *entries)
 
 int git_config_entries_append(git_config_entries *entries, git_config_entry *entry)
 {
-	config_entry_list *existing, *var;
-	int error = 0;
-
-	var = git__calloc(1, sizeof(config_entry_list));
-	GIT_ERROR_CHECK_ALLOC(var);
-	var->entry = entry;
-
-	if ((existing = git_strmap_get(entries->map, entry->name)) == NULL) {
-		/*
-		 * We only ever inspect `last` from the first config
-		 * entry in a multivar. In case where this new entry is
-		 * the first one in the entry map, it will also be the
-		 * last one at the time of adding it, which is
-		 * why we set `last` here to itself. Otherwise we
-		 * do not have to set `last` and leave it set to
-		 * `NULL`.
-		 */
-		var->last = var;
-
-		error = git_strmap_set(entries->map, entry->name, var);
+	config_entry_list *existing, *head;
+
+	head = git__calloc(1, sizeof(config_entry_list));
+	GIT_ERROR_CHECK_ALLOC(head);
+	head->entry = entry;
+
+	/*
+	 * This is a micro-optimization for configuration files
+	 * with a lot of same keys. As for multivars the entry's
+	 * key will be the same for all entries, we can just free
+	 * all except the first entry's name and just re-use it.
+	 */
+	if ((existing = git_strmap_get(entries->map, entry->name)) != NULL) {
+		git__free((char *) entry->name);
+		entry->name = existing->entry->name;
 	} else {
-		config_entry_list_append(&existing, var);
+		head->first = 1;
 	}
 
-	var = git__calloc(1, sizeof(config_entry_list));
-	GIT_ERROR_CHECK_ALLOC(var);
-	var->entry = entry;
-	config_entry_list_append(&entries->list, var);
-
-	return error;
-}
-
-int config_entry_get(config_entry_list **out, git_config_entries *entries, const char *key)
-{
-	config_entry_list *list;
-
-	if ((list = git_strmap_get(entries->map, key)) == NULL)
-		return GIT_ENOTFOUND;
+	if (entries->list)
+		entries->list->last->next = head;
+	else
+		entries->list = head;
+	entries->list->last = head;
 
-	*out = list;
+	if (git_strmap_set(entries->map, entry->name, head) < 0)
+		return -1;
 
 	return 0;
 }
@@ -195,24 +161,20 @@ int config_entry_get(config_entry_list **out, git_config_entries *entries, const
 int git_config_entries_get(git_config_entry **out, git_config_entries *entries, const char *key)
 {
 	config_entry_list *entry;
-	int error;
-
-	if ((error = config_entry_get(&entry, entries, key)) < 0)
-		return error;
-	*out = entry->last->entry;
-
+	if ((entry = git_strmap_get(entries->map, key)) == NULL)
+		return GIT_ENOTFOUND;
+	*out = entry->entry;
 	return 0;
 }
 
 int git_config_entries_get_unique(git_config_entry **out, git_config_entries *entries, const char *key)
 {
 	config_entry_list *entry;
-	int error;
 
-	if ((error = config_entry_get(&entry, entries, key)) < 0)
-		return error;
+	if ((entry = git_strmap_get(entries->map, key)) == NULL)
+		return GIT_ENOTFOUND;
 
-	if (entry->next != NULL) {
+	if (!entry->first) {
 		git_error_set(GIT_ERROR_CONFIG, "entry is not unique due to being a multivar");
 		return -1;
 	}
@@ -227,14 +189,14 @@ int git_config_entries_get_unique(git_config_entry **out, git_config_entries *en
 	return 0;
 }
 
-void config_iterator_free(git_config_iterator *iter)
+static void config_iterator_free(git_config_iterator *iter)
 {
 	config_entries_iterator *it = (config_entries_iterator *) iter;
 	git_config_entries_free(it->entries);
 	git__free(it);
 }
 
-int config_iterator_next(
+static int config_iterator_next(
 	git_config_entry **entry,
 	git_config_iterator *iter)
 {

tests/config/stress.c

@@ -176,3 +176,23 @@ void test_config_stress__foreach_refreshes_snapshot(void)
 	git_config_free(config);
 	git__free(value);
 }
+
+void test_config_stress__huge_section_with_many_values(void)
+{
+	git_config *config;
+
+	/*
+	 * The config file is structured in such a way that is
+	 * has a section header that is approximately 500kb of
+	 * size followed by 40k entries. While the resulting
+	 * configuration file itself is roughly 650kb in size and
+	 * thus considered to be rather small, in the past we'd
+	 * balloon to more than 20GB of memory (20000x500kb)
+	 * while parsing the file. It thus was a trivial way to
+	 * cause an out-of-memory situation and thus cause denial
+	 * of service, e.g. via gitmodules.
+	 */
+	cl_git_pass(git_config_open_ondisk(&config, cl_fixture("config/config-oom")));
+
+	git_config_free(config);
+}