refs: catch cases where the ref type has changed

If the type of the on-disk reference has changed, the old value
comparison should fail.

src/refdb_fs.c

@@ -937,9 +937,21 @@ static int cmp_old_ref(int *cmp, git_refdb_backend *backend, const char *name,
 	git_reference *old_ref = NULL;
 
 	*cmp = 0;
-	if (old_id || old_target) {
-		if ((error = refdb_fs_backend__lookup(&old_ref, backend, name)) < 0)
-			goto out;
+	/* It "matches" if there is no old value to compare against */
+	if (!old_id && !old_target)
+		return 0;
+
+	if ((error = refdb_fs_backend__lookup(&old_ref, backend, name)) < 0)
+		goto out;
+
+	/* If the types don't match, there's no way the values do */
+	if (old_id && old_ref->type != GIT_REF_OID) {
+		*cmp = -1;
+		goto out;
+	}
+	if (old_target && old_ref->type != GIT_REF_SYMBOLIC) {
+		*cmp = 1;
+		goto out;
 	}
 
 	if (old_id && old_ref->type == GIT_REF_OID)

tests/refs/races.c

@@ -92,3 +92,61 @@ void test_refs_races__delete(void)
 	git_reference_free(ref);
 	git_reference_free(ref2);
 }
+
+void test_refs_races__switch_oid_to_symbolic(void)
+{
+	git_reference *ref, *ref2, *ref3;
+	git_oid id, other_id;
+
+	git_oid_fromstr(&id, commit_id);
+	git_oid_fromstr(&other_id, other_commit_id);
+
+	/* Removing a direct ref when it's currently symbolic should fail */
+	cl_git_pass(git_reference_lookup(&ref, g_repo, refname));
+	cl_git_pass(git_reference_symbolic_create(&ref2, g_repo, refname, other_refname, 1, NULL, NULL));
+	cl_git_fail_with(GIT_EMODIFIED, git_reference_delete(ref));
+
+	git_reference_free(ref);
+	git_reference_free(ref2);
+
+	cl_git_pass(git_reference_create(&ref, g_repo, refname, &id, 1, NULL, NULL));
+	git_reference_free(ref);
+
+	/* Updating a direct ref when it's currently symbolic should fail */
+	cl_git_pass(git_reference_lookup(&ref, g_repo, refname));
+	cl_git_pass(git_reference_symbolic_create(&ref2, g_repo, refname, other_refname, 1, NULL, NULL));
+	cl_git_fail_with(GIT_EMODIFIED, git_reference_set_target(&ref3, ref, &other_id, NULL, NULL));
+
+	git_reference_free(ref);
+	git_reference_free(ref2);
+	git_reference_free(ref3);
+}
+
+void test_refs_races__switch_symbolic_to_oid(void)
+{
+	git_reference *ref, *ref2, *ref3;
+	git_oid id, other_id;
+
+	git_oid_fromstr(&id, commit_id);
+	git_oid_fromstr(&other_id, other_commit_id);
+
+	/* Removing a symbolic ref when it's currently direct should fail */
+	cl_git_pass(git_reference_lookup(&ref, g_repo, "HEAD"));
+	cl_git_pass(git_reference_create(&ref2, g_repo, "HEAD", &id, 1, NULL, NULL));
+	cl_git_fail_with(GIT_EMODIFIED, git_reference_delete(ref));
+
+	git_reference_free(ref);
+	git_reference_free(ref2);
+
+	cl_git_pass(git_reference_symbolic_create(&ref, g_repo, "HEAD", refname, 1, NULL, NULL));
+	git_reference_free(ref);
+
+	/* Updating a symbolic ref when it's currently direct should fail */
+	cl_git_pass(git_reference_lookup(&ref, g_repo, "HEAD"));
+	cl_git_pass(git_reference_create(&ref2, g_repo, "HEAD", &id, 1, NULL, NULL));
+	cl_git_fail_with(GIT_EMODIFIED, git_reference_symbolic_set_target(&ref3, ref, other_refname, NULL, NULL));
+
+	git_reference_free(ref);
+	git_reference_free(ref2);
+	git_reference_free(ref3);
+}