Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

remote: introduce config to set prefetch refs #1782

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

remote: introduce config to set prefetch refs #1782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Documentation/config/remote.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ remote.<name>.fetch::
The default set of "refspec" for linkgit:git-fetch[1]. See
linkgit:git-fetch[1].

remote.<name>.prefetchref::
Specify the refs to be prefetched when fetching from this
remote. The value is a space-separated list of ref patterns
(e.g., "refs/heads/main !refs/heads/develop*"). This can be
used to optimize fetch operations by specifying exactly which
refs should be prefetched.

remote.<name>.push::
The default set of "refspec" for linkgit:git-push[1]. See
linkgit:git-push[1].
Expand Down
53 changes: 53 additions & 0 deletions builtin/fetch.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,32 @@ static void filter_prefetch_refspec(struct refspec *rs)
}
}

static int pattern_matches_ref(const char *pattern, const char *refname)
{
if (strchr(pattern, '*'))
return match_refspec_name_with_pattern(pattern, refname, NULL, NULL) != 0;
return strcmp(pattern, refname) == 0;
}

static int matches_prefetch_refs(const char *refname, const struct string_list *prefetch_refs)
{
int has_positive = 0, matched_positive = 0, matched_negative = 0;

for (int i = 0; i < prefetch_refs->nr; i++) {
const char *pattern = prefetch_refs->items[i].string;
int is_negative = (*pattern == '!');
if (is_negative) pattern++;
else has_positive = 1;

if (pattern_matches_ref(pattern, refname)) {
if (is_negative) matched_negative = 1;
else matched_positive = 1;
}
}

return has_positive ? (matched_positive && !matched_negative) : !matched_negative;
}

static struct ref *get_ref_map(struct remote *remote,
const struct ref *remote_refs,
struct refspec *rs,
Expand All @@ -501,7 +527,11 @@ static struct ref *get_ref_map(struct remote *remote,
struct hashmap existing_refs;
int existing_refs_populated = 0;

struct ref *prefetch_filtered_ref_map = NULL, **ref_map_tail = &prefetch_filtered_ref_map;
struct ref *next;

filter_prefetch_refspec(rs);

if (remote)
filter_prefetch_refspec(&remote->fetch);

Expand Down Expand Up @@ -610,6 +640,29 @@ static struct ref *get_ref_map(struct remote *remote,
else
ref_map = apply_negative_refspecs(ref_map, &remote->fetch);

/**
* Filter out advertised refs that we don't want to fetch during
* prefetch if a prefetchref config is set
*/

if (prefetch && remote->prefetch_refs.nr) {
prefetch_filtered_ref_map = NULL;
ref_map_tail = &prefetch_filtered_ref_map;

for (rm = ref_map; rm; rm = next) {
next = rm->next;
rm->next = NULL;

if (matches_prefetch_refs(rm->name, &remote->prefetch_refs)) {
*ref_map_tail = rm;
ref_map_tail = &rm->next;
} else {
free_one_ref(rm);
}
}
ref_map = prefetch_filtered_ref_map;
}

ref_map = ref_remove_duplicates(ref_map);

for (rm = ref_map; rm; rm = rm->next) {
Expand Down
24 changes: 16 additions & 8 deletions remote.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ static struct remote *make_remote(struct remote_state *remote_state,
ret->prune = -1; /* unspecified */
ret->prune_tags = -1; /* unspecified */
ret->name = xstrndup(name, len);
string_list_init_dup(&ret->prefetch_refs);
refspec_init(&ret->push, REFSPEC_PUSH);
refspec_init(&ret->fetch, REFSPEC_FETCH);

Expand All @@ -166,6 +167,7 @@ static void remote_clear(struct remote *remote)
free((char *)remote->uploadpack);
FREE_AND_NULL(remote->http_proxy);
FREE_AND_NULL(remote->http_proxy_authmethod);
string_list_clear(&remote->prefetch_refs, 0);
}

static void add_merge(struct branch *branch, const char *name)
Expand Down Expand Up @@ -456,6 +458,12 @@ static int handle_config(const char *key, const char *value,
remote->prune = git_config_bool(key, value);
else if (!strcmp(subkey, "prunetags"))
remote->prune_tags = git_config_bool(key, value);
else if (!strcmp(subkey, "prefetchref")) {
if (!value)
return config_error_nonbool(key);
string_list_split(&remote->prefetch_refs, value, ' ', -1);
return 0;
}
else if (!strcmp(subkey, "url")) {
if (!value)
return config_error_nonbool(key);
Expand Down Expand Up @@ -868,7 +876,7 @@ struct strvec *push_url_of_remote(struct remote *remote)
return remote->pushurl.nr ? &remote->pushurl : &remote->url;
}

static int match_name_with_pattern(const char *key, const char *name,
int match_refspec_name_with_pattern(const char *key, const char *name,
const char *value, char **result)
{
const char *kstar = strchr(key, '*');
Expand Down Expand Up @@ -900,7 +908,7 @@ static int refspec_match(const struct refspec_item *refspec,
const char *name)
{
if (refspec->pattern)
return match_name_with_pattern(refspec->src, name, NULL, NULL);
return match_refspec_name_with_pattern(refspec->src, name, NULL, NULL);

return !strcmp(refspec->src, name);
}
Expand Down Expand Up @@ -969,7 +977,7 @@ static int query_matches_negative_refspec(struct refspec *rs, struct refspec_ite
const char *key = refspec->dst ? refspec->dst : refspec->src;
const char *value = refspec->src;

if (match_name_with_pattern(key, needle, value, &expn_name))
if (match_refspec_name_with_pattern(key, needle, value, &expn_name))
string_list_append_nodup(&reversed, expn_name);
} else if (refspec->matching) {
/* For the special matching refspec, any query should match */
Expand Down Expand Up @@ -1014,7 +1022,7 @@ static void query_refspecs_multiple(struct refspec *rs,
if (!refspec->dst || refspec->negative)
continue;
if (refspec->pattern) {
if (match_name_with_pattern(key, needle, value, result))
if (match_refspec_name_with_pattern(key, needle, value, result))
string_list_append_nodup(results, *result);
} else if (!strcmp(needle, key)) {
string_list_append(results, value);
Expand Down Expand Up @@ -1043,7 +1051,7 @@ int query_refspecs(struct refspec *rs, struct refspec_item *query)
if (!refspec->dst || refspec->negative)
continue;
if (refspec->pattern) {
if (match_name_with_pattern(key, needle, value, result)) {
if (match_refspec_name_with_pattern(key, needle, value, result)) {
query->force = refspec->force;
return 0;
}
Expand Down Expand Up @@ -1456,9 +1464,9 @@ static char *get_ref_match(const struct refspec *rs, const struct ref *ref,
const char *dst_side = item->dst ? item->dst : item->src;
int match;
if (direction == FROM_SRC)
match = match_name_with_pattern(item->src, ref->name, dst_side, &name);
match = match_refspec_name_with_pattern(item->src, ref->name, dst_side, &name);
else
match = match_name_with_pattern(dst_side, ref->name, item->src, &name);
match = match_refspec_name_with_pattern(dst_side, ref->name, item->src, &name);
if (match) {
matching_refs = i;
break;
Expand Down Expand Up @@ -2076,7 +2084,7 @@ static struct ref *get_expanded_map(const struct ref *remote_refs,

if (strchr(ref->name, '^'))
continue; /* a dereference item */
if (match_name_with_pattern(refspec->src, ref->name,
if (match_refspec_name_with_pattern(refspec->src, ref->name,
refspec->dst, &expn_name) &&
!ignore_symref_update(expn_name, &scratch)) {
struct ref *cpy = copy_ref(ref);
Expand Down
6 changes: 6 additions & 0 deletions remote.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include "hashmap.h"
#include "refspec.h"
#include "strvec.h"
#include "string-list.h"

struct option;
struct transport_ls_refs_options;
Expand Down Expand Up @@ -77,6 +78,8 @@ struct remote {

struct refspec fetch;

struct string_list prefetch_refs;

/*
* The setting for whether to fetch tags (as a separate rule from the
* configured refspecs);
Expand Down Expand Up @@ -207,6 +210,9 @@ int count_refspec_match(const char *, struct ref *refs, struct ref **matched_ref

int check_ref_type(const struct ref *ref, int flags);

int match_refspec_name_with_pattern(const char *key, const char *name,
const char *value, char **result);

/*
* Free a single ref and its peer, or an entire list of refs and their peers,
* respectively.
Expand Down
85 changes: 85 additions & 0 deletions t/t7900-maintenance.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,91 @@ test_expect_success 'prefetch multiple remotes' '
test_subcommand git fetch remote2 $fetchargs <skip-remote1.txt
'

test_expect_success 'prefetch with positive prefetch ref patterns' '
test_create_repo filter-prefetch-positive &&
(
cd filter-prefetch-positive &&
test_commit initial &&
git clone . clone2 &&
git remote add remote2 "file://$(pwd)/clone2" &&

cd clone2 &&
git checkout -b feature && test_commit feature-commit-2 &&
git checkout -b wip/test && test_commit wip-test-commit-2 &&
git checkout -b topic/x && test_commit topic-x-commit-2 &&
git push -f origin feature wip/test topic/x &&
cd .. &&

git config remote.remote2.prefetchref "refs/heads/feature" &&
fetchargs="--prefetch --prune --no-tags --no-write-fetch-head \
--recurse-submodules=no --quiet" &&
GIT_TRACE2_EVENT="$(pwd)/prefetch-positive.txt" \
git maintenance run --task=prefetch 2>/dev/null &&
test_subcommand git fetch remote2 $fetchargs <prefetch-positive.txt &&

git rev-parse refs/prefetch/remotes/remote2/feature &&
test_must_fail git rev-parse refs/prefetch/remotes/remote2/wip/test &&
test_must_fail git rev-parse refs/prefetch/remotes/remote2/topic/x
)
'

test_expect_success 'prefetch with negative prefetch ref patterns' '
test_create_repo filter-prefetch-negative &&
(
cd filter-prefetch-negative &&
test_commit initial &&
git clone . clone3 &&
git remote add remote3 "file://$(pwd)/clone3" &&
cat .git/config &&

cd clone3 &&
git checkout -b feature && test_commit feature-commit-3 &&
git checkout -b wip/test && test_commit wip-test-commit-3 &&
git checkout -b topic/x && test_commit topic-x-commit-3 &&
git push -f origin feature wip/test topic/x &&
cd .. &&

git config remote.remote3.prefetchref "!refs/heads/wip/*" &&
fetchargs="--prefetch --prune --no-tags --no-write-fetch-head \
--recurse-submodules=no --quiet" &&
GIT_TRACE2_EVENT="$(pwd)/prefetch-negative.txt" \
git maintenance run --task=prefetch 2>/dev/null &&
test_subcommand git fetch remote3 $fetchargs <prefetch-negative.txt &&
git rev-parse refs/prefetch/remotes/remote3/feature &&
git rev-parse refs/prefetch/remotes/remote3/topic/x &&
test_must_fail git rev-parse refs/prefetch/remotes/remote3/wip/test
)
'

test_expect_success 'prefetch with positive & negative prefetch ref patterns' '
test_create_repo filter-prefetch-mixed &&
(
cd filter-prefetch-mixed &&
test_commit initial &&
git clone . clone4 &&
git remote add remote4 "file://$(pwd)/clone4" &&

cd clone4 &&
git checkout -b feature && test_commit feature-commit-4 &&
git checkout -b topic/x && test_commit topic-x-commit-4 &&
git checkout -b topic/y && test_commit topic-y-commit-4 &&
git push -f origin feature topic/x topic/y &&
cd .. &&

git config remote.remote4.prefetchref \
"refs/heads/topic/* !refs/heads/topic/y" &&
fetchargs="--prefetch --prune --no-tags --no-write-fetch-head \
--recurse-submodules=no --quiet" &&
GIT_TRACE2_EVENT="$(pwd)/prefetch-mixed.txt" \
git maintenance run --task=prefetch 2>/dev/null &&
test_subcommand git fetch remote4 $fetchargs <prefetch-mixed.txt &&

test_must_fail git rev-parse refs/prefetch/remotes/remote4/feature &&
test_must_fail git rev-parse refs/prefetch/remotes/remote4/topic/y &&
git rev-parse refs/prefetch/remotes/remote4/topic/x
)
'

test_expect_success 'loose-objects task' '
# Repack everything so we know the state of the object dir
git repack -adk &&
Expand Down
Loading