admin/git
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bloom: enable bloom filter with wildcard pathspec in revision traversal

Browse Source 
When traversing commits, a pathspec item can be used to limit the
traversal to commits that modify the specified paths. And the
commit-graph includes a Bloom filter to exclude commits that definitely
did not modify a given pathspec item. During commit traversal, the
Bloom filter can significantly improve performance. However, it is
disabled if the specified pathspec item contains wildcard characters
or magic signatures.

For performance reason, enable Bloom filter even if a pathspec item
contains wildcard characters by filtering only the non-wildcard part of
the pathspec item.

The function of pathspec magic signature is generally to narrow down
the path specified by the pathspecs. So, enable Bloom filter when
the magic signature is "top", "glob", "attr", "--depth" or "literal".
"exclude" is used to select paths other than the specified path, rather
than serving as a filtering function, so it cannot be used together with
the Bloom filter. Since Bloom filter is not case insensitive even in
case insensitive system (e.g. MacOS), it cannot be used together with
"icase" magic.

With this optimization, we get some improvements for pathspecs with
wildcards or magic signatures. First, in the Git repository we see these
modest results:

git log -100 -- "t/*"

Benchmark 1: new
  Time (mean ± σ):      20.4 ms ±   0.6 ms
  Range (min … max):    19.3 ms …  24.4 ms

Benchmark 2: old
  Time (mean ± σ):      23.4 ms ±   0.5 ms
  Range (min … max):    22.5 ms …  24.7 ms

git log -100 -- ":(top)t"

Benchmark 1: new
  Time (mean ± σ):      16.2 ms ±   0.4 ms
  Range (min … max):    15.3 ms …  17.2 ms

Benchmark 2: old
  Time (mean ± σ):      18.6 ms ±   0.5 ms
  Range (min … max):    17.6 ms …  20.4 ms

But in a larger repo, such as the LLVM project repo below, we get even
better results:

git log -100 -- "libc/*"

Benchmark 1: new
  Time (mean ± σ):      16.0 ms ±   0.6 ms
  Range (min … max):    14.7 ms …  17.8 ms

Benchmark 2: old
  Time (mean ± σ):      26.7 ms ±   0.5 ms
  Range (min … max):    25.4 ms …  27.8 ms

git log -100 -- ":(top)libc"

Benchmark 1: new
  Time (mean ± σ):      15.6 ms ±   0.6 ms
  Range (min … max):    14.4 ms …  17.7 ms

Benchmark 2: old
  Time (mean ± σ):      19.6 ms ±   0.5 ms
  Range (min … max):    18.6 ms …  20.6 ms

Helped-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Lidong Yan <yldhome2d2@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Lidong Yan
2025-08-11 14:01:37 +08:00
committed by Junio C Hamano
parent 2c2ba49d55
commit 6d192462eb
2 changed files with 56 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
revision.c
View File

@@ -671,12 +671,17 @@ static void trace2_bloom_filter_statistics_atexit(void)
static int forbid_bloom_filters(struct pathspec *spec)
{
if (spec->has_wildcard)
return 1;
if (spec->magic & ~PATHSPEC_LITERAL)
unsigned int allowed_magic =
PATHSPEC_FROMTOP |
PATHSPEC_MAXDEPTH |
PATHSPEC_LITERAL |
PATHSPEC_GLOB |
PATHSPEC_ATTR;
if (spec->magic & ~allowed_magic)
return 1;
for (size_t nr = 0; nr < spec->nr; nr++)
if (spec->items[nr].magic & ~PATHSPEC_LITERAL)
if (spec->items[nr].magic & ~allowed_magic)
return 1;
return 0;
@@ -691,23 +696,34 @@ static int convert_pathspec_to_bloom_keyvec(struct bloom_keyvec **out,
char *path_alloc = NULL;
const char *path;
size_t len;
int res = 0;
int res = -1;
len = pi->nowildcard_len;
if (len != pi->len) {
/*
* for path like "dir/file*", nowildcard part would be
* "dir/file", but only "dir" should be used for the
* bloom filter.
*/
while (len > 0 && pi->match[len - 1] != '/')
len--;
}
/* remove single trailing slash from path, if needed */
if (pi->len > 0 && pi->match[pi->len - 1] == '/') {
path_alloc = xmemdupz(pi->match, pi->len - 1);
if (len > 0 && pi->match[len - 1] == '/')
len--;
if (!len)
goto cleanup;
if (len != pi->len) {
path_alloc = xmemdupz(pi->match, len);
path = path_alloc;
} else
path = pi->match;
len = strlen(path);
if (!len) {
res = -1;
goto cleanup;
}
*out = bloom_keyvec_new(path, len, settings);
res = 0;
cleanup:
free(path_alloc);
return res;

31
t/t4216-log-bloom.sh
View File

@@ -154,11 +154,34 @@ test_expect_success 'git log with multiple literal paths uses Bloom filter' '
test_bloom_filters_used "-- file*"
'
test_expect_success 'git log with path contains a wildcard does not use Bloom filter' '
test_expect_success 'git log with paths all contain non-wildcard part uses Bloom filter' '
test_bloom_filters_used "-- A/\* file4" &&
test_bloom_filters_used "-- A/file\*" &&
test_bloom_filters_used "-- * A/\*"
'
test_expect_success 'git log with path only contains wildcard part does not use Bloom filter' '
test_bloom_filters_not_used "-- file\*" &&
test_bloom_filters_not_used "-- A/\* file4" &&
test_bloom_filters_not_used "-- file4 A/\*" &&
test_bloom_filters_not_used "-- * A/\*"
test_bloom_filters_not_used "-- file\* A/\*" &&
test_bloom_filters_not_used "-- file\* *" &&
test_bloom_filters_not_used "-- \*"
'
test_expect_success 'git log with path contains various magic signatures' '
cd A &&
test_bloom_filters_used "-- \:\(top\)B" &&
cd .. &&
test_bloom_filters_used "-- \:\(glob\)A/\*\*/C" &&
test_bloom_filters_not_used "-- \:\(icase\)FILE4" &&
test_bloom_filters_not_used "-- \:\(exclude\)A/B/C" &&
test_when_finished "rm -f .gitattributes" &&
cat >.gitattributes <<-EOF &&
A/file1 text
A/B/file2 -text
EOF
test_bloom_filters_used "-- \:\(attr\:text\)A"
'
test_expect_success 'setup - add commit-graph to the chain without Bloom filters' '