Skip to content

Commit

Permalink
RepositoryCache: add blake3 key type
Browse files Browse the repository at this point in the history 
Support storing repository cache with blake3 hash function.

Closes #21998.

PiperOrigin-RevId: 625727463
Change-Id: I2198adac617c5d6301e56694fe416f3130c61033
  • Loading branch information
@sluongng @copybara-github
sluongng authored and copybara-github committed Apr 17, 2024
1 parent 32def70 commit dc5ec2b
Show file tree
Hide file tree
Showing 8 changed files with 101 additions and 50 deletions.
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ java_library(
srcs = ["RepositoryCache.java"],
deps = [
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs/bazel",
"//third_party:guava",
"//third_party:jsr305",
],
Expand Down
8 changes: 7 additions & 1 deletion src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.devtools.build.lib.vfs.FileAccessException;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.bazel.Blake3HashFunction;
import java.io.IOException;
import java.io.InputStream;
import java.util.UUID;
Expand All @@ -43,7 +44,8 @@ public enum KeyType {
SHA1("SHA-1", "\\p{XDigit}{40}", "sha1", Hashing.sha1()),
SHA256("SHA-256", "\\p{XDigit}{64}", "sha256", Hashing.sha256()),
SHA384("SHA-384", "\\p{XDigit}{96}", "sha384", Hashing.sha384()),
SHA512("SHA-512", "\\p{XDigit}{128}", "sha512", Hashing.sha512());
SHA512("SHA-512", "\\p{XDigit}{128}", "sha512", Hashing.sha512()),
BLAKE3("BLAKE3", "\\p{XDigit}{64}", "blake3", Blake3HashFunction.INSTANCE);

private final String stringRepr;
private final String regexp;
Expand All @@ -70,6 +72,10 @@ public Hasher newHasher() {
return hashFunction.newHasher();
}

public HashFunction getHashFunction() {
return hashFunction;
}

public String getHashName() {
return hashName;
}
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ public static Checksum fromSubresourceIntegrity(String integrity)
expectedLength = 64;
hash = decoder.decode(integrity.substring(7));
}
if (integrity.startsWith("blake3-")) {
keyType = KeyType.BLAKE3;
expectedLength = 32;
hash = decoder.decode(integrity.substring(7));
}

if (keyType == null) {
throw new InvalidChecksumException(
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public final class BazelHashFunctions {
if (JniLoader.isJniAvailable()) {
try {
Security.addProvider(new Blake3Provider());
hashFunction = DigestHashFunction.register(new Blake3HashFunction(), "BLAKE3");
hashFunction = DigestHashFunction.register(Blake3HashFunction.INSTANCE, "BLAKE3");
} catch (UnsatisfiedLinkError ignored) {
// This can happen if bazel was compiled manually (with compile.sh),
// on windows. In that case jni is available, but missing the blake3
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@

/** A {@link HashFunction} for BLAKE3. */
public final class Blake3HashFunction implements HashFunction {
public static final Blake3HashFunction INSTANCE = new Blake3HashFunction();

@Override
public int bits() {
return 256;
Expand Down
4 changes: 4 additions & 0 deletions src/test/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@ java_library(
srcs = glob(["*.java"]),
deps = [
"//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
"//src/main/java/com/google/devtools/build/lib/clock",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs/bazel",
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
"//src/test/java/com/google/devtools/build/lib/testutil",
"//src/test/java/com/google/devtools/build/lib/testutil:TestConstants",
"//third_party:guava",
"//third_party:junit4",
"//third_party:mockito",
Expand Down
126 changes: 78 additions & 48 deletions src/test/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCacheTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,23 +20,29 @@

import com.google.common.io.BaseEncoding;
import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType;
import com.google.devtools.build.lib.clock.JavaClock;
import com.google.devtools.build.lib.testutil.Scratch;
import com.google.devtools.build.lib.testutil.TestConstants;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.bazel.BazelHashFunctions;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;

/**
* Tests for {@link RepositoryCache}.
*/
@RunWith(JUnit4.class)
/** Tests for {@link RepositoryCache}. */
@RunWith(Parameterized.class)
public class RepositoryCacheTest {

@Rule public ExpectedException thrown = ExpectedException.none();
Expand All @@ -46,7 +52,44 @@ public class RepositoryCacheTest {
private Path repositoryCachePath;
private Path contentAddressableCachePath;
private Path downloadedFile;
private String downloadedFileSha256;

private final DigestHashFunction digestHashFunction;
private final KeyType keyType;
private final String hash;

@Parameters
public static List<Object[]> getKeyType() {
List<Object[]> keyTypes = new ArrayList<>();
keyTypes.add(
new Object[] {
// digestHashFunction
DigestHashFunction.SHA256,
// keyType
KeyType.SHA256,
// hash
"bfe5ed57e6e323555b379c660aa8d35b70c2f8f07cf03ad6747266495ac13be0",
// echo 'contents' | sha256sum
});
if (TestConstants.BLAKE3_AVAILABLE) {
keyTypes.add(
new Object[] {
// digestHashFunction
BazelHashFunctions.BLAKE3,
// keyType
KeyType.BLAKE3,
// hash
"54e00265e2516f168096da17059a6109563d9ba64a0b77cdc4b33e44600c2a39",
// echo 'contents' | b3sum
});
}
return keyTypes;
}

public RepositoryCacheTest(DigestHashFunction digestHashFunction, KeyType keyType, String hash) {
this.digestHashFunction = digestHashFunction;
this.keyType = keyType;
this.hash = hash;
}

@Before
public void setUp() throws Exception {
Expand All @@ -57,7 +100,6 @@ public void setUp() throws Exception {
contentAddressableCachePath = repositoryCache.getContentAddressableCachePath();

downloadedFile = scratch.file("file.tmp", Charset.defaultCharset(), "contents");
downloadedFileSha256 = "bfe5ed57e6e323555b379c660aa8d35b70c2f8f07cf03ad6747266495ac13be0";
}

@After
Expand All @@ -67,17 +109,16 @@ public void tearDown() throws IOException {

@Test
public void testNonExistentCacheValue() {
String fakeSha256 = "a".repeat(64);
assertThat(repositoryCache.exists(fakeSha256, KeyType.SHA256)).isFalse();
String fakeHash = "a".repeat(64);
assertThat(repositoryCache.exists(fakeHash, keyType)).isFalse();
}

/** Test that the put method correctly stores the downloaded file into the cache. */
@Test
public void testPutCacheValue() throws Exception {
repositoryCache.put(
downloadedFileSha256, downloadedFile, KeyType.SHA256, /* canonicalId= */ null);
repositoryCache.put(hash, downloadedFile, keyType, /* canonicalId= */ null);

Path cacheEntry = KeyType.SHA256.getCachePath(contentAddressableCachePath).getChild(downloadedFileSha256);
Path cacheEntry = keyType.getCachePath(contentAddressableCachePath).getChild(hash);
Path cacheValue = cacheEntry.getChild(RepositoryCache.DEFAULT_CACHE_FILENAME);

assertThat(FileSystemUtils.readContent(downloadedFile, Charset.defaultCharset()))
Expand All @@ -89,11 +130,10 @@ public void testPutCacheValue() throws Exception {
*/
@Test
public void testPutCacheValueWithoutHash() throws Exception {
String cacheKey = repositoryCache.put(downloadedFile, KeyType.SHA256, /* canonicalId= */ null);
assertThat(cacheKey).isEqualTo(downloadedFileSha256);
String cacheKey = repositoryCache.put(downloadedFile, keyType, /* canonicalId= */ null);
assertThat(cacheKey).isEqualTo(hash);

Path cacheEntry =
KeyType.SHA256.getCachePath(contentAddressableCachePath).getChild(downloadedFileSha256);
Path cacheEntry = keyType.getCachePath(contentAddressableCachePath).getChild(hash);
Path cacheValue = cacheEntry.getChild(RepositoryCache.DEFAULT_CACHE_FILENAME);

assertThat(FileSystemUtils.readContent(downloadedFile, Charset.defaultCharset()))
Expand All @@ -106,12 +146,10 @@ public void testPutCacheValueWithoutHash() throws Exception {
*/
@Test
public void testPutCacheValueIdempotent() throws Exception {
repositoryCache.put(
downloadedFileSha256, downloadedFile, KeyType.SHA256, /* canonicalId= */ null);
repositoryCache.put(
downloadedFileSha256, downloadedFile, KeyType.SHA256, /* canonicalId= */ null);
repositoryCache.put(hash, downloadedFile, keyType, /* canonicalId= */ null);
repositoryCache.put(hash, downloadedFile, keyType, /* canonicalId= */ null);

Path cacheEntry = KeyType.SHA256.getCachePath(contentAddressableCachePath).getChild(downloadedFileSha256);
Path cacheEntry = keyType.getCachePath(contentAddressableCachePath).getChild(hash);
Path cacheValue = cacheEntry.getChild(RepositoryCache.DEFAULT_CACHE_FILENAME);

assertThat(FileSystemUtils.readContent(downloadedFile, Charset.defaultCharset()))
Expand All @@ -122,14 +160,11 @@ public void testPutCacheValueIdempotent() throws Exception {
@Test
public void testGetCacheValue() throws Exception {
// Inject file into cache
repositoryCache.put(
downloadedFileSha256, downloadedFile, KeyType.SHA256, /* canonicalId= */ null);
repositoryCache.put(hash, downloadedFile, keyType, /* canonicalId= */ null);

Path targetDirectory = scratch.dir("/external");
Path targetPath = targetDirectory.getChild(downloadedFile.getBaseName());
Path actualTargetPath =
repositoryCache.get(
downloadedFileSha256, targetPath, KeyType.SHA256, /* canonicalId= */ null);
Path actualTargetPath = repositoryCache.get(hash, targetPath, keyType, /* canonicalId= */ null);

// Check that the contents are the same.
assertThat(FileSystemUtils.readContent(downloadedFile, Charset.defaultCharset()))
Expand All @@ -144,9 +179,7 @@ public void testGetCacheValue() throws Exception {
public void testGetNullCacheValue() throws Exception {
Path targetDirectory = scratch.dir("/external");
Path targetPath = targetDirectory.getChild(downloadedFile.getBaseName());
Path actualTargetPath =
repositoryCache.get(
downloadedFileSha256, targetPath, KeyType.SHA256, /* canonicalId= */ null);
Path actualTargetPath = repositoryCache.get(hash, targetPath, keyType, /* canonicalId= */ null);

assertThat(actualTargetPath).isNull();
}
Expand All @@ -155,14 +188,13 @@ public void testGetNullCacheValue() throws Exception {
public void testInvalidSha256Throws() throws Exception {
String invalidSha = "foo";
thrown.expect(IOException.class);
thrown.expectMessage("Invalid key \"foo\" of type SHA-256");
repositoryCache.put(invalidSha, downloadedFile, KeyType.SHA256, /* canonicalId= */ null);
thrown.expectMessage("Invalid key \"foo\" of type " + keyType);
repositoryCache.put(invalidSha, downloadedFile, keyType, /* canonicalId= */ null);
}

@Test
public void testPoisonedCache() throws Exception {
Path poisonedEntry = KeyType.SHA256
.getCachePath(contentAddressableCachePath).getChild(downloadedFileSha256);
Path poisonedEntry = keyType.getCachePath(contentAddressableCachePath).getChild(hash);
Path poisonedValue = poisonedEntry.getChild(RepositoryCache.DEFAULT_CACHE_FILENAME);
scratch.file(poisonedValue.getPathString(), Charset.defaultCharset(), "poisoned");

Expand All @@ -173,30 +205,32 @@ public void testPoisonedCache() throws Exception {
thrown.expectMessage("does not match expected");
thrown.expectMessage("Please delete the directory");

repositoryCache.get(downloadedFileSha256, targetPath, KeyType.SHA256, /* canonicalId= */ null);
repositoryCache.get(hash, targetPath, keyType, /* canonicalId= */ null);
}

@Test
public void testGetChecksum() throws Exception {
String actualChecksum = RepositoryCache.getChecksum(KeyType.SHA256, downloadedFile);
assertThat(actualChecksum).isEqualTo(downloadedFileSha256);
String actualChecksum = RepositoryCache.getChecksum(keyType, downloadedFile);
assertThat(actualChecksum).isEqualTo(hash);
}

@Test
public void testGetChecksumWithFastDigest() throws Exception {
String fastDigestChecksum = "cfe5ed57e6e323555b379c660aa8d35b70c2f8f07cf03ad6747266495ac13be0";
var fastDigestChecksum = "cfe5ed57e6e323555b379c660aa8d35b70c2f8f07cf03ad6747266495ac13be0";
var fs = new InMemoryFileSystem(new JavaClock(), digestHashFunction);
downloadedFile = spy(downloadedFile);
doReturn(BaseEncoding.base16().lowerCase().decode(fastDigestChecksum))
.when(downloadedFile)
.getFastDigest();
doReturn(fs).when(downloadedFile).getFileSystem();

String actualChecksum = RepositoryCache.getChecksum(KeyType.SHA256, downloadedFile);
String actualChecksum = RepositoryCache.getChecksum(keyType, downloadedFile);
assertThat(actualChecksum).isEqualTo(fastDigestChecksum);
}

@Test
public void testAssertFileChecksumPass() throws Exception {
RepositoryCache.assertFileChecksum(downloadedFileSha256, downloadedFile, KeyType.SHA256);
RepositoryCache.assertFileChecksum(hash, downloadedFile, keyType);
}

@Test
Expand All @@ -206,26 +240,22 @@ public void testAssertFileChecksumFail() throws Exception {
RepositoryCache.assertFileChecksum(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
downloadedFile,
KeyType.SHA256);
keyType);
}

@Test
public void testCanonicalId() throws Exception {
repositoryCache.put(downloadedFileSha256, downloadedFile, KeyType.SHA256, "fooid");
repositoryCache.put(hash, downloadedFile, keyType, "fooid");
Path targetDirectory = scratch.dir("/external");
Path targetPath = targetDirectory.getChild(downloadedFile.getBaseName());

Path lookupWithSameId =
repositoryCache.get(downloadedFileSha256, targetPath, KeyType.SHA256, "fooid");
Path lookupWithSameId = repositoryCache.get(hash, targetPath, keyType, "fooid");
assertThat(lookupWithSameId).isEqualTo(targetPath);

Path lookupOtherId =
repositoryCache.get(downloadedFileSha256, targetPath, KeyType.SHA256, "barid");
Path lookupOtherId = repositoryCache.get(hash, targetPath, keyType, "barid");
assertThat(lookupOtherId).isNull();

Path lookupNoId =
repositoryCache.get(
downloadedFileSha256, targetPath, KeyType.SHA256, /* canonicalId= */ null);
Path lookupNoId = repositoryCache.get(hash, targetPath, keyType, /* canonicalId= */ null);
assertThat(lookupNoId).isEqualTo(targetPath);
}
}
3 changes: 3 additions & 0 deletions src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,9 @@ private TestConstants() {
/** The cpp toolchain type. */
public static final String CPP_TOOLCHAIN_TYPE = "@@bazel_tools//tools/cpp:toolchain_type";

/** Whether blake3 can be used through JNI */
public static final boolean BLAKE3_AVAILABLE = true;

/** A choice of test execution mode, only varies internally. */
public enum InternalTestExecutionMode {
NORMAL
Expand Down

0 comments on commit dc5ec2b

Please sign in to comment.