Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: use cache storeV2 #58

Merged
merged 5 commits into from
Jul 25, 2024
Merged

feat: use cache storeV2 #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5838852
fix: do not include branch name when saving cache
Cammisuli Jun 28, 2024
715f119
use cache.storev2
Cammisuli Jul 3, 2024
39d902b
add logging
Cammisuli Jul 3, 2024
0d2416c
add logging
Cammisuli Jul 3, 2024
0b62b53
remove logging
Cammisuli Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions workflow-steps/cache/generated_protos/cache_connect.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,14 @@ export const CacheService = {
O: StoreResponse,
kind: MethodKind.Unary,
},
/**
* @generated from rpc nrwl.grpc.CacheService.StoreV2
*/
storeV2: {
name: 'StoreV2',
I: StoreRequest,
O: StoreResponse,
kind: MethodKind.Unary,
},
},
} as const;
34 changes: 17 additions & 17 deletions workflow-steps/cache/generated_protos/cache_pb.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// @generated by protoc-gen-es v1.3.1 with parameter "target=ts"
// @generated by protoc-gen-es v1.7.2 with parameter "target=ts"
// @generated from file cache.proto (package nrwl.grpc, syntax proto3)
/* eslint-disable */
// @ts-nocheck
Expand Down Expand Up @@ -53,28 +53,28 @@ export class RestoreRequest extends Message<RestoreRequest> {

static fromBinary(
bytes: Uint8Array,
options?: Partial<BinaryReadOptions>
options?: Partial<BinaryReadOptions>,
): RestoreRequest {
return new RestoreRequest().fromBinary(bytes, options);
}

static fromJson(
jsonValue: JsonValue,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): RestoreRequest {
return new RestoreRequest().fromJson(jsonValue, options);
}

static fromJsonString(
jsonString: string,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): RestoreRequest {
return new RestoreRequest().fromJsonString(jsonString, options);
}

static equals(
a: RestoreRequest | PlainMessage<RestoreRequest> | undefined,
b: RestoreRequest | PlainMessage<RestoreRequest> | undefined
b: RestoreRequest | PlainMessage<RestoreRequest> | undefined,
): boolean {
return proto3.util.equals(RestoreRequest, a, b);
}
Expand Down Expand Up @@ -108,28 +108,28 @@ export class RestoreResponse extends Message<RestoreResponse> {

static fromBinary(
bytes: Uint8Array,
options?: Partial<BinaryReadOptions>
options?: Partial<BinaryReadOptions>,
): RestoreResponse {
return new RestoreResponse().fromBinary(bytes, options);
}

static fromJson(
jsonValue: JsonValue,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): RestoreResponse {
return new RestoreResponse().fromJson(jsonValue, options);
}

static fromJsonString(
jsonString: string,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): RestoreResponse {
return new RestoreResponse().fromJsonString(jsonString, options);
}

static equals(
a: RestoreResponse | PlainMessage<RestoreResponse> | undefined,
b: RestoreResponse | PlainMessage<RestoreResponse> | undefined
b: RestoreResponse | PlainMessage<RestoreResponse> | undefined,
): boolean {
return proto3.util.equals(RestoreResponse, a, b);
}
Expand Down Expand Up @@ -169,28 +169,28 @@ export class StoreRequest extends Message<StoreRequest> {

static fromBinary(
bytes: Uint8Array,
options?: Partial<BinaryReadOptions>
options?: Partial<BinaryReadOptions>,
): StoreRequest {
return new StoreRequest().fromBinary(bytes, options);
}

static fromJson(
jsonValue: JsonValue,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): StoreRequest {
return new StoreRequest().fromJson(jsonValue, options);
}

static fromJsonString(
jsonString: string,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): StoreRequest {
return new StoreRequest().fromJsonString(jsonString, options);
}

static equals(
a: StoreRequest | PlainMessage<StoreRequest> | undefined,
b: StoreRequest | PlainMessage<StoreRequest> | undefined
b: StoreRequest | PlainMessage<StoreRequest> | undefined,
): boolean {
return proto3.util.equals(StoreRequest, a, b);
}
Expand Down Expand Up @@ -224,28 +224,28 @@ export class StoreResponse extends Message<StoreResponse> {

static fromBinary(
bytes: Uint8Array,
options?: Partial<BinaryReadOptions>
options?: Partial<BinaryReadOptions>,
): StoreResponse {
return new StoreResponse().fromBinary(bytes, options);
}

static fromJson(
jsonValue: JsonValue,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): StoreResponse {
return new StoreResponse().fromJson(jsonValue, options);
}

static fromJsonString(
jsonString: string,
options?: Partial<JsonReadOptions>
options?: Partial<JsonReadOptions>,
): StoreResponse {
return new StoreResponse().fromJsonString(jsonString, options);
}

static equals(
a: StoreResponse | PlainMessage<StoreResponse> | undefined,
b: StoreResponse | PlainMessage<StoreResponse> | undefined
b: StoreResponse | PlainMessage<StoreResponse> | undefined,
): boolean {
return proto3.util.equals(StoreResponse, a, b);
}
Expand Down
62 changes: 31 additions & 31 deletions workflow-steps/cache/hashing-utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,42 +3,42 @@ const crypto = require('crypto');
import { glob } from 'glob';

function hashFileContents(pattern: string) {
const files = glob
.sync(pattern, { ignore: 'node_modules/**' })
.filter((path) => {
return fs.statSync(path).isFile();
});
let megaHash = '';
files.forEach((file) => {
const fileContent = fs.readFileSync(file);
const fileHash = crypto
.createHash('sha256')
.update(fileContent)
.digest('hex');
megaHash = crypto
.createHash('sha256')
.update(fileHash + megaHash)
.digest('hex');
const files = glob
.sync(pattern, { ignore: 'node_modules/**' })
.filter((path) => {
return fs.statSync(path).isFile();
});
return megaHash;
let megaHash = '';
files.forEach((file) => {
const fileContent = fs.readFileSync(file);
const fileHash = crypto
.createHash('sha256')
.update(fileContent)
.digest('hex');
megaHash = crypto
.createHash('sha256')
.update(fileHash + megaHash)
.digest('hex');
});
return megaHash;
}

export function hashKey(key: string): string {
const keyParts = key.split('|').map((s) => s.trim());
const keyParts = key.split('|').map((s) => s.trim());

const hardcodedKeys: string[] = [];
const globsToHash: string[] = [];
const hardcodedKeys: string[] = [];
const globsToHash: string[] = [];

keyParts.forEach((key) => {
if (key.startsWith('"') && key.endsWith('"')) {
hardcodedKeys.push(key.slice(1, -1));
} else {
globsToHash.push(key);
}
});
keyParts.forEach((key) => {
if (key.startsWith('"') && key.endsWith('"')) {
hardcodedKeys.push(key.slice(1, -1));
} else {
globsToHash.push(key);
}
});

const globHashes = globsToHash.map((globPattern) => {
return hashFileContents(globPattern);
});
return [...hardcodedKeys, ...globHashes].join(' | ');
const globHashes = globsToHash.map((globPattern) => {
return hashFileContents(globPattern);
});
return [...hardcodedKeys, ...globHashes].join(' | ');
}
9 changes: 9 additions & 0 deletions workflow-steps/cache/output/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5935,6 +5935,15 @@ var CacheService = {
I: StoreRequest,
O: StoreResponse,
kind: MethodKind.Unary
},
/**
* @generated from rpc nrwl.grpc.CacheService.StoreV2
*/
storeV2: {
name: "StoreV2",
I: StoreRequest,
O: StoreResponse,
kind: MethodKind.Unary
}
}
};
Expand Down
14 changes: 11 additions & 3 deletions workflow-steps/cache/output/post.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5923,6 +5923,15 @@ var CacheService = {
I: StoreRequest,
O: StoreResponse,
kind: MethodKind.Unary
},
/**
* @generated from rpc nrwl.grpc.CacheService.StoreV2
*/
storeV2: {
name: "StoreV2",
I: StoreRequest,
O: StoreResponse,
kind: MethodKind.Unary
}
}
};
Expand Down Expand Up @@ -5973,15 +5982,14 @@ if (!!cacheWasHit) {
baseUrl: "http://127.0.0.1:9000"
})
);
const currentBranch = process.env.NX_BRANCH;
if (!input_key || !input_paths) {
throw new Error("No cache restore key or paths provided.");
}
const key = hashKey(input_key);
const paths = input_paths.split("\n").filter((p) => p);
cacheClient.store(
cacheClient.storeV2(
new StoreRequest({
key: `${currentBranch}-${key}`,
key,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Cammisuli Is key correct? I'm seeing store v2 keeps failing to find cache from base-branch.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My bad. I should have set base_branch.

paths
})
).then((r) => {
Expand Down
5 changes: 2 additions & 3 deletions workflow-steps/cache/post.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,16 @@ if (!!cacheWasHit) {
}),
);

const currentBranch = process.env.NX_BRANCH;
if (!input_key || !input_paths) {
throw new Error('No cache restore key or paths provided.');
}
const key = hashKey(input_key);
const paths = input_paths.split('\n').filter((p) => p);

cacheClient
.store(
.storeV2(
new StoreRequest({
key: `${currentBranch}-${key}`,
key,
paths,
}),
)
Expand Down