Skip to content

Commit

Permalink
generate a single caching hash (#60)
Browse files Browse the repository at this point in the history
* generate a single caching hash

* add backwards compatibility

* print out key info

* add cache step test setup

* add cache step test setup

* add tests for caching step

* build source files

* improve messaging

* commit changed source files

* add comment
  • Loading branch information
rarmatei authored and github-actions[bot] committed Aug 2, 2024
1 parent fccf737 commit 4503d86
Show file tree
Hide file tree
Showing 12 changed files with 46,144 additions and 70 deletions.
6 changes: 5 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@
"lint-staged": "^15.2.0",
"nx": "18.3.4",
"prettier": "^3.1.0",
"typescript": "5.4.5"
"typescript": "5.4.5",
"@types/jest": "^29.4.0",
"jest": "^29.4.1",
"jest-environment-node": "^29.4.1",
"ts-jest": "^29.1.0"
},
"lint-staged": {
"*.{ts,js,json,md}": [
Expand Down
25 changes: 16 additions & 9 deletions workflow-steps/cache/hashing-utils.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import * as string_decoder from 'string_decoder';

const fs = require('fs');
const crypto = require('crypto');
import { glob } from 'glob';
Expand All @@ -11,14 +13,8 @@ function hashFileContents(pattern: string) {
let megaHash = '';
files.forEach((file) => {
const fileContent = fs.readFileSync(file);
const fileHash = crypto
.createHash('sha256')
.update(fileContent)
.digest('hex');
megaHash = crypto
.createHash('sha256')
.update(fileHash + megaHash)
.digest('hex');
const fileHash = hash(fileContent);
megaHash = hash(fileHash + megaHash);
});
return megaHash;
}
Expand All @@ -40,5 +36,16 @@ export function hashKey(key: string): string {
const globHashes = globsToHash.map((globPattern) => {
return hashFileContents(globPattern);
});
return [...hardcodedKeys, ...globHashes].join(' | ');
const hashCollections = [...hardcodedKeys, ...globHashes];

// we are only doing this for backwards compatibility purposes, so we don't bust everyone's cache when it gets merged in
// otherwise, it would've been fine to hash another hash
if (hashCollections.length > 1) {
return hash(hashCollections.join(' | '));
}
return hashCollections.join(' | ');
}

function hash(input: string) {
return crypto.createHash('sha256').update(input).digest('hex');
}
31 changes: 31 additions & 0 deletions workflow-steps/cache/hasing-utils.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import { hashKey } from './hashing-utils';
import * as path from 'path';

console.log(hashKey);
describe('hashing-utils', () => {
const testDir = path.join(__dirname, 'test-files');
it('should hash a single file', () => {
expect(hashKey(`${testDir}/yarn.lock`)).toEqual(
'6ef0d64a2ac614adc8dac86db67244e77cdad3253a65fb8e2b7c11ed4cbb466a',
);
});

it('should hash multiple files', () => {
expect(hashKey(`${testDir}/yarn.lock | ${testDir}/main.js`)).toEqual(
'e5c39d066ffa2cd0d38165a018aaac3118b1c81d8f4631335e0f19cda6fa8e65',
);
});

it('should hash simple strings', () => {
expect(
hashKey(`${testDir}/yarn.lock | ${testDir}/main.js | "test1"`),
).toEqual(
'16b5f107c209ad4d76dffc803beba37b7e836ca2ca229731c8bc88040874a003',
);
expect(
hashKey(`${testDir}/yarn.lock | ${testDir}/main.js | "test2"`),
).toEqual(
'226f813c92638665c8daa0920cfb83e5f33732f8843042deee348032a1abee40',
);
});
});
5 changes: 5 additions & 0 deletions workflow-steps/cache/jest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
/* eslint-disable */
export default {
displayName: 'cache-step',
preset: 'ts-jest',
};
4 changes: 2 additions & 2 deletions workflow-steps/cache/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ cacheClient
)
.then((resp: RestoreResponse) => {
if (resp.success) {
console.log('Found cache entry under key: ' + resp.key);
console.log('Found cache entry on hashed key: ' + resp.key);
rememberCacheRestorationForPostStep();
} else {
console.log('Cache miss.');
console.log('Cache miss on hashed key: ' + key);
}
});

Expand Down
25 changes: 16 additions & 9 deletions workflow-steps/cache/output/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -380,9 +380,9 @@ var require_balanced_match = __commonJS({
}
});

// ../../node_modules/minimatch/node_modules/brace-expansion/index.js
// ../../node_modules/glob/node_modules/brace-expansion/index.js
var require_brace_expansion = __commonJS({
"../../node_modules/minimatch/node_modules/brace-expansion/index.js"(exports, module2) {
"../../node_modules/glob/node_modules/brace-expansion/index.js"(exports, module2) {
var concatMap = require_concat_map();
var balanced = require_balanced_match();
module2.exports = expandTop;
Expand Down Expand Up @@ -525,9 +525,9 @@ var require_brace_expansion = __commonJS({
}
});

// ../../node_modules/minimatch/minimatch.js
// ../../node_modules/glob/node_modules/minimatch/minimatch.js
var require_minimatch = __commonJS({
"../../node_modules/minimatch/minimatch.js"(exports, module2) {
"../../node_modules/glob/node_modules/minimatch/minimatch.js"(exports, module2) {
module2.exports = minimatch;
minimatch.Minimatch = Minimatch;
var path = function() {
Expand Down Expand Up @@ -5959,8 +5959,8 @@ function hashFileContents(pattern) {
let megaHash = "";
files.forEach((file) => {
const fileContent = fs.readFileSync(file);
const fileHash = crypto.createHash("sha256").update(fileContent).digest("hex");
megaHash = crypto.createHash("sha256").update(fileHash + megaHash).digest("hex");
const fileHash = hash(fileContent);
megaHash = hash(fileHash + megaHash);
});
return megaHash;
}
Expand All @@ -5978,7 +5978,14 @@ function hashKey(key2) {
const globHashes = globsToHash.map((globPattern) => {
return hashFileContents(globPattern);
});
return [...hardcodedKeys, ...globHashes].join(" | ");
const hashCollections = [...hardcodedKeys, ...globHashes];
if (hashCollections.length > 1) {
return hash(hashCollections.join(" | "));
}
return hashCollections.join(" | ");
}
function hash(input) {
return crypto.createHash("sha256").update(input).digest("hex");
}

// main.ts
Expand All @@ -6005,10 +6012,10 @@ cacheClient.restore(
})
).then((resp) => {
if (resp.success) {
console.log("Found cache entry under key: " + resp.key);
console.log("Found cache entry on hashed key: " + resp.key);
rememberCacheRestorationForPostStep();
} else {
console.log("Cache miss.");
console.log("Cache miss on hashed key: " + key);
}
});
function rememberCacheRestorationForPostStep() {
Expand Down
21 changes: 14 additions & 7 deletions workflow-steps/cache/output/post.js
Original file line number Diff line number Diff line change
Expand Up @@ -375,9 +375,9 @@ var require_balanced_match = __commonJS({
}
});

// ../../node_modules/minimatch/node_modules/brace-expansion/index.js
// ../../node_modules/glob/node_modules/brace-expansion/index.js
var require_brace_expansion = __commonJS({
"../../node_modules/minimatch/node_modules/brace-expansion/index.js"(exports, module2) {
"../../node_modules/glob/node_modules/brace-expansion/index.js"(exports, module2) {
var concatMap = require_concat_map();
var balanced = require_balanced_match();
module2.exports = expandTop;
Expand Down Expand Up @@ -520,9 +520,9 @@ var require_brace_expansion = __commonJS({
}
});

// ../../node_modules/minimatch/minimatch.js
// ../../node_modules/glob/node_modules/minimatch/minimatch.js
var require_minimatch = __commonJS({
"../../node_modules/minimatch/minimatch.js"(exports, module2) {
"../../node_modules/glob/node_modules/minimatch/minimatch.js"(exports, module2) {
module2.exports = minimatch;
minimatch.Minimatch = Minimatch;
var path = function() {
Expand Down Expand Up @@ -5947,8 +5947,8 @@ function hashFileContents(pattern) {
let megaHash = "";
files.forEach((file) => {
const fileContent = fs.readFileSync(file);
const fileHash = crypto.createHash("sha256").update(fileContent).digest("hex");
megaHash = crypto.createHash("sha256").update(fileHash + megaHash).digest("hex");
const fileHash = hash(fileContent);
megaHash = hash(fileHash + megaHash);
});
return megaHash;
}
Expand All @@ -5966,7 +5966,14 @@ function hashKey(key) {
const globHashes = globsToHash.map((globPattern) => {
return hashFileContents(globPattern);
});
return [...hardcodedKeys, ...globHashes].join(" | ");
const hashCollections = [...hardcodedKeys, ...globHashes];
if (hashCollections.length > 1) {
return hash(hashCollections.join(" | "));
}
return hashCollections.join(" | ");
}
function hash(input) {
return crypto.createHash("sha256").update(input).digest("hex");
}

// post.ts
Expand Down
17 changes: 9 additions & 8 deletions workflow-steps/cache/package.json
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
{
"name": "cache-step",
"version": "0.0.0",
"main": "output/main.js",
"devDependencies": {},
"scripts": {
"build": "npx esbuild main.ts --bundle --platform=node --target=node20 --outfile=output/main.js && npx esbuild post.ts --bundle --platform=node --target=node20 --outfile=output/post.js"
}
}
"name": "cache-step",
"version": "0.0.0",
"main": "output/main.js",
"devDependencies": {},
"scripts": {
"build": "npx esbuild main.ts --bundle --platform=node --target=node20 --outfile=output/main.js && npx esbuild post.ts --bundle --platform=node --target=node20 --outfile=output/post.js",
"test": "npx jest --cwd=workflow-steps/cache"
}
}
55 changes: 55 additions & 0 deletions workflow-steps/cache/test-files/main.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
const { execSync } = require('child_process');
const { existsSync, readFileSync, writeFileSync } = require('fs');

const command = getInstallCommand();
if (command) {
console.log(`Installing dependencies using ${command.split(' ')[0]}`);
console.log(` Running command: ${command}\n`);
execSync(command, { stdio: 'inherit' });
patchJest();
} else {
throw new Error(
'Could not find lock file. Please ensure you have a lock file before running this command.',
);
}

function getInstallCommand() {
if (existsSync('package-lock.json')) {
return 'npm ci --legacy-peer-deps';
} else if (existsSync('yarn.lock')) {
const [major] = execSync(`yarn --version`, {
encoding: 'utf-8',
})
.trim()
.split('.');

const useBerry = +major >= 2;
if (useBerry) {
return 'yarn install --immutable';
} else {
return 'yarn install --frozen-lockfile';
}
} else if (existsSync('pnpm-lock.yaml') || existsSync('pnpm-lock.yml')) {
return 'pnpm install --frozen-lockfile';
}
}

function patchJest() {
try {
const path =
'node_modules/jest-config/build/readConfigFileAndSetRootDir.js';
const contents = readFileSync(path, 'utf-8');
writeFileSync(
path,
contents.replace(
"const tsNode = await import('ts-node');",
"require('ts-node'); const tsNode = await import('ts-node');",
),
);
} catch (e) {
if (process.env.NX_VERBOSE_LOGGING == 'true') {
console.log(e);
}
console.log('no need to patch jest');
}
}
Loading

0 comments on commit 4503d86

Please sign in to comment.