From b6395da67c9887f995671dbf1729f9f9a5c413db Mon Sep 17 00:00:00 2001 From: daz Date: Thu, 1 Aug 2024 09:39:30 -0600 Subject: [PATCH] Cache validated checksums for later executions The most common case for validation will be that the wrapper jars are unchanged from a previous workflow run. In this case, we cache the validated wrapper checksums to minimise the work required on a subsequent run. Fixes #172 --- .../caching/gradle-home-extry-extractor.ts | 5 +-- sources/src/caching/gradle-user-home-cache.ts | 8 ++-- sources/src/configuration.ts | 2 + sources/src/setup-gradle.ts | 4 +- sources/src/wrapper-validation/cache.ts | 26 +++++++++++++ sources/src/wrapper-validation/validate.ts | 7 +++- .../wrapper-validation/wrapper-validator.ts | 22 ++++++++++- .../jest/wrapper-validation/validate.test.ts | 37 +++++++++++++++++++ 8 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 sources/src/wrapper-validation/cache.ts diff --git a/sources/src/caching/gradle-home-extry-extractor.ts b/sources/src/caching/gradle-home-extry-extractor.ts index 2f4ecee..35a1f48 100644 --- a/sources/src/caching/gradle-home-extry-extractor.ts +++ b/sources/src/caching/gradle-home-extry-extractor.ts @@ -4,12 +4,11 @@ import * as core from '@actions/core' import * as glob from '@actions/glob' import * as semver from 'semver' -import {META_FILE_DIR} from './gradle-user-home-cache' import {CacheEntryListener, CacheListener} from './cache-reporting' import {cacheDebug, hashFileNames, isCacheDebuggingEnabled, restoreCache, saveCache, tryDelete} from './cache-utils' import {BuildResult, loadBuildResults} from '../build-results' -import {CacheConfig} from '../configuration' +import {CacheConfig, ACTION_METADATA_DIR} from '../configuration' import {getCacheKeyBase} from './cache-key' const SKIP_RESTORE_VAR = 'GRADLE_BUILD_ACTION_SKIP_RESTORE' @@ -298,7 +297,7 @@ abstract class AbstractEntryExtractor { } private getCacheMetadataFile(): string { - const actionMetadataDirectory = path.resolve(this.gradleUserHome, META_FILE_DIR) + const actionMetadataDirectory = path.resolve(this.gradleUserHome, ACTION_METADATA_DIR) fs.mkdirSync(actionMetadataDirectory, {recursive: true}) return path.resolve(actionMetadataDirectory, `${this.extractorName}-entry-metadata.json`) diff --git a/sources/src/caching/gradle-user-home-cache.ts b/sources/src/caching/gradle-user-home-cache.ts index 04f4e08..5d86d84 100644 --- a/sources/src/caching/gradle-user-home-cache.ts +++ b/sources/src/caching/gradle-user-home-cache.ts @@ -7,14 +7,12 @@ import fs from 'fs' import {generateCacheKey} from './cache-key' import {CacheListener} from './cache-reporting' import {saveCache, restoreCache, cacheDebug, isCacheDebuggingEnabled, tryDelete} from './cache-utils' -import {CacheConfig} from '../configuration' +import {CacheConfig, ACTION_METADATA_DIR} from '../configuration' import {GradleHomeEntryExtractor, ConfigurationCacheEntryExtractor} from './gradle-home-extry-extractor' import {getPredefinedToolchains, mergeToolchainContent, readResourceFileAsString} from './gradle-user-home-utils' const RESTORED_CACHE_KEY_KEY = 'restored-cache-key' -export const META_FILE_DIR = '.setup-gradle' - export class GradleUserHomeCache { private readonly cacheName = 'home' private readonly cacheDescription = 'Gradle User Home' @@ -172,7 +170,7 @@ export class GradleUserHomeCache { */ protected getCachePath(): string[] { const rawPaths: string[] = this.cacheConfig.getCacheIncludes() - rawPaths.push(META_FILE_DIR) + rawPaths.push(ACTION_METADATA_DIR) const resolvedPaths = rawPaths.map(x => this.resolveCachePath(x)) cacheDebug(`Using cache paths: ${resolvedPaths}`) return resolvedPaths @@ -188,7 +186,7 @@ export class GradleUserHomeCache { private initializeGradleUserHome(): void { // Create a directory for storing action metadata - const actionCacheDir = path.resolve(this.gradleUserHome, META_FILE_DIR) + const actionCacheDir = path.resolve(this.gradleUserHome, ACTION_METADATA_DIR) fs.mkdirSync(actionCacheDir, {recursive: true}) this.copyInitScripts() diff --git a/sources/src/configuration.ts b/sources/src/configuration.ts index a354a2e..9bbefe3 100644 --- a/sources/src/configuration.ts +++ b/sources/src/configuration.ts @@ -8,6 +8,8 @@ import path from 'path' const ACTION_ID_VAR = 'GRADLE_ACTION_ID' +export const ACTION_METADATA_DIR = '.setup-gradle' + export class DependencyGraphConfig { getDependencyGraphOption(): DependencyGraphOption { const val = core.getInput('dependency-graph') diff --git a/sources/src/setup-gradle.ts b/sources/src/setup-gradle.ts index 023d644..8f8353f 100644 --- a/sources/src/setup-gradle.ts +++ b/sources/src/setup-gradle.ts @@ -46,13 +46,13 @@ export async function setup( core.saveState(USER_HOME, userHome) core.saveState(GRADLE_USER_HOME, gradleUserHome) - await wrapperValidator.validateWrappers(wrapperValidationConfig, getWorkspaceDirectory()) - const cacheListener = new CacheListener() await caches.restore(userHome, gradleUserHome, cacheListener, cacheConfig) core.saveState(CACHE_LISTENER, cacheListener.stringify()) + await wrapperValidator.validateWrappers(wrapperValidationConfig, getWorkspaceDirectory(), gradleUserHome) + await buildScan.setup(buildScanConfig) return true diff --git a/sources/src/wrapper-validation/cache.ts b/sources/src/wrapper-validation/cache.ts new file mode 100644 index 0000000..90aba54 --- /dev/null +++ b/sources/src/wrapper-validation/cache.ts @@ -0,0 +1,26 @@ +import fs from 'fs' +import path from 'path' +import {ACTION_METADATA_DIR} from '../configuration' + +export class ChecksumCache { + private readonly cacheFile: string + + constructor(gradleUserHome: string) { + this.cacheFile = path.resolve(gradleUserHome, ACTION_METADATA_DIR, 'valid-wrappers.json') + } + + load(): string[] { + // Load previously validated checksums saved in Gradle User Home + if (fs.existsSync(this.cacheFile)) { + return JSON.parse(fs.readFileSync(this.cacheFile, 'utf-8')) + } + return [] + } + + save(checksums: string[]): void { + const uniqueChecksums = [...new Set(checksums)] + // Save validated checksums to Gradle User Home + fs.mkdirSync(path.dirname(this.cacheFile), {recursive: true}) + fs.writeFileSync(this.cacheFile, JSON.stringify(uniqueChecksums)) + } +} diff --git a/sources/src/wrapper-validation/validate.ts b/sources/src/wrapper-validation/validate.ts index 13be7f6..e831462 100644 --- a/sources/src/wrapper-validation/validate.ts +++ b/sources/src/wrapper-validation/validate.ts @@ -8,6 +8,7 @@ export async function findInvalidWrapperJars( minWrapperCount: number, allowSnapshots: boolean, allowedChecksums: string[], + previouslyValidatedChecksums: string[] = [], knownValidChecksums: checksums.WrapperChecksums = checksums.KNOWN_CHECKSUMS ): Promise { const wrapperJars = await find.findWrapperJars(gitRepoRoot) @@ -21,7 +22,11 @@ export async function findInvalidWrapperJars( const notYetValidatedWrappers = [] for (const wrapperJar of wrapperJars) { const sha = await hash.sha256File(resolve(gitRepoRoot, wrapperJar)) - if (allowedChecksums.includes(sha) || knownValidChecksums.checksums.has(sha)) { + if ( + allowedChecksums.includes(sha) || + previouslyValidatedChecksums.includes(sha) || + knownValidChecksums.checksums.has(sha) + ) { result.valid.push(new WrapperJar(wrapperJar, sha)) } else { notYetValidatedWrappers.push(new WrapperJar(wrapperJar, sha)) diff --git a/sources/src/wrapper-validation/wrapper-validator.ts b/sources/src/wrapper-validation/wrapper-validator.ts index 440f7a1..3bc4992 100644 --- a/sources/src/wrapper-validation/wrapper-validator.ts +++ b/sources/src/wrapper-validation/wrapper-validator.ts @@ -1,17 +1,33 @@ import * as core from '@actions/core' + import {WrapperValidationConfig} from '../configuration' +import {ChecksumCache} from './cache' import {findInvalidWrapperJars} from './validate' import {JobFailure} from '../errors' -export async function validateWrappers(config: WrapperValidationConfig, workspaceRoot: string): Promise { +export async function validateWrappers( + config: WrapperValidationConfig, + workspaceRoot: string, + gradleUserHome: string +): Promise { if (!config.doValidateWrappers()) { return // Wrapper validation is disabled } + const checksumCache = new ChecksumCache(gradleUserHome) const allowedChecksums = process.env['ALLOWED_GRADLE_WRAPPER_CHECKSUMS']?.split(',') || [] - const result = await findInvalidWrapperJars(workspaceRoot, 0, config.allowSnapshotWrappers(), allowedChecksums) + const previouslyValidatedChecksums = checksumCache.load() + + const result = await findInvalidWrapperJars( + workspaceRoot, + 0, + config.allowSnapshotWrappers(), + allowedChecksums, + previouslyValidatedChecksums + ) if (result.isValid()) { await core.group('All Gradle Wrapper jars are valid', async () => { + core.info(`Loaded previously validated checksums from cache: ${previouslyValidatedChecksums.join(', ')}`) core.info(result.toDisplayString()) }) } else { @@ -20,4 +36,6 @@ export async function validateWrappers(config: WrapperValidationConfig, workspac `Gradle Wrapper Validation Failed!\n See https://github.com/gradle/actions/blob/main/docs/wrapper-validation.md#reporting-failures\n${result.toDisplayString()}` ) } + + checksumCache.save(result.valid.map(wrapper => wrapper.checksum)) } diff --git a/sources/test/jest/wrapper-validation/validate.test.ts b/sources/test/jest/wrapper-validation/validate.test.ts index 42c0d87..c4b86ce 100644 --- a/sources/test/jest/wrapper-validation/validate.test.ts +++ b/sources/test/jest/wrapper-validation/validate.test.ts @@ -1,11 +1,15 @@ import * as path from 'path' +import * as fs from 'fs' import * as validate from '../../../src/wrapper-validation/validate' import {expect, test, jest} from '@jest/globals' import { WrapperChecksums } from '../../../src/wrapper-validation/checksums' +import { ChecksumCache } from '../../../src/wrapper-validation/cache' +import exp from 'constants' jest.setTimeout(30000) const baseDir = path.resolve('./test/jest/wrapper-validation') +const tmpDir = path.resolve('./test/jest/tmp') test('succeeds if all found wrapper jars are valid', async () => { const result = await validate.findInvalidWrapperJars(baseDir, 3, false, [ @@ -24,6 +28,24 @@ test('succeeds if all found wrapper jars are valid', async () => { ) }) +test('succeeds if all found wrapper jars are previously valid', async () => { + const result = await validate.findInvalidWrapperJars(baseDir, 3, false, [], [ + 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + '3888c76faa032ea8394b8a54e04ce2227ab1f4be64f65d450f8509fe112d38ce' + ]) + + expect(result.isValid()).toBe(true) + // Only hardcoded and explicitly allowed checksums should have been used + expect(result.fetchedChecksums).toBe(false) + + expect(result.toDisplayString()).toBe( + '✓ Found known Gradle Wrapper JAR files:\n' + + ' e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 data/invalid/gradle-wrapper.jar\n' + + ' e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 data/invalid/gradlе-wrapper.jar\n' + // homoglyph + ' 3888c76faa032ea8394b8a54e04ce2227ab1f4be64f65d450f8509fe112d38ce data/valid/gradle-wrapper.jar' + ) +}) + test('succeeds if all found wrapper jars are valid (and checksums are fetched from Gradle API)', async () => { const knownValidChecksums = new WrapperChecksums() const result = await validate.findInvalidWrapperJars( @@ -31,6 +53,7 @@ test('succeeds if all found wrapper jars are valid (and checksums are fetched fr 1, false, ['e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'], + [], knownValidChecksums ) console.log(`fetchedChecksums = ${result.fetchedChecksums}`) @@ -98,3 +121,17 @@ test('fails if not enough wrapper jars are found', async () => { ' 3888c76faa032ea8394b8a54e04ce2227ab1f4be64f65d450f8509fe112d38ce data/valid/gradle-wrapper.jar' ) }) + +test('can save and load checksums', async () => { + const cacheDir = path.join(tmpDir, 'wrapper-validation-cache') + fs.rmSync(cacheDir, {recursive: true, force: true}) + + const checksumCache = new ChecksumCache(cacheDir) + + expect(checksumCache.load()).toEqual([]) + + checksumCache.save(['123', '456']) + + expect(checksumCache.load()).toEqual(['123', '456']) + expect(fs.existsSync(cacheDir)).toBe(true) +})