gradle/actions
1
0
Fork 0
mirror of https://github.com/gradle/actions synced 2024-11-23 18:02:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Refactor: use a single .json file to describe all cached artifact bundles (#121)

Browse source 
This is a pure refactor, moving from a separate .cache file per bundle to a single cache-metadata.json file describing all bundles. Instead of storing cache metadata in a separate .cache file per artifact bundle, all of the metadata is now stored in a single `.json` file.

This will make it easier to implement more flexible artifact-caching strategies, such as caching each wrapper zip separately.

* Always include cache protocol version in cache key
* Store all cache metadata in a single JSON file
* Rename cache-metadata file and bump protocol version
* Polish and documentation
This commit is contained in:
Daz DeBoer 2021-11-28 10:19:56 -07:00 committed by GitHub
parent 92a1f98d35
commit 322805e800
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 199 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dist/main/index.js vendored
View file

File diff suppressed because one or more lines are too long

2
dist/main/index.js.map vendored
View file

File diff suppressed because one or more lines are too long

2
dist/post/index.js vendored
View file

File diff suppressed because one or more lines are too long

2
dist/post/index.js.map vendored
View file

File diff suppressed because one or more lines are too long

90
src/cache-base.ts
View file

@ -3,14 +3,47 @@ import * as cache from '@actions/cache'
import * as github from '@actions/github'
import {isCacheDebuggingEnabled, getCacheKeyPrefix, hashStrings, handleCacheFailure} from './cache-utils'
const CACHE_PROTOCOL_VERSION = 'v5-'
const JOB_CONTEXT_PARAMETER = 'workflow-job-context'
/**
* Represents a key used to restore a cache entry.
* The Github Actions cache will first try for an exact match on the key.
* If that fails, it will try for a prefix match on any of the restoreKeys.
*/
class CacheKey {
key: string
restoreKeys: string[]
constructor(key: string, restoreKeys: string[]) {
this.key = key
this.restoreKeys = restoreKeys
}
}
/**
* Generates a cache key specific to the current job execution.
* The key is constructed from the following inputs:
* - A user-defined prefix (optional)
* - The cache protocol version
* - The name of the cache
* - The runner operating system
* - The name of the Job being executed
* - The matrix values for the Job being executed (job context)
* - The SHA of the commit being executed
*
* Caches are restored by trying to match the these key prefixes in order:
* - The full key with SHA
* - A previous key for this Job + matrix
* - Any previous key for this Job (any matrix)
* - Any previous key for this cache on the current OS
*/
function generateCacheKey(cacheName: string): CacheKey {
const cacheKeyPrefix = getCacheKeyPrefix()
const cacheKeyBase = `${getCacheKeyPrefix()}${CACHE_PROTOCOL_VERSION}${cacheName}`
// At the most general level, share caches for all executions on the same OS
const runnerOs = process.env['RUNNER_OS'] || ''
const cacheKeyForOs = `${cacheKeyPrefix}${cacheName}|${runnerOs}`
const cacheKeyForOs = `${cacheKeyBase}|${runnerOs}`
// Prefer caches that run this job
const cacheKeyForJob = `${cacheKeyForOs}|${github.context.job}`
@ -26,20 +59,15 @@ function generateCacheKey(cacheName: string): CacheKey {
function determineJobContext(): string {
// By default, we hash the full `matrix` data for the run, to uniquely identify this job invocation
// The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml.
const workflowJobContext = core.getInput(JOB_CONTEXT_PARAMETER)
return hashStrings([workflowJobContext])
}
class CacheKey {
key: string
restoreKeys: string[]
constructor(key: string, restoreKeys: string[]) {
this.key = key
this.restoreKeys = restoreKeys
}
}
/**
* Collects information on what entries were saved and restored during the action.
* This information is used to generate a summary of the cache usage.
*/
export class CacheListener {
cacheEntries: CacheEntryListener[] = []
@ -74,6 +102,9 @@ export class CacheListener {
}
}
/**
* Collects information on the state of a single cache entry.
*/
export class CacheEntryListener {
entryName: string
requestedKey: string | undefined
@ -127,15 +158,18 @@ export abstract class AbstractCache {
this.cacheDebuggingEnabled = isCacheDebuggingEnabled()
}
/**
* Restores the cache entry, finding the closest match to the currently running job.
* If the target output already exists, caching will be skipped.
*/
async restore(listener: CacheListener): Promise<void> {
if (this.cacheOutputExists()) {
core.info(`${this.cacheDescription} already exists. Not restoring from cache.`)
return
}
const entryListener = listener.entry(this.cacheDescription)
const cacheKey = this.prepareCacheKey()
const entryReport = listener.entry(this.cacheDescription)
entryReport.markRequested(cacheKey.key, cacheKey.restoreKeys)
this.debug(
`Requesting ${this.cacheDescription} with
@ -144,6 +178,7 @@ export abstract class AbstractCache {
)
const cacheResult = await this.restoreCache(this.getCachePath(), cacheKey.key, cacheKey.restoreKeys)
entryListener.markRequested(cacheKey.key, cacheKey.restoreKeys)
if (!cacheResult) {
core.info(`${this.cacheDescription} cache not found. Will initialize empty.`)
@ -151,7 +186,8 @@ export abstract class AbstractCache {
}
core.saveState(this.cacheResultStateKey, cacheResult.key)
entryReport.markRestored(cacheResult.key, cacheResult.size)
entryListener.markRestored(cacheResult.key, cacheResult.size)
core.info(`Restored ${this.cacheDescription} from cache key: ${cacheResult.key}`)
try {
@ -163,7 +199,6 @@ export abstract class AbstractCache {
prepareCacheKey(): CacheKey {
const cacheKey = generateCacheKey(this.cacheName)
core.saveState(this.cacheKeyStateKey, cacheKey.key)
return cacheKey
}
@ -183,22 +218,31 @@ export abstract class AbstractCache {
protected async afterRestore(_listener: CacheListener): Promise<void> {}
/**
* Saves the cache entry based on the current cache key, unless:
* - If the cache output existed before restore, then it is not saved.
* - If the cache was restored with the exact key, we cannot overwrite it.
*
* If the cache entry was restored with a partial match on a restore key, then
* it is saved with the exact key.
*/
async save(listener: CacheListener): Promise<void> {
if (!this.cacheOutputExists()) {
core.info(`No ${this.cacheDescription} to cache.`)
return
}
const cacheKey = core.getState(this.cacheKeyStateKey)
const cacheResult = core.getState(this.cacheResultStateKey)
// Retrieve the state set in the previous 'restore' step.
const cacheKeyFromRestore = core.getState(this.cacheKeyStateKey)
const cacheResultFromRestore = core.getState(this.cacheResultStateKey)
if (!cacheKey) {
if (!cacheKeyFromRestore) {
core.info(`${this.cacheDescription} existed prior to cache restore. Not saving.`)
return
}
if (cacheResult && cacheKey === cacheResult) {
core.info(`Cache hit occurred on the cache key ${cacheKey}, not saving cache.`)
if (cacheResultFromRestore && cacheKeyFromRestore === cacheResultFromRestore) {
core.info(`Cache hit occurred on the cache key ${cacheKeyFromRestore}, not saving cache.`)
return
}
@ -209,9 +253,9 @@ export abstract class AbstractCache {
return
}
core.info(`Caching ${this.cacheDescription} with cache key: ${cacheKey}`)
core.info(`Caching ${this.cacheDescription} with cache key: ${cacheKeyFromRestore}`)
const cachePath = this.getCachePath()
const savedEntry = await this.saveCache(cachePath, cacheKey)
const savedEntry = await this.saveCache(cachePath, cacheKeyFromRestore)
if (savedEntry) {
listener.entry(this.cacheDescription).markSaved(savedEntry.key, savedEntry.size)

174
src/cache-gradle-user-home.ts
View file

@ -9,11 +9,31 @@ import {AbstractCache, CacheEntryListener, CacheListener} from './cache-base'
import {getCacheKeyPrefix, hashFileNames, tryDelete} from './cache-utils'
const META_FILE_DIR = '.gradle-build-action'
const META_FILE = 'cache-metadata.json'
const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes'
const EXCLUDE_PATHS_PARAMETER = 'gradle-home-cache-excludes'
const ARTIFACT_BUNDLES_PARAMETER = 'gradle-home-cache-artifact-bundles'
/**
* Represents the result of attempting to load or store a cache bundle entry.
* An undefined cacheKey indicates that the operation did not succeed.
* The collected results are then used to populate the `cache-metadata.json` file for later use.
*/
class CacheBundleResult {
readonly bundle: string
readonly cacheKey: string | undefined
constructor(bundle: string, cacheKey: string | undefined) {
this.bundle = bundle
this.cacheKey = cacheKey
}
}
/**
* Caches and restores the entire Gradle User Home directory, extracting bundles of common artifacts
* for more efficient storage.
*/
export class GradleUserHomeCache extends AbstractCache {
private gradleUserHome: string
@ -27,31 +47,35 @@ export class GradleUserHomeCache extends AbstractCache {
initializeGradleUserHome(this.gradleUserHome)
}
/**
* Restore any artifact bundles after the main Gradle User Home entry is restored.
*/
async afterRestore(listener: CacheListener): Promise<void> {
await this.reportGradleUserHomeSize('as restored from cache')
await this.debugReportGradleUserHomeSize('as restored from cache')
await this.restoreArtifactBundles(listener)
await this.reportGradleUserHomeSize('after restoring common artifacts')
await this.debugReportGradleUserHomeSize('after restoring common artifacts')
}
/**
* Restores any artifacts that were cached separately, based on the information in the `cache-metadata.json` file.
* Each artifact bundle is restored in parallel, except when debugging is enabled.
*/
private async restoreArtifactBundles(listener: CacheListener): Promise<void> {
const processes: Promise<void>[] = []
const bundleMetadata = this.loadBundleMetadata()
const bundlePatterns = this.getArtifactBundleDefinitions()
const bundleMetaFiles = await this.getBundleMetaFiles()
const bundlePatterns = this.getArtifactBundles()
const processes: Promise<CacheBundleResult>[] = []
// Iterate over all bundle meta files and try to restore
for (const bundleMetaFile of bundleMetaFiles) {
const bundle = path.basename(bundleMetaFile, '.cache')
for (const [bundle, cacheKey] of bundleMetadata) {
const entryListener = listener.entry(bundle)
const bundlePattern = bundlePatterns.get(bundle)
// Handle case where the 'artifactBundlePatterns' have been changed
if (bundlePattern === undefined) {
core.info(`Found bundle metafile for ${bundle} but no such bundle defined`)
core.info(`Found bundle metadata for ${bundle} but no such bundle defined`)
entryListener.markRequested('BUNDLE_NOT_CONFIGURED')
tryDelete(bundleMetaFile)
} else {
const p = this.restoreArtifactBundle(bundle, bundlePattern, bundleMetaFile, entryListener)
const p = this.restoreArtifactBundle(bundle, cacheKey, bundlePattern, entryListener)
// Run sequentially when debugging enabled
if (this.cacheDebuggingEnabled) {
await p
@ -60,48 +84,45 @@ export class GradleUserHomeCache extends AbstractCache {
}
}
await Promise.all(processes)
const results = await Promise.all(processes)
this.saveMetadataForCacheResults(results)
}
private async restoreArtifactBundle(
bundle: string,
cacheKey: string,
bundlePattern: string,
bundleMetaFile: string,
listener: CacheEntryListener
): Promise<void> {
const cacheKey = fs.readFileSync(bundleMetaFile, 'utf-8').trim()
): Promise<CacheBundleResult> {
listener.markRequested(cacheKey)
const restoredEntry = await this.restoreCache([bundlePattern], cacheKey)
if (restoredEntry) {
core.info(`Restored ${bundle} with key ${cacheKey} to ${bundlePattern}`)
listener.markRestored(restoredEntry.key, restoredEntry.size)
return new CacheBundleResult(bundle, cacheKey)
} else {
core.info(`Did not restore ${bundle} with key ${cacheKey} to ${bundlePattern}`)
tryDelete(bundleMetaFile)
return new CacheBundleResult(bundle, undefined)
}
}
private getBundleMetaFile(name: string): string {
return path.resolve(this.gradleUserHome, META_FILE_DIR, `${name}.cache`)
}
private async getBundleMetaFiles(): Promise<string[]> {
const metaFiles = path.resolve(this.gradleUserHome, META_FILE_DIR, '*.cache')
const globber = await glob.create(metaFiles)
const bundleFiles = await globber.glob()
return bundleFiles
}
/**
* Save and delete any artifact bundles prior to the main Gradle User Home entry being saved.
*/
async beforeSave(listener: CacheListener): Promise<void> {
await this.reportGradleUserHomeSize('before saving common artifacts')
await this.debugReportGradleUserHomeSize('before saving common artifacts')
this.removeExcludedPaths()
await this.saveArtifactBundles(listener)
await this.reportGradleUserHomeSize(
await this.debugReportGradleUserHomeSize(
"after saving common artifacts (only 'caches' and 'notifications' will be stored)"
)
}
/**
* Delete any file paths that are excluded by the `gradle-home-cache-excludes` parameter.
*/
private removeExcludedPaths(): void {
const rawPaths: string[] = core.getMultilineInput(EXCLUDE_PATHS_PARAMETER)
const resolvedPaths = rawPaths.map(x => path.resolve(this.gradleUserHome, x))
@ -112,12 +133,19 @@ export class GradleUserHomeCache extends AbstractCache {
}
}
/**
* Saves any artifacts that are configured to be cached separately, based on the artifact bundle definitions.
* These definitions are normally fixed, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter.
* Each artifact bundle is saved in parallel, except when debugging is enabled.
*/
private async saveArtifactBundles(listener: CacheListener): Promise<void> {
const processes: Promise<void>[] = []
for (const [bundle, pattern] of this.getArtifactBundles()) {
const entryListener = listener.entry(bundle)
const bundleMetadata = this.loadBundleMetadata()
const p = this.saveArtifactBundle(bundle, pattern, entryListener)
const processes: Promise<CacheBundleResult>[] = []
for (const [bundle, pattern] of this.getArtifactBundleDefinitions()) {
const entryListener = listener.entry(bundle)
const previouslyRestoredKey = bundleMetadata.get(bundle)
const p = this.saveArtifactBundle(bundle, pattern, previouslyRestoredKey, entryListener)
// Run sequentially when debugging enabled
if (this.cacheDebuggingEnabled) {
await p
@ -125,16 +153,17 @@ export class GradleUserHomeCache extends AbstractCache {
processes.push(p)
}
await Promise.all(processes)
const results = await Promise.all(processes)
this.saveMetadataForCacheResults(results)
}
private async saveArtifactBundle(
bundle: string,
artifactPath: string,
previouslyRestoredKey: string | undefined,
listener: CacheEntryListener
): Promise<void> {
const bundleMetaFile = this.getBundleMetaFile(bundle)
): Promise<CacheBundleResult> {
const globber = await glob.create(artifactPath, {
implicitDescendants: false,
followSymbolicLinks: false
@ -144,16 +173,10 @@ export class GradleUserHomeCache extends AbstractCache {
// Handle no matching files
if (bundleFiles.length === 0) {
this.debug(`No files found to cache for ${bundle}`)
if (fs.existsSync(bundleMetaFile)) {
tryDelete(bundleMetaFile)
}
return
return new CacheBundleResult(bundle, undefined)
}
const previouslyRestoredKey = fs.existsSync(bundleMetaFile)
? fs.readFileSync(bundleMetaFile, 'utf-8').trim()
: ''
const cacheKey = this.createCacheKey(bundle, bundleFiles)
const cacheKey = this.createCacheKeyForArtifacts(bundle, bundleFiles)
if (previouslyRestoredKey === cacheKey) {
this.debug(`No change to previously restored ${bundle}. Not caching.`)
@ -161,7 +184,6 @@ export class GradleUserHomeCache extends AbstractCache {
core.info(`Caching ${bundle} with cache key: ${cacheKey}`)
const savedEntry = await this.saveCache([artifactPath], cacheKey)
if (savedEntry !== undefined) {
this.writeBundleMetaFile(bundleMetaFile, cacheKey)
listener.markSaved(savedEntry.key, savedEntry.size)
}
}
@ -169,9 +191,11 @@ export class GradleUserHomeCache extends AbstractCache {
for (const file of bundleFiles) {
tryDelete(file)
}
return new CacheBundleResult(bundle, cacheKey)
}
protected createCacheKey(bundle: string, files: string[]): string {
protected createCacheKeyForArtifacts(bundle: string, files: string[]): string {
const cacheKeyPrefix = getCacheKeyPrefix()
const relativeFiles = files.map(x => path.relative(this.gradleUserHome, x))
const key = hashFileNames(relativeFiles)
@ -181,15 +205,39 @@ export class GradleUserHomeCache extends AbstractCache {
return `${cacheKeyPrefix}${bundle}-${key}`
}
private writeBundleMetaFile(metaFile: string, cacheKey: string): void {
this.debug(`Writing bundle metafile: ${metaFile}`)
const dirName = path.dirname(metaFile)
if (!fs.existsSync(dirName)) {
fs.mkdirSync(dirName)
/**
* Load information about the previously restored/saved artifact bundles from the 'cache-metadata.json' file.
*/
private loadBundleMetadata(): Map<string, string> {
const bundleMetaFile = path.resolve(this.gradleUserHome, META_FILE_DIR, META_FILE)
if (!fs.existsSync(bundleMetaFile)) {
return new Map<string, string>()
}
const filedata = fs.readFileSync(bundleMetaFile, 'utf-8')
core.debug(`Loaded bundle metadata: ${filedata}`)
return new Map(JSON.parse(filedata))
}
fs.writeFileSync(metaFile, cacheKey)
/**
* Saves information about the artifact bundle restore/save into the 'cache-metadata.json' file.
*/
private saveMetadataForCacheResults(results: CacheBundleResult[]): void {
const metadata = new Map<string, string>()
for (const result of results) {
if (result.cacheKey !== undefined) {
metadata.set(result.bundle, result.cacheKey)
}
}
const filedata = JSON.stringify(Array.from(metadata))
core.debug(`Saving bundle metadata: ${filedata}`)
const bundleMetaDir = path.resolve(this.gradleUserHome, META_FILE_DIR)
const bundleMetaFile = path.resolve(bundleMetaDir, META_FILE)
if (!fs.existsSync(bundleMetaDir)) {
fs.mkdirSync(bundleMetaDir, {recursive: true})
}
fs.writeFileSync(bundleMetaFile, filedata, 'utf-8')
}
protected determineGradleUserHome(rootDir: string): string {
@ -207,6 +255,11 @@ export class GradleUserHomeCache extends AbstractCache {
return fs.existsSync(dir)
}
/**
* Determines the paths within Gradle User Home to cache.
* By default, this is the 'caches' and 'notifications' directories,
* but this can be overridden by the `gradle-home-cache-includes` parameter.
*/
protected getCachePath(): string[] {
const rawPaths: string[] = core.getMultilineInput(INCLUDE_PATHS_PARAMETER)
rawPaths.push(META_FILE_DIR)
@ -223,14 +276,23 @@ export class GradleUserHomeCache extends AbstractCache {
return path.resolve(this.gradleUserHome, rawPath)
}
private getArtifactBundles(): Map<string, string> {
/**
* Return the artifact bundle definitions, which determine which artifacts will be cached
* separately from the rest of the Gradle User Home cache entry.
* This is normally a fixed set, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter.
*/
private getArtifactBundleDefinitions(): Map<string, string> {
const artifactBundleDefinition = core.getInput(ARTIFACT_BUNDLES_PARAMETER)
this.debug(`Using artifact bundle definition: ${artifactBundleDefinition}`)
const artifactBundles = JSON.parse(artifactBundleDefinition)
return new Map(Array.from(artifactBundles, ([key, value]) => [key, path.resolve(this.gradleUserHome, value)]))
}
private async reportGradleUserHomeSize(label: string): Promise<void> {
/**
* When cache debugging is enabled, this method will give a detailed report
* of the Gradle User Home contents.
*/
private async debugReportGradleUserHomeSize(label: string): Promise<void> {
if (!this.cacheDebuggingEnabled) {
return
}

3
src/cache-project-dot-gradle.ts
View file

@ -7,6 +7,9 @@ const PATHS_TO_CACHE = [
'configuration-cache' // Only configuration-cache is stored at present
]
/**
* A simple cache that saves and restores the '.gradle/configuration-cache' directory in the project root.
*/
export class ProjectDotGradleCache extends AbstractCache {
private rootDir: string
constructor(rootDir: string) {

4
src/cache-utils.ts
View file

@ -4,8 +4,6 @@ import * as crypto from 'crypto'
import * as path from 'path'
import * as fs from 'fs'
const CACHE_PROTOCOL_VERSION = 'v4-'
const CACHE_DISABLED_PARAMETER = 'cache-disabled'
const CACHE_READONLY_PARAMETER = 'cache-read-only'
const CACHE_DEBUG_VAR = 'GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED'
@ -25,7 +23,7 @@ export function isCacheDebuggingEnabled(): boolean {
export function getCacheKeyPrefix(): string {
// Prefix can be used to force change all cache keys (defaults to cache protocol version)
return process.env[CACHE_PREFIX_VAR] || CACHE_PROTOCOL_VERSION
return process.env[CACHE_PREFIX_VAR] || ''
}
export function hashStrings(values: string[]): string {

4
src/main.ts
View file

@ -7,7 +7,9 @@ import * as execution from './execution'
import * as gradlew from './gradlew'
import * as provision from './provision'
// Invoked by GitHub Actions
/**
* The main entry point for the action, called by Github Actions for the step.
*/
export async function run(): Promise<void> {
try {
const workspaceDirectory = process.env[`GITHUB_WORKSPACE`] || ''

4
src/post.ts
View file

@ -6,7 +6,9 @@ import * as caches from './caches'
// throw an uncaught exception. Instead of failing this action, just warn.
process.on('uncaughtException', e => handleFailure(e))
// Invoked by GitHub Actions
/**
* The post-execution entry point for the action, called by Github Actions after completing all steps for the Job.
*/
export async function run(): Promise<void> {
try {
await caches.save()