Caching improvements (#294)

- Better reporting of cache status in Job Summary (including cache
cleanup)
- Allow cache cleanup to be skipped when Gradle builds fail
- Refactoring
This commit is contained in:
Daz DeBoer 2024-07-18 11:33:36 -06:00 committed by GitHub
commit 723ca4de01
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 222 additions and 74 deletions

View file

@ -23,6 +23,9 @@ jobs:
- name: Setup Gradle
uses: ./setup-gradle
with:
cache-read-only: false
cache-cleanup: 'on-success'
- name: Build kotlin-dsl project
working-directory: .github/workflow-samples/kotlin-dsl
run: ./gradlew assemble

View file

@ -55,7 +55,7 @@ jobs:
uses: ./setup-gradle
with:
cache-read-only: false
gradle-home-cache-cleanup: true
cache-cleanup: 'on-success'
- name: Build with 3.1.1
working-directory: sources/test/jest/resources/cache-cleanup
run: ./gradlew --no-daemon --build-cache -Dcommons_math3_version="3.1.1" build

View file

@ -57,6 +57,20 @@ inputs:
Configuration-cache data will not be saved/restored without an encryption key being provided.
required: false
cache-cleanup:
description: |
Specifies if the action should attempt to remove any stale/unused entries from the Gradle User Home prior to saving to the GitHub Actions cache.
By default, no cleanup is performed. It can be configured to run every time, or only when all Gradle builds succeed for the Job.
Valid values are 'never', 'on-success' and 'always'.
required: false
default: 'never'
gradle-home-cache-cleanup:
description: When 'true', the action will attempt to remove any stale/unused entries from the Gradle User Home prior to saving to the GitHub Actions cache.
required: false
default: false
deprecation-message: This input has been superceded by the 'cache-cleanup' input parameter.
gradle-home-cache-includes:
description: Paths within Gradle User Home to cache.
required: false
@ -68,11 +82,6 @@ inputs:
description: Paths within Gradle User Home to exclude from cache.
required: false
gradle-home-cache-cleanup:
description: When 'true', the action will attempt to remove any stale/unused entries from the Gradle User Home prior to saving to the GitHub Actions cache.
required: false
default: false
# Job summary configuration
add-job-summary:
description: Specifies when a Job Summary should be inluded in the action results. Valid values are 'never', 'always' (default), and 'on-failure'.

View file

@ -153,6 +153,23 @@ In certain circumstances it may be desirable to start with a clean Gradle User H
cache-write-only: true
```
### Enabling cache cleanup
The Gradle User Home directory tends to grow over time. When you switch to a new Gradle wrapper version or upgrade a dependency version
the old files are not automatically and immediately removed. While this can make sense in a local environment, in a GitHub Actions environment
it can lead to ever-larger Gradle User Home cache entries being saved and restored.
To avoid this situation, The `setup-gradle` action supports the `cache-cleanup` parameter.
When cache-cleanup is enabled, this feature will attempt to delete any files in the Gradle User Home that were not used by Gradle during the GitHub Actions Workflow, before saving the Gradle User Home to the GitHub Actions cache.
If cache cleanup runs after a failing Gradle build, it is possible that some required files and dependencies will not be touched, and will be removed.
To prevent this scenario, cache cleanup can be configured to run only when all Gradle builds in the Job are successful.
Gradle Home cache cleanup is considered experimental and is disabled by default. You can enable this feature for the action as follows:
```yaml
cache-cleanup: 'on-success' # Valid values are 'never' (default), 'on-success' and 'always'
```
### Overwriting an existing Gradle User Home
When the action detects that the Gradle User Home caches directory already exists (`~/.gradle/caches`), then by default it will not overwrite the existing content of this directory.
@ -362,13 +379,7 @@ The Gradle User Home directory tends to grow over time. When you switch to a new
the old files are not automatically and immediately removed. While this can make sense in a local environment, in a GitHub Actions environment
it can lead to ever-larger Gradle User Home cache entries being saved and restored.
To avoid this situation, The `setup-gradle` action supports the `gradle-home-cache-cleanup` parameter.
When enabled, this feature will attempt to delete any files in the Gradle User Home that were not used by Gradle during the GitHub Actions Workflow, before saving the Gradle User Home to the GitHub Actions cache.
Gradle Home cache cleanup is considered experimental and is disabled by default. You can enable this feature for the action as follows:
```yaml
gradle-home-cache-cleanup: true
```
See [Enabling cache cleanup](#enabling-cache-cleanup) for a mechanism to mitigate this problem.
### Disable local build-cache when remote build-cache is available

View file

@ -40,6 +40,20 @@ inputs:
Configuration-cache data will not be saved/restored without an encryption key being provided.
required: false
cache-cleanup:
description: |
Specifies if the action should attempt to remove any stale/unused entries from the Gradle User Home prior to saving to the GitHub Actions cache.
By default, no cleanup is performed. It can be configured to run every time, or only when all Gradle builds succeed for the Job.
Valid values are 'never', 'on-success' and 'always'.
required: false
default: 'never'
gradle-home-cache-cleanup:
description: When 'true', the action will attempt to remove any stale/unused entries from the Gradle User Home prior to saving to the GitHub Actions cache.
required: false
default: false
deprecation-message: This input has been superceded by the 'cache-cleanup' input parameter.
gradle-home-cache-includes:
description: Paths within Gradle User Home to cache.
required: false
@ -51,11 +65,6 @@ inputs:
description: Paths within Gradle User Home to exclude from cache.
required: false
gradle-home-cache-cleanup:
description: When 'true', the action will attempt to remove any stale/unused entries from the Gradle User Home prior to saving to the GitHub Actions cache.
required: false
default: false
# Job summary configuration
add-job-summary:
description: Specifies when a Job Summary should be inluded in the action results. Valid values are 'never', 'always' (default), and 'on-failure'.

View file

@ -8,11 +8,11 @@
"prettier-write": "prettier --write 'src/**/*.ts'",
"prettier-check": "prettier --check 'src/**/*.ts'",
"lint": "eslint 'src/**/*.ts'",
"compile-dependency-submission-main": "ncc build src/dependency-submission/main.ts --out dist/dependency-submission/main --source-map --no-source-map-register",
"compile-dependency-submission-post": "ncc build src/dependency-submission/post.ts --out dist/dependency-submission/post --source-map --no-source-map-register",
"compile-setup-gradle-main": "ncc build src/setup-gradle/main.ts --out dist/setup-gradle/main --source-map --no-source-map-register",
"compile-setup-gradle-post": "ncc build src/setup-gradle/post.ts --out dist/setup-gradle/post --source-map --no-source-map-register",
"compile-wrapper-validation-main": "ncc build src/wrapper-validation/main.ts --out dist/wrapper-validation/main --source-map --no-source-map-register",
"compile-dependency-submission-main": "ncc build src/actions/dependency-submission/main.ts --out dist/dependency-submission/main --source-map --no-source-map-register",
"compile-dependency-submission-post": "ncc build src/actions/dependency-submission/post.ts --out dist/dependency-submission/post --source-map --no-source-map-register",
"compile-setup-gradle-main": "ncc build src/actions/setup-gradle/main.ts --out dist/setup-gradle/main --source-map --no-source-map-register",
"compile-setup-gradle-post": "ncc build src/actions/setup-gradle/post.ts --out dist/setup-gradle/post --source-map --no-source-map-register",
"compile-wrapper-validation-main": "ncc build src/actions/wrapper-validation/main.ts --out dist/wrapper-validation/main --source-map --no-source-map-register",
"compile": "npm-run-all --parallel compile-*",
"check": "npm-run-all --parallel prettier-check lint",
"format": "npm-run-all --parallel prettier-write lint",

View file

@ -1,7 +1,7 @@
import * as core from '@actions/core'
import * as setupGradle from '../setup-gradle'
import * as gradle from '../execution/gradle'
import * as dependencyGraph from '../dependency-graph'
import * as setupGradle from '../../setup-gradle'
import * as gradle from '../../execution/gradle'
import * as dependencyGraph from '../../dependency-graph'
import {parseArgsStringToArgv} from 'string-argv'
import {
@ -11,9 +11,9 @@ import {
DependencyGraphOption,
GradleExecutionConfig,
setActionId
} from '../configuration'
import {saveDeprecationState} from '../deprecation-collector'
import {handleMainActionError} from '../errors'
} from '../../configuration'
import {saveDeprecationState} from '../../deprecation-collector'
import {handleMainActionError} from '../../errors'
/**
* The main entry point for the action, called by Github Actions for the step.

View file

@ -1,7 +1,7 @@
import * as setupGradle from '../setup-gradle'
import * as setupGradle from '../../setup-gradle'
import {CacheConfig, SummaryConfig} from '../configuration'
import {handlePostActionError} from '../errors'
import {CacheConfig, SummaryConfig} from '../../configuration'
import {handlePostActionError} from '../../errors'
// Catch and log any unhandled exceptions. These exceptions can leak out of the uploadChunk method in
// @actions/toolkit when a failed upload closes the file descriptor causing any in-process reads to

View file

@ -1,6 +1,6 @@
import * as setupGradle from '../setup-gradle'
import * as gradle from '../execution/gradle'
import * as dependencyGraph from '../dependency-graph'
import * as setupGradle from '../../setup-gradle'
import * as gradle from '../../execution/gradle'
import * as dependencyGraph from '../../dependency-graph'
import {
BuildScanConfig,
CacheConfig,
@ -9,9 +9,9 @@ import {
doValidateWrappers,
getActionId,
setActionId
} from '../configuration'
import {recordDeprecation, saveDeprecationState} from '../deprecation-collector'
import {handleMainActionError} from '../errors'
} from '../../configuration'
import {recordDeprecation, saveDeprecationState} from '../../deprecation-collector'
import {handleMainActionError} from '../../errors'
/**
* The main entry point for the action, called by Github Actions for the step.

View file

@ -1,9 +1,9 @@
import * as setupGradle from '../setup-gradle'
import * as dependencyGraph from '../dependency-graph'
import * as setupGradle from '../../setup-gradle'
import * as dependencyGraph from '../../dependency-graph'
import {CacheConfig, DependencyGraphConfig, SummaryConfig} from '../configuration'
import {handlePostActionError} from '../errors'
import {emitDeprecationWarnings, restoreDeprecationState} from '../deprecation-collector'
import {CacheConfig, DependencyGraphConfig, SummaryConfig} from '../../configuration'
import {handlePostActionError} from '../../errors'
import {emitDeprecationWarnings, restoreDeprecationState} from '../../deprecation-collector'
// Catch and log any unhandled exceptions. These exceptions can leak out of the uploadChunk method in
// @actions/toolkit when a failed upload closes the file descriptor causing any in-process reads to

View file

@ -1,10 +1,10 @@
import * as path from 'path'
import * as core from '@actions/core'
import * as validate from './validate'
import {getActionId, setActionId} from '../configuration'
import {recordDeprecation, emitDeprecationWarnings} from '../deprecation-collector'
import {handleMainActionError} from '../errors'
import * as validate from '../../wrapper-validation/validate'
import {getActionId, setActionId} from '../../configuration'
import {recordDeprecation, emitDeprecationWarnings} from '../../deprecation-collector'
import {handleMainActionError} from '../../errors'
export async function run(): Promise<void> {
try {

View file

@ -12,11 +12,29 @@ export interface BuildResult {
get buildScanFailed(): boolean
}
export function loadBuildResults(): BuildResult[] {
return getUnprocessedResults().map(filePath => {
export class BuildResults {
results: BuildResult[]
constructor(results: BuildResult[]) {
this.results = results
}
anyFailed(): boolean {
return this.results.some(result => result.buildFailed)
}
uniqueGradleHomes(): string[] {
const allHomes = this.results.map(buildResult => buildResult.gradleHomeDir)
return Array.from(new Set(allHomes))
}
}
export function loadBuildResults(): BuildResults {
const results = getUnprocessedResults().map(filePath => {
const content = fs.readFileSync(filePath, 'utf8')
return JSON.parse(content) as BuildResult
})
return new BuildResults(results)
}
export function markBuildResultsProcessed(): void {

View file

@ -1,5 +1,24 @@
import * as cache from '@actions/cache'
export const DEFAULT_CACHE_ENABLED_REASON = `[Cache was enabled](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#caching-build-state-between-jobs). Action attempted to both restore and save the Gradle User Home.`
export const DEFAULT_READONLY_REASON = `[Cache was read-only](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#using-the-cache-read-only). By default, the action will only write to the cache for Jobs running on the default branch.`
export const DEFAULT_DISABLED_REASON = `[Cache was disabled](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#disabling-caching) via action confiugration. Gradle User Home was not restored from or saved to the cache.`
export const DEFAULT_WRITEONLY_REASON = `[Cache was set to write-only](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#using-the-cache-write-only) via action configuration. Gradle User Home was not restored from cache.`
export const EXISTING_GRADLE_HOME = `[Cache was disabled to avoid overwriting a pre-existing Gradle User Home](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#overwriting-an-existing-gradle-user-home). Gradle User Home was not restored from or saved to the cache.`
export const CLEANUP_DISABLED_READONLY = `[Cache cleanup](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#enabling-cache-cleanup) is always disabled when cache is read-only or disabled.`
export const DEFAULT_CLEANUP_DISABLED_REASON = `[Cache cleanup](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#enabling-cache-cleanup) was not enabled. It must be explicitly enabled.`
export const DEFAULT_CLEANUP_ENABLED_REASON = `[Cache cleanup](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#enabling-cache-cleanup) was enabled.`
export const CLEANUP_DISABLED_DUE_TO_FAILURE =
'[Cache cleanup was disabled due to build failure](https://github.com/gradle/actions/blob/v3/docs/setup-gradle.md#enabling-cache-cleanup). Use `cache-cleanup: always` to override this behavior.'
/**
* Collects information on what entries were saved and restored during the action.
* This information is used to generate a summary of the cache usage.
@ -9,7 +28,8 @@ export class CacheListener {
cacheReadOnly = false
cacheWriteOnly = false
cacheDisabled = false
cacheDisabledReason = 'disabled'
cacheStatusReason: string = DEFAULT_CACHE_ENABLED_REASON
cacheCleanupMessage: string = DEFAULT_CLEANUP_DISABLED_REASON
get fullyRestored(): boolean {
return this.cacheEntries.every(x => !x.wasRequestedButNotRestored())
@ -17,12 +37,37 @@ export class CacheListener {
get cacheStatus(): string {
if (!cache.isFeatureAvailable()) return 'not available'
if (this.cacheDisabled) return this.cacheDisabledReason
if (this.cacheDisabled) return 'disabled'
if (this.cacheWriteOnly) return 'write-only'
if (this.cacheReadOnly) return 'read-only'
return 'enabled'
}
setReadOnly(reason: string = DEFAULT_READONLY_REASON): void {
this.cacheReadOnly = true
this.cacheStatusReason = reason
this.cacheCleanupMessage = CLEANUP_DISABLED_READONLY
}
setDisabled(reason: string = DEFAULT_DISABLED_REASON): void {
this.cacheDisabled = true
this.cacheStatusReason = reason
this.cacheCleanupMessage = CLEANUP_DISABLED_READONLY
}
setWriteOnly(reason: string = DEFAULT_WRITEONLY_REASON): void {
this.cacheWriteOnly = true
this.cacheStatusReason = reason
}
setCacheCleanupEnabled(): void {
this.cacheCleanupMessage = DEFAULT_CLEANUP_ENABLED_REASON
}
setCacheCleanupDisabled(reason: string = DEFAULT_CLEANUP_DISABLED_REASON): void {
this.cacheCleanupMessage = reason
}
entry(name: string): CacheEntryListener {
for (const entry of this.cacheEntries) {
if (entry.entryName === name) {
@ -117,6 +162,10 @@ export function generateCachingReport(listener: CacheListener): string {
return `
<details>
<summary><h4>Caching for Gradle actions was ${listener.cacheStatus} - expand for details</h4></summary>
- ${listener.cacheStatusReason}
- ${listener.cacheCleanupMessage}
${renderEntryTable(entries)}
<h5>Cache Entry Details</h5>

View file

@ -1,9 +1,10 @@
import * as core from '@actions/core'
import {CacheListener} from './cache-reporting'
import {CacheListener, EXISTING_GRADLE_HOME, CLEANUP_DISABLED_DUE_TO_FAILURE} from './cache-reporting'
import {GradleUserHomeCache} from './gradle-user-home-cache'
import {CacheCleaner} from './cache-cleaner'
import {DaemonController} from '../daemon-controller'
import {CacheConfig} from '../configuration'
import {BuildResults} from '../build-results'
const CACHE_RESTORED_VAR = 'GRADLE_BUILD_ACTION_CACHE_RESTORED'
@ -26,7 +27,7 @@ export async function restore(
core.info('Cache is disabled: will not restore state from previous builds.')
// Initialize the Gradle User Home even when caching is disabled.
gradleStateCache.init()
cacheListener.cacheDisabled = true
cacheListener.setDisabled()
return
}
@ -35,8 +36,7 @@ export async function restore(
core.info('Gradle User Home already exists: will not restore from cache.')
// Initialize pre-existing Gradle User Home.
gradleStateCache.init()
cacheListener.cacheDisabled = true
cacheListener.cacheDisabledReason = 'disabled due to pre-existing Gradle User Home'
cacheListener.setDisabled(EXISTING_GRADLE_HOME)
return
}
core.info('Gradle User Home already exists: will overwrite with cached contents.')
@ -48,7 +48,7 @@ export async function restore(
if (cacheConfig.isCacheWriteOnly()) {
core.info('Cache is write-only: will not restore from cache.')
cacheListener.cacheWriteOnly = true
cacheListener.setWriteOnly()
return
}
@ -68,6 +68,7 @@ export async function save(
gradleUserHome: string,
cacheListener: CacheListener,
daemonController: DaemonController,
buildResults: BuildResults,
cacheConfig: CacheConfig
): Promise<void> {
if (cacheConfig.isCacheDisabled()) {
@ -82,19 +83,19 @@ export async function save(
if (cacheConfig.isCacheReadOnly()) {
core.info('Cache is read-only: will not save state for use in subsequent builds.')
cacheListener.cacheReadOnly = true
cacheListener.setReadOnly()
return
}
await daemonController.stopAllDaemons()
if (cacheConfig.isCacheCleanupEnabled()) {
core.info('Forcing cache cleanup.')
const cacheCleaner = new CacheCleaner(gradleUserHome, process.env['RUNNER_TEMP']!)
try {
await cacheCleaner.forceCleanup()
} catch (e) {
core.warning(`Cache cleanup failed. Will continue. ${String(e)}`)
if (cacheConfig.shouldPerformCacheCleanup(buildResults.anyFailed())) {
cacheListener.setCacheCleanupEnabled()
await performCacheCleanup(gradleUserHome)
} else {
core.info('Not performing cache-cleanup due to build failure')
cacheListener.setCacheCleanupDisabled(CLEANUP_DISABLED_DUE_TO_FAILURE)
}
}
@ -102,3 +103,13 @@ export async function save(
return new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig).save(cacheListener)
})
}
async function performCacheCleanup(gradleUserHome: string): Promise<void> {
core.info('Forcing cache cleanup.')
const cacheCleaner = new CacheCleaner(gradleUserHome, process.env['RUNNER_TEMP']!)
try {
await cacheCleaner.forceCleanup()
} catch (e) {
core.warning(`Cache cleanup failed. Will continue. ${String(e)}`)
}
}

View file

@ -449,7 +449,7 @@ export class ConfigurationCacheEntryExtractor extends AbstractEntryExtractor {
}
private getConfigCacheDirectoriesWithAssociatedBuildResults(): Record<string, BuildResult[]> {
return loadBuildResults().reduce(
return loadBuildResults().results.reduce(
(acc, buildResult) => {
// For each build result, find the config-cache dir
const configCachePath = path.resolve(buildResult.rootProjectDir, '.gradle/configuration-cache')

View file

@ -111,7 +111,40 @@ export class CacheConfig {
}
isCacheCleanupEnabled(): boolean {
return getBooleanInput('gradle-home-cache-cleanup') && !this.isCacheReadOnly()
if (this.isCacheReadOnly()) {
return false
}
const cleanupOption = this.getCacheCleanupOption()
return cleanupOption === CacheCleanupOption.Always || cleanupOption === CacheCleanupOption.OnSuccess
}
shouldPerformCacheCleanup(hasFailure: boolean): boolean {
const cleanupOption = this.getCacheCleanupOption()
if (cleanupOption === CacheCleanupOption.Always) {
return true
}
if (cleanupOption === CacheCleanupOption.OnSuccess) {
return !hasFailure
}
return false
}
private getCacheCleanupOption(): CacheCleanupOption {
const val = core.getInput('cache-cleanup')
switch (val.toLowerCase().trim()) {
case 'always':
return CacheCleanupOption.Always
case 'on-success':
return CacheCleanupOption.OnSuccess
case 'never':
// When set to 'never' (the default), honour the legacy parameter setting.
return getBooleanInput('gradle-home-cache-cleanup')
? CacheCleanupOption.Always
: CacheCleanupOption.Never
}
throw TypeError(
`The value '${val}' is not valid for cache-cleanup. Valid values are: [never, always, on-success].`
)
}
getCacheEncryptionKey(): string {
@ -127,6 +160,12 @@ export class CacheConfig {
}
}
export enum CacheCleanupOption {
Never = 'never',
OnSuccess = 'on-success',
Always = 'always'
}
export class SummaryConfig {
shouldGenerateJobSummary(hasFailure: boolean): boolean {
// Check if Job Summary is supported on this platform

View file

@ -2,14 +2,13 @@ import * as core from '@actions/core'
import * as exec from '@actions/exec'
import * as fs from 'fs'
import * as path from 'path'
import {BuildResult} from './build-results'
import {BuildResults} from './build-results'
export class DaemonController {
private readonly gradleHomes
constructor(buildResults: BuildResult[]) {
const allHomes = buildResults.map(buildResult => buildResult.gradleHomeDir)
this.gradleHomes = Array.from(new Set(allHomes))
constructor(buildResults: BuildResults) {
this.gradleHomes = buildResults.uniqueGradleHomes()
}
async stopAllDaemons(): Promise<void> {

View file

@ -2,18 +2,18 @@ import * as core from '@actions/core'
import * as github from '@actions/github'
import {RequestError} from '@octokit/request-error'
import {BuildResult} from './build-results'
import {BuildResults, BuildResult} from './build-results'
import {SummaryConfig, getActionId, getGithubToken} from './configuration'
import {Deprecation, getDeprecations} from './deprecation-collector'
export async function generateJobSummary(
buildResults: BuildResult[],
buildResults: BuildResults,
cachingReport: string,
config: SummaryConfig
): Promise<void> {
const summaryTable = renderSummaryTable(buildResults)
const summaryTable = renderSummaryTable(buildResults.results)
const hasFailure = buildResults.some(result => result.buildFailed)
const hasFailure = buildResults.anyFailed()
if (config.shouldGenerateJobSummary(hasFailure)) {
core.info('Generating Job Summary')

View file

@ -60,7 +60,7 @@ export async function complete(cacheConfig: CacheConfig, summaryConfig: SummaryC
const cacheListener: CacheListener = CacheListener.rehydrate(core.getState(CACHE_LISTENER))
const daemonController = new DaemonController(buildResults)
await caches.save(userHome, gradleUserHome, cacheListener, daemonController, cacheConfig)
await caches.save(userHome, gradleUserHome, cacheListener, daemonController, buildResults, cacheConfig)
const cachingReport = generateCachingReport(cacheListener)
await jobSummary.generateJobSummary(buildResults, cachingReport, summaryConfig)