From 83709b49fea4302afa8a90883e8fad756bbf51db Mon Sep 17 00:00:00 2001 From: Daz DeBoer Date: Thu, 14 Nov 2024 15:44:07 -0700 Subject: [PATCH] Fix checksum test by reducing network calls (#444) This test was originally starting with an empty set of checksums, leading to the download of a checksum for every released and snapshot version. This resulted in in sporadic test failures. We now start with a known set of checksums and ensure that those that are missing are downloaded. This involved some refactoring and improvement in the way snapshot checksums are processed. --- sources/src/wrapper-validation/checksums.ts | 42 +++++++-------- sources/src/wrapper-validation/validate.ts | 2 +- .../jest/wrapper-validation/checksums.test.ts | 54 ++++++++++--------- 3 files changed, 52 insertions(+), 46 deletions(-) diff --git a/sources/src/wrapper-validation/checksums.ts b/sources/src/wrapper-validation/checksums.ts index 2748508..dd51011 100644 --- a/sources/src/wrapper-validation/checksums.ts +++ b/sources/src/wrapper-validation/checksums.ts @@ -38,7 +38,7 @@ export const KNOWN_CHECKSUMS = loadKnownChecksums() export async function fetchUnknownChecksums( allowSnapshots: boolean, knownChecksums: WrapperChecksums -): Promise> { +): Promise { const all = await httpGetJsonArray('https://services.gradle.org/versions/all') const withChecksum = all.filter( entry => typeof entry === 'object' && entry != null && entry.hasOwnProperty('wrapperChecksumUrl') @@ -51,20 +51,21 @@ export async function fetchUnknownChecksums( // eslint-disable-next-line @typescript-eslint/no-explicit-any (entry: any) => !knownChecksums.versions.has(entry.version) ) - const checksumUrls = notKnown.map( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (entry: any) => entry.wrapperChecksumUrl as string - ) + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const checksumUrls = notKnown.map((entry: any) => [entry.version, entry.wrapperChecksumUrl] as [string, string]) if (allowSnapshots) { - await addDistributionSnapshotChecksums(checksumUrls) + await addDistributionSnapshotChecksumUrls(checksumUrls) } - const checksums = await Promise.all( - checksumUrls.map(async (url: string) => { - // console.log(`Fetching checksum from ${url}`) - return httpGetText(url) + + const wrapperChecksums = new WrapperChecksums() + await Promise.all( + checksumUrls.map(async ([version, url]) => { + const checksum = await httpGetText(url) + wrapperChecksums.add(version, checksum) }) ) - return new Set(checksums) + return wrapperChecksums } async function httpGetJsonArray(url: string): Promise { @@ -76,21 +77,20 @@ async function httpGetText(url: string): Promise { return await response.readBody() } -// Public for testing -export async function addDistributionSnapshotChecksums(checksumUrls: string[]): Promise { - // Load the index page of the distribution snapshot repository +async function addDistributionSnapshotChecksumUrls(checksumUrls: [string, string][]): Promise { + // Load the index page of the distribution snapshot repository into cheerio const indexPage = await httpGetText('https://services.gradle.org/distributions-snapshots/') - - // // Extract all wrapper checksum from the index page. These end in -wrapper.jar.sha256 - // // Load the HTML into cheerio const $ = cheerio.load(indexPage) // // Find all links ending with '-wrapper.jar.sha256' const wrapperChecksumLinks = $('a[href$="-wrapper.jar.sha256"]') - - // build the absolute URL for each wrapper checksum wrapperChecksumLinks.each((index, element) => { - const url = $(element).attr('href') - checksumUrls.push(`https://services.gradle.org${url}`) + const url = $(element).attr('href')! + + // Extract the version from the url + const version = url.match(/\/distributions-snapshots\/gradle-(.*?)-wrapper\.jar\.sha256/)?.[1] + if (version) { + checksumUrls.push([version, `https://services.gradle.org${url}`]) + } }) } diff --git a/sources/src/wrapper-validation/validate.ts b/sources/src/wrapper-validation/validate.ts index 85543f4..2fc5e1f 100644 --- a/sources/src/wrapper-validation/validate.ts +++ b/sources/src/wrapper-validation/validate.ts @@ -33,7 +33,7 @@ export async function findInvalidWrapperJars( const fetchedValidChecksums = await checksums.fetchUnknownChecksums(allowSnapshots, knownValidChecksums) for (const wrapperJar of notYetValidatedWrappers) { - if (!fetchedValidChecksums.has(wrapperJar.checksum)) { + if (!fetchedValidChecksums.checksums.has(wrapperJar.checksum)) { result.invalid.push(wrapperJar) } else { result.valid.push(wrapperJar) diff --git a/sources/test/jest/wrapper-validation/checksums.test.ts b/sources/test/jest/wrapper-validation/checksums.test.ts index 62f2478..299f554 100644 --- a/sources/test/jest/wrapper-validation/checksums.test.ts +++ b/sources/test/jest/wrapper-validation/checksums.test.ts @@ -4,6 +4,22 @@ import {afterEach, describe, expect, test, jest} from '@jest/globals' jest.setTimeout(30000) +const CHECKSUM_8_1 = 'ed2c26eba7cfb93cc2b7785d05e534f07b5b48b5e7fc941921cd098628abca58' + +function knownChecksumsWithout8_1(): checksums.WrapperChecksums { + const knownChecksums = new checksums.WrapperChecksums() + // iterate over all known checksums and add them to the knownChecksums object + for (const [checksum, versions] of checksums.KNOWN_CHECKSUMS.checksums) { + if (checksum !== CHECKSUM_8_1) { + for (const version of versions) { + knownChecksums.add(version, checksum) + } + } + } + return knownChecksums +} + + test('has loaded hardcoded wrapper jars checksums', async () => { // Sanity check that generated checksums file is not empty and was properly imported expect(checksums.KNOWN_CHECKSUMS.checksums.size).toBeGreaterThan(10) @@ -20,33 +36,23 @@ test('has loaded hardcoded wrapper jars checksums', async () => { ).toEqual(new Set(['6.0-rc-1', '6.0-rc-2', '6.0-rc-3', '6.0', '6.0.1'])) }) -test('fetches wrapper jars checksums', async () => { - const validChecksums = await checksums.fetchUnknownChecksums(false, new checksums.WrapperChecksums) - expect(validChecksums.size).toBeGreaterThan(10) - // Verify that checksum of arbitrary version is contained - expect( - validChecksums.has( - // Checksum for version 6.0 - '28b330c20a9a73881dfe9702df78d4d78bf72368e8906c70080ab6932462fe9e' - ) - ).toBe(true) +test('fetches wrapper jar checksums that are missing from hardcoded set', async () => { + const unknownChecksums = await checksums.fetchUnknownChecksums(false, knownChecksumsWithout8_1()) + + expect(unknownChecksums.checksums.size).toEqual(1) + expect(unknownChecksums.checksums.has(CHECKSUM_8_1)).toBe(true) + expect(unknownChecksums.checksums.get(CHECKSUM_8_1)).toEqual(new Set(['8.1-rc-1', '8.1-rc-2', '8.1-rc-3', '8.1-rc-4', '8.1', '8.1.1'])) }) test('fetches wrapper jar checksums for snapshots', async () => { - const nonSnapshotChecksums = await checksums.fetchUnknownChecksums(false, new checksums.WrapperChecksums) - const validChecksums = await checksums.fetchUnknownChecksums(true, new checksums.WrapperChecksums) + const knownChecksums = knownChecksumsWithout8_1() + const nonSnapshotChecksums = await checksums.fetchUnknownChecksums(false, knownChecksums) + const allValidChecksums = await checksums.fetchUnknownChecksums(true, knownChecksums) - // Expect that at least one snapshot checksum is different from the non-snapshot checksums - expect(nonSnapshotChecksums.size).toBeGreaterThan(10) - expect(validChecksums.size).toBeGreaterThanOrEqual(nonSnapshotChecksums.size) -}) - -test('fetches all wrapper checksum URLS for snapshots', async () => { - const checksumUrls: string[] = [] - await checksums.addDistributionSnapshotChecksums(checksumUrls) - - expect(checksumUrls.length).toBeGreaterThan(100) // May only be a few unique checksums - console.log(checksumUrls) + // Should always be many more snapshot versions + expect(allValidChecksums.versions.size - nonSnapshotChecksums.versions.size).toBeGreaterThan(20) + // May not be any unique snapshot checksums + expect(allValidChecksums.checksums.size).toBeGreaterThanOrEqual(nonSnapshotChecksums.checksums.size) }) describe('retry', () => { @@ -65,7 +71,7 @@ describe('retry', () => { }) const validChecksums = await checksums.fetchUnknownChecksums(false, new checksums.WrapperChecksums) - expect(validChecksums.size).toBeGreaterThan(10) + expect(validChecksums.checksums.size).toBeGreaterThan(10) nock.isDone() }) })