diff --git a/sources/src/wrapper-validation/checksums.ts b/sources/src/wrapper-validation/checksums.ts index a1835cb..6a04af8 100644 --- a/sources/src/wrapper-validation/checksums.ts +++ b/sources/src/wrapper-validation/checksums.ts @@ -1,4 +1,5 @@ import * as httpm from 'typed-rest-client/HttpClient' +import * as cheerio from 'cheerio' import fileWrapperChecksums from './wrapper-checksums.json' @@ -54,7 +55,15 @@ export async function fetchUnknownChecksums( // eslint-disable-next-line @typescript-eslint/no-explicit-any (entry: any) => entry.wrapperChecksumUrl as string ) - const checksums = await Promise.all(checksumUrls.map(async (url: string) => httpGetText(url))) + console.log(`Fetching checksums for ${checksumUrls.length} versions`) + if (allowSnapshots) { + await addDistributionSnapshotChecksums(checksumUrls) + } + console.log(`Fetching checksums for ${checksumUrls.length} versions after snapshot check`) + const checksums = await Promise.all(checksumUrls.map(async (url: string) => { + // console.log(`Fetching checksum from ${url}`) + return httpGetText(url) + })) return new Set(checksums) } @@ -66,3 +75,22 @@ async function httpGetText(url: string): Promise { const response = await httpc.get(url) return await response.readBody() } + +// Public for testing +export async function addDistributionSnapshotChecksums(checksumUrls: string[]): Promise { + // Load the index page of the distribution snapshot repository + const indexPage = await httpGetText('https://services.gradle.org/distributions-snapshots/') + + // // Extract all wrapper checksum from the index page. These end in -wrapper.jar.sha256 + // // Load the HTML into cheerio + const $ = cheerio.load(indexPage); + + // // Find all links ending with '-wrapper.jar.sha256' + const wrapperChecksumLinks = $('a[href$="-wrapper.jar.sha256"]'); + + // build the absolute URL for each wrapper checksum + wrapperChecksumLinks.each((index, element) => { + const url = $(element).attr('href') + checksumUrls.push(`https://services.gradle.org${url}`) + }) +} diff --git a/sources/test/jest/wrapper-validation/checksums.test.ts b/sources/test/jest/wrapper-validation/checksums.test.ts index 91794c5..b9d87c9 100644 --- a/sources/test/jest/wrapper-validation/checksums.test.ts +++ b/sources/test/jest/wrapper-validation/checksums.test.ts @@ -32,6 +32,22 @@ test('fetches wrapper jars checksums', async () => { ).toBe(true) }) +test('fetches wrapper jar checksums for snapshots', async () => { + const nonSnapshotChecksums = await checksums.fetchUnknownChecksums(false, new checksums.WrapperChecksums) + const validChecksums = await checksums.fetchUnknownChecksums(true, new checksums.WrapperChecksums) + + // Expect that at least one snapshot checksum is different from the non-snapshot checksums + expect(validChecksums.size).toBeGreaterThan(nonSnapshotChecksums.size) +}) + +test('fetches all wrapper checksum URLS for snapshots', async () => { + const checksumUrls: string[] = [] + await checksums.addDistributionSnapshotChecksums(checksumUrls) + + expect(checksumUrls.length).toBeGreaterThan(100) // May only be a few unique checksums + console.log(checksumUrls) +}) + describe('retry', () => { afterEach(() => { nock.cleanAll()