Verify wrappers for distribution-snapshots

By slurping the checksum URLs from https://services.gradle.org/distributions-snapshots/
we can include these unpublished wrapper checksums in validation.

Fixes #281

sources/src/wrapper-validation/checksums.ts

@@ -1,4 +1,5 @@
 import * as httpm from 'typed-rest-client/HttpClient'
+import * as cheerio from 'cheerio'
 
 import fileWrapperChecksums from './wrapper-checksums.json'
 
@@ -54,7 +55,15 @@ export async function fetchUnknownChecksums(
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         (entry: any) => entry.wrapperChecksumUrl as string
     )
-    const checksums = await Promise.all(checksumUrls.map(async (url: string) => httpGetText(url)))
+    console.log(`Fetching checksums for ${checksumUrls.length} versions`)
+    if (allowSnapshots) {
+        await addDistributionSnapshotChecksums(checksumUrls)
+    }
+    console.log(`Fetching checksums for ${checksumUrls.length} versions after snapshot check`)
+    const checksums = await Promise.all(checksumUrls.map(async (url: string) => {
+        // console.log(`Fetching checksum from ${url}`)
+        return httpGetText(url)
+    }))
     return new Set(checksums)
 }
 
@@ -66,3 +75,22 @@ async function httpGetText(url: string): Promise<string> {
     const response = await httpc.get(url)
     return await response.readBody()
 }
+
+// Public for testing
+export async function addDistributionSnapshotChecksums(checksumUrls: string[]): Promise<void> {
+    // Load the index page of the distribution snapshot repository
+    const indexPage = await httpGetText('https://services.gradle.org/distributions-snapshots/')
+
+    // // Extract all wrapper checksum from the index page. These end in -wrapper.jar.sha256
+    // // Load the HTML into cheerio
+    const $ = cheerio.load(indexPage);
+
+    // // Find all links ending with '-wrapper.jar.sha256'
+    const wrapperChecksumLinks = $('a[href$="-wrapper.jar.sha256"]');
+
+    // build the absolute URL for each wrapper checksum
+    wrapperChecksumLinks.each((index, element) => {
+        const url = $(element).attr('href')
+        checksumUrls.push(`https://services.gradle.org${url}`)
+    })
+}

sources/test/jest/wrapper-validation/checksums.test.ts

@@ -32,6 +32,22 @@ test('fetches wrapper jars checksums', async () => {
   ).toBe(true)
 })
 
+test('fetches wrapper jar checksums for snapshots', async () => {
+  const nonSnapshotChecksums = await checksums.fetchUnknownChecksums(false, new checksums.WrapperChecksums)
+  const validChecksums = await checksums.fetchUnknownChecksums(true, new checksums.WrapperChecksums)
+
+  // Expect that at least one snapshot checksum is different from the non-snapshot checksums
+  expect(validChecksums.size).toBeGreaterThan(nonSnapshotChecksums.size)
+})
+
+test('fetches all wrapper checksum URLS for snapshots', async () => {
+  const checksumUrls: string[] = []
+  await checksums.addDistributionSnapshotChecksums(checksumUrls)
+
+  expect(checksumUrls.length).toBeGreaterThan(100) // May only be a few unique checksums
+  console.log(checksumUrls)
+})
+
 describe('retry', () => {
   afterEach(() => {
     nock.cleanAll()