Include bottle checksums in CI artifacts

v3: wherein everyone stays in their lane

scripts/bottle.ts

@@ -18,32 +18,42 @@ import useCellar from "hooks/useCellar.ts"
 import { run } from "utils"
 import useCache from "hooks/useCache.ts"
 import useFlags from "hooks/useFlags.ts"
+import { crypto } from "deno/crypto/mod.ts"
+import { encodeToString } from "encodeToString"
 
 useFlags()
 
 const cellar = useCellar()
 const filesListName = 'files.txt'
 
-const rv: Path[] = []
+const bottles: Path[] = []
+const fileLists: Path[] = []
 for (const pkg of Deno.args.map(parsePackageRequirement)) {
   console.log({ bottling: { pkg } })
 
   const installation = await cellar.resolve(pkg)
   const path = await bottle(installation)
+  const checksum = await sha256(path)
 
   if (!path) throw new Error("wtf: bottle already exists")
+  if (!checksum) throw new Error("failed to compute checksum")
 
   console.log({ bottled: { path } })
 
-  rv.push(path)
-  rv.push(installation.path.join(filesListName))
+  bottles.push(path)
+  bottles.push(checksum)
+  fileLists.push(installation.path.join(filesListName))
 }
 
-if (rv.length === 0) throw new Error("no input provided")
+if (bottles.length === 0) throw new Error("no input provided")
 
-const paths = rv.map(x => x.string).join('%0A')
-const txt = `::set-output name=filenames::${paths}\n`
-await Deno.stdout.write(new TextEncoder().encode(txt))
+const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
+
+const bottleList = bottles.map(x => x.string).join(" ")
+await Deno.stdout.write(encode(`::set-output name=bottles::${bottleList}\n`))
+
+const paths = [...bottles, ...fileLists].map(x => x.string).join('%0A')
+await Deno.stdout.write(encode(`::set-output name=filenames::${paths}\n`))
 
 
 //------------------------------------------------------------------------- funcs
@@ -107,3 +117,13 @@ export async function walk(root: Path, body: (entry: Path) => Continuation): Pro
 
   return rv
 }
+
+async function sha256(file: Path): Promise<Path> {
+  const file_contents = await Deno.readFile(file.string)
+  const checksum = encodeToString(new Uint8Array(await crypto.subtle.digest("SHA-256", file_contents)))
+  const checksum_contents = new TextEncoder().encode(`${checksum}  ${file.basename()}`)
+  const checksum_file = new Path(`${file.string}.sha256sum`)
+  await Deno.writeFile(checksum_file.string, checksum_contents)
+
+  return checksum_file
+}

scripts/upload.ts

@@ -6,15 +6,15 @@ args:
   - run
   - --allow-net
   - --allow-read=/opt
+  - --allow-write=/opt/tea.xyz/var/www
   - --allow-env
   - --import-map={{ srcroot }}/import-map.json
 ---*/
 
 import { S3 } from "s3"
-import { crypto } from "deno/crypto/mod.ts"
+import { PackageRequirement, Path } from "types"
 import useCache from "hooks/useCache.ts"
 import useCellar from "hooks/useCellar.ts"
-import { encodeToString } from "encodeToString"
 import { Package, parsePackageRequirement, SemVer, semver } from "types"
 import useFlags from "hooks/useFlags.ts"
 
@@ -31,27 +31,56 @@ const s3 = new S3({
 const bucket = s3.getBucket(Deno.env.get("AWS_S3")!)
 const cellar = useCellar()
 
-for (const rq of Deno.args.map(parsePackageRequirement)) {
+const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
+
+const bottles = new Set<PackageRequirement>()
+const checksums = new Set<string>()
+
+for (const filename of Deno.args) {
+  const path = new Path(filename).isFile()
+
+  if (!path) { throw new Error(`${filename} is missing`)}
+
+  if (path.basename() == "files.txt") { continue } // We don't need to upload this
+
+  const match = path.basename().match(/(.*)-([0-9]+\.[0-9]+\.[0-9]+)\+.*\.tar\.gz.*/)
+
+  if (!match)  { throw new Error(`${filename} doesn't appear to be our bottle/checksum`) }
+
+  const req = parsePackageRequirement(`${match[1]}@${match[2]}`)
+
+  if (path.basename().match(/\.sha256sum$/)) { checksums.add(`${req.project}@${req.constraint.raw}`) }
+  else { bottles.add(req) }
+}
+
+// Ensure our sets are the same:
+if (bottles.size !== checksums.size || ![...bottles].every(b => checksums.has(`${b.project}@${b.constraint.raw}`))) {
+  throw new Error("bottles and checksums don't align")
+}
+
+for (const rq of bottles) {
   const {pkg} = await cellar.resolve(rq)
   const key = useCache().s3Key(pkg)
   const bottle = useCache().bottle(pkg)
+  const checksum = new Path(`${bottle.string}.sha256sum`)
 
   console.log({ key });
 
   //FIXME stream it to S3
   const bottle_contents = await Deno.readFile(bottle.string)
-  const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
-  const [basename, dirname] = (split => [split.pop(), split.join("/")])(key.split("/"))
-  const checksum = encodeToString(new Uint8Array(await crypto.subtle.digest("SHA-256", bottle_contents)))
+  const checksum_contents = await Deno.readFile(checksum.string)
+  const dirname = key.split("/").slice(0, -1).join("/")
   const versions = await get_versions(pkg)
 
   await bucket.putObject(key, bottle_contents)
-  await bucket.putObject(`${key}.sha256sum`, encode(`${checksum}  ${basename}`))
+  await bucket.putObject(`${key}.sha256sum`, checksum_contents)
   await bucket.putObject(`${dirname}/versions.txt`, encode(versions.join("\n")))
 
   console.log({ uploaded: key })
 }
 
+//end
+
 import { dirname, basename } from "deno/path/mod.ts"
 
 async function get_versions(pkg: Package): Promise<SemVer[]> {