fix making tar.xz bottles (#158)

scripts/bottle.ts

@@ -21,6 +21,7 @@ import { useCellar, usePrefix, useFlags, useCache } from "hooks"
 import { run, pkg as pkgutils } from "utils"
 import { crypto } from "deno/crypto/mod.ts"
 import { encode } from "deno/encoding/hex.ts"
+import { set_output } from "./utils/gha.ts"
 import Path from "path"
 
 const cellar = useCellar()
@@ -32,36 +33,24 @@ if (import.meta.main) {
   useFlags()
 
   const compression = Deno.env.get("COMPRESSION") == 'xz' ? 'xz' : 'gz'
-
+  const checksums: string[] = []
   const bottles: Path[] = []
-  const checksums: Path[] = []
-  const artifacts: Path[] = []
+
   for (const pkg of Deno.args.map(pkgutils.parse)) {
-    console.log({ bottling: { pkg } })
+    console.log({ bottling: pkg })
 
     const installation = await cellar.resolve(pkg)
     const path = await bottle(installation, compression)
     const checksum = await sha256(path)
 
-    console.log({ bottled: { path } })
+    console.log({ bottled: path })
 
     bottles.push(path)
     checksums.push(checksum)
   }
 
-  if (bottles.length === 0) throw new Error("no input provided")
-
-  const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
-
-  const bottles_out = bottles.map(x => x.string).join(' ')
-  await Deno.stdout.write(encode(`::set-output name=bottles::${bottles_out}\n`))
-
-  const checksums_out = checksums.map(x => x.string).join(' ')
-  await Deno.stdout.write(encode(`::set-output name=checksums::${checksums_out}\n`))
-
-  // newline separated for the upload-artifact action
-  artifacts.push(...bottles, ...checksums)
-  await Deno.stdout.write(encode(`::set-output name=artifacts::${artifacts.join('%0A')}\n`))
+  await set_output("bottles", bottles)
+  await set_output("checksums", checksums)
 }
 
 
@@ -75,10 +64,8 @@ export async function bottle({ path: kegdir, pkg }: Installation, compression: '
   return tarball
 }
 
-async function sha256(file: Path): Promise<Path> {
-  const sha = await Deno.open(file.string, { read: true })
+async function sha256(file: Path): Promise<string> {
+  return await Deno.open(file.string, { read: true })
     .then(file => crypto.subtle.digest("SHA-256", file.readable))
     .then(buf => new TextDecoder().decode(encode(new Uint8Array(buf))))
-  const text = `${sha}  ${file.basename()}`
-  return new Path(`${file}.sha256sum`).write({ text, force: true })
 }

scripts/build.ts

@@ -13,10 +13,11 @@ args:
 ---*/
 
 import { usePantry } from "hooks"
-import build from "./build/build.ts"
-import { Package } from "types"
+import { Installation, Package } from "types"
 import { pkg as pkgutils } from "utils"
 import { useFlags, usePrefix } from "hooks"
+import { set_output } from "./utils/gha.ts"
+import build from "./build/build.ts"
 
 useFlags()
 
@@ -24,7 +25,7 @@ const pantry = usePantry()
 const dry = Deno.args.map(pkgutils.parse)
 const gha = !!Deno.env.get("GITHUB_ACTIONS")
 const group_it = gha && dry.length > 1
-const rv: Package[] = []
+const rv: Installation[] = []
 
 if (usePrefix().string != "/opt") {
   console.error({ TEA_PREFIX: usePrefix().string })
@@ -40,14 +41,13 @@ for (const rq of dry) {
     console.log({ building: pkg.project })
   }
 
-  await build(pkg)
-  rv.push(pkg)
+  const install = await build(pkg)
+  rv.push(install)
 
   if (group_it) {
     console.log("::endgroup::")
   }
 }
 
-const built_pkgs = rv.map(pkgutils.str).join(" ")
-const txt = `::set-output name=pkgs::${built_pkgs}\n`
-await Deno.stdout.write(new TextEncoder().encode(txt))
+await set_output("pkgs", rv.map(x => pkgutils.str(x.pkg)))
+await set_output("paths", rv.map(x => x.path), '%0A')

scripts/build/build.ts

@@ -14,7 +14,7 @@ const { platform } = host()
 
 export default async function _build(pkg: Package) {
   try {
-    await __build(pkg)
+    return await __build(pkg)
   } catch (e) {
     cellar.keg(pkg).isDirectory()?.isEmpty()?.rm()  // don’t leave empty kegs around
     throw e

scripts/upload-sync.ts

@@ -23,14 +23,16 @@ const s3 = new S3({
   region: "us-east-1",
 })
 
+const offy = useOffLicense('s3')
 const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!)
 
 for (const stowed of await useCache().ls()) {
-  const url = useOffLicense('s3').url(stowed)
+  const url = offy.url(stowed)
+  const key = offy.key(stowed)
 
   console.log({ checking: url })
 
-  const inRepo = await bucket.headObject(url.pathname)
+  const inRepo = await bucket.headObject(key)
   const repoChecksum = inRepo ? await checksum(`${url}.sha256sum`) : undefined
 
   // path.read() returns a string; this is easier to get a UInt8Array
@@ -43,8 +45,8 @@ for (const stowed of await useCache().ls()) {
 
     console.log({ uploading: url })
 
-    await bucket.putObject(url.pathname, contents)
-    await bucket.putObject(`${url.pathname}.sha256sum`, body)
+    await bucket.putObject(key, contents)
+    await bucket.putObject(`${key}.sha256sum`, body)
 
     console.log({ uploaded: url })
   }

scripts/upload.ts

@@ -12,11 +12,12 @@ args:
 
 import { S3 } from "s3"
 import { pkg as pkgutils } from "utils"
-import { useFlags, useOffLicense } from "hooks"
+import { useFlags, useOffLicense, useCache } from "hooks"
 import { Package, PackageRequirement } from "types"
 import SemVer, * as semver from "semver"
 import { dirname, basename } from "deno/path/mod.ts"
 import Path from "path"
+import { set_output } from "./utils/gha.ts"
 
 useFlags()
 
@@ -30,6 +31,7 @@ const s3 = new S3({
 
 const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!)
 const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
+const cache = useCache()
 
 const pkgs = args_get("pkgs").map(pkgutils.parse).map(assert_pkg)
 const bottles = args_get("bottles")
@@ -52,30 +54,33 @@ function args_get(key: string): string[] {
   }
 }
 
+const rv: string[] = []
+const put = async (key: string, body: string | Path | Uint8Array) => {
+  console.log({ uploading: body, to: key })
+  rv.push(`/${key}`)
+  if (body instanceof Path) {
+    body = await Deno.readFile(body.string)
+  } else if (typeof body === "string") {
+    body = encode(body)
+  }
+  return bucket.putObject(key, body)
+}
+
 for (const [index, pkg] of pkgs.entries()) {
   const bottle = new Path(bottles[index])
   const checksum = checksums[index]
-  const compression = bottle.extname() == '.tar.gz' ? 'gz' : 'xz'
-  const key = useOffLicense('s3').key({
-    pkg,
-    type: 'bottle',
-    compression,
-  })
-
-  //FIXME stream it to S3
-  const bottle_contents = await Deno.readFile(bottle.string)
-  const checksum_contents = fixup_checksum(await Deno.readFile(checksum), bottle.basename())
+  const stowed = cache.decode(bottle)!
+  const key = useOffLicense('s3').key(stowed)
   const versions = await get_versions(key, pkg)
 
-  console.log({ uploading: key })
-
-  await bucket.putObject(key, bottle_contents)
-  await bucket.putObject(`${key}.sha256sum`, checksum_contents)
-  await bucket.putObject(`${dirname(key)}/versions.txt`, encode(versions.join("\n")))
-
-  console.log({ uploaded: key })
+  //FIXME stream the bottle (at least) to S3
+  await put(key, bottle)
+  await put(`${key}.sha256sum`, `${checksum}  ${basename(key)}`)
+  await put(`${dirname(key)}/versions.txt`, versions.join("\n"))
 }
 
+await set_output('cf-invalidation-paths', rv)
+
 //end
 
 async function get_versions(key: string, pkg: Package): Promise<SemVer[]> {

scripts/utils/gha.ts

@@ -0,0 +1,19 @@
+
+const e = new TextEncoder()
+const encode = e.encode.bind(e)
+
+export function set_output<T>(name: string, arr: T[], separator = " ") {
+  const value = arr.map(escape).join(separator)
+  const txt = `::set-output name=${name}::${value}`
+  return Deno.stdout.write(encode(`${txt}\n`))
+}
+
+//TODO HTML escapes probs
+function escape<T>(input: T): string {
+  const out = `${input}`
+  if (/[<>]/.test(out)) {
+    return `"${out}"`
+  } else {
+    return out
+  }
+}