pantry/.github/actions/upload/upload.ts

146 lines
4.3 KiB
TypeScript
Raw Normal View History

2023-02-11 17:59:00 +03:00
#!/usr/bin/env tea
2022-08-01 22:43:40 +03:00
/*---
args:
- deno
- run
- --allow-net
2022-09-08 23:40:35 +03:00
- --allow-read
2022-08-17 15:22:22 +03:00
- --allow-env
- --allow-write
2022-08-01 22:43:40 +03:00
---*/
import { S3, S3Bucket } from "s3"
2022-09-27 23:42:26 +03:00
import { pkg as pkgutils } from "utils"
import { useCache, useFlags, useOffLicense, usePrefix } from "hooks"
2022-09-27 22:11:35 +03:00
import { Package, PackageRequirement } from "types"
2022-09-20 14:53:40 +03:00
import SemVer, * as semver from "semver"
import { basename, dirname } from "deno/path/mod.ts"
import { retry } from "deno/async/retry.ts"
2022-11-25 06:32:26 +03:00
import { decode as base64Decode } from "deno/encoding/base64.ts"
2022-09-27 22:11:35 +03:00
import Path from "path"
2023-02-24 23:50:10 +03:00
import { set_output } from "../../scripts/utils/gha.ts"
import { sha256 } from "../bottle/bottle.ts"
2022-08-17 15:22:22 +03:00
//------------------------------------------------------------------------- funcs
function args_get(key: string): string[] {
const it = Deno.args[Symbol.iterator]()
while (true) {
const { value, done } = it.next()
if (done) throw new Error()
if (value === `--${key}`) break
}
const rv: string[] = []
while (true) {
const { value, done } = it.next()
if (done) return rv
if (value.startsWith("--")) return rv
rv.push(value)
2022-09-08 23:40:35 +03:00
}
}
function assert_pkg(pkg: Package | PackageRequirement) {
if ("version" in pkg) {
return pkg
} else {
return {
project: pkg.project,
version: new SemVer(pkg.constraint, {tolerant: true}),
}
}
}
async function get_versions(key: string, pkg: Package, bucket: S3Bucket): Promise<SemVer[]> {
const prefix = dirname(key)
const rsp = await bucket.listObjects({ prefix })
//FIXME? API isnt clear if these nulls indicate failure or not
//NOTE if this is a new package then some empty results is expected
const got = rsp
?.contents
?.compact((x) => x.key)
.map((x) => basename(x))
.filter((x) => x.match(/v.*\.tar\.gz$/))
.map((x) => x.replace(/v(.*)\.tar\.gz/, "$1")) ??
[]
// have to add pkg.version as put and get are not atomic
return [...new Set([...got, pkg.version.toString()])]
.compact(semver.parse)
.sort(semver.compare)
}
async function put(key: string, body: string | Path | Uint8Array, bucket: S3Bucket) {
2022-09-28 18:14:46 +03:00
console.log({ uploading: body, to: key })
rv.push(`/${key}`)
if (body instanceof Path) {
body = await Deno.readFile(body.string)
} else if (typeof body === "string") {
body = encode(body)
}
// @ts-ignore typescript doesn't narrow the types properly here
return retry(()=>bucket.putObject(key, body))
2022-09-28 18:14:46 +03:00
}
//------------------------------------------------------------------------- main
useFlags()
if (Deno.args.length === 0) throw new Error("no args supplied")
const s3 = new S3({
accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
region: "us-east-1",
})
const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!)
const encode = (() => {
const e = new TextEncoder()
return e.encode.bind(e)
})()
const cache = useCache()
const pkgs = args_get("pkgs").map(pkgutils.parse).map(assert_pkg)
const srcs = args_get("srcs")
const bottles = args_get("bottles")
const checksums = args_get("checksums")
2022-11-25 06:32:26 +03:00
const signatures = args_get("signatures")
const rv: string[] = []
for (const [index, pkg] of pkgs.entries()) {
const bottle = usePrefix().join(bottles[index])
const checksum = checksums[index]
2022-11-25 06:32:26 +03:00
const signature = base64Decode(signatures[index])
2022-09-28 18:14:46 +03:00
const stowed = cache.decode(bottle)!
const key = useOffLicense("s3").key(stowed)
const versions = await get_versions(key, pkg, bucket)
2022-08-01 22:43:40 +03:00
2022-09-28 18:14:46 +03:00
//FIXME stream the bottle (at least) to S3
await put(key, bottle, bucket)
await put(`${key}.sha256sum`, `${checksum} ${basename(key)}`, bucket)
2022-11-25 06:32:26 +03:00
await put(`${key}.asc`, signature, bucket)
await put(`${dirname(key)}/versions.txt`, versions.join("\n"), bucket)
// mirror the sources
if (srcs[index] != "~") {
2022-10-17 22:20:42 +03:00
const src = usePrefix().join(srcs[index])
if (src.isDirectory()) {
// we almost certainly expanded `~` to the users home directory
continue
}
const srcKey = useOffLicense("s3").key({
pkg: stowed.pkg,
type: "src",
extname: src.extname(),
})
const srcChecksum = await sha256(src)
const srcVersions = await get_versions(srcKey, pkg, bucket)
await put(srcKey, src, bucket)
await put(`${srcKey}.sha256sum`, `${srcChecksum} ${basename(srcKey)}`, bucket)
await put(`${dirname(srcKey)}/versions.txt`, srcVersions.join("\n"), bucket)
}
2022-08-01 22:43:40 +03:00
}
await set_output("cf-invalidation-paths", rv)