pantry/scripts/upload-sync.ts

65 lines
1.8 KiB
TypeScript
Raw Normal View History

2022-09-08 23:40:35 +03:00
#!/usr/bin/env -S tea -E
/*---
args:
- deno
- run
2022-09-13 19:46:10 +03:00
- --allow-read
2022-09-08 23:40:35 +03:00
- --allow-net
2022-09-13 19:46:10 +03:00
- --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_S3_BUCKET,TEA_PREFIX
2022-09-08 23:40:35 +03:00
- --import-map={{ srcroot }}/import-map.json
---*/
2022-09-20 14:53:40 +03:00
import { readAll, readerFromStreamReader } from "deno/streams/mod.ts"
2022-09-27 22:11:35 +03:00
import { useCache, useOffLicense } from "hooks"
import { Package } from "types"
import { Sha256 } from "deno/hash/sha256.ts"
import { S3 } from "s3"
import Path from "path"
2022-09-08 23:40:35 +03:00
const s3 = new S3({
accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
region: "us-east-1",
2022-09-20 14:53:40 +03:00
})
2022-09-08 23:40:35 +03:00
2022-09-28 18:14:46 +03:00
const offy = useOffLicense('s3')
2022-09-20 14:53:40 +03:00
const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!)
2022-09-08 23:40:35 +03:00
2022-09-27 22:11:35 +03:00
for (const stowed of await useCache().ls()) {
2022-09-28 18:14:46 +03:00
const url = offy.url(stowed)
const key = offy.key(stowed)
2022-09-08 23:40:35 +03:00
2022-09-27 23:42:26 +03:00
console.log({ checking: url })
2022-09-08 23:40:35 +03:00
2022-09-28 18:14:46 +03:00
const inRepo = await bucket.headObject(key)
2022-09-27 23:42:26 +03:00
const repoChecksum = inRepo ? await checksum(`${url}.sha256sum`) : undefined
2022-09-08 23:40:35 +03:00
// path.read() returns a string; this is easier to get a UInt8Array
2022-09-27 22:11:35 +03:00
const contents = await Deno.readFile(stowed.path.string)
2022-09-20 14:53:40 +03:00
const sha256sum = new Sha256().update(contents).toString()
2022-09-08 23:40:35 +03:00
if (!inRepo || repoChecksum !== sha256sum) {
2022-09-27 23:42:26 +03:00
const basename = url.path().basename()
2022-09-08 23:40:35 +03:00
const body = new TextEncoder().encode(`${sha256sum} ${basename}`)
2022-09-27 23:42:26 +03:00
console.log({ uploading: url })
2022-09-08 23:40:35 +03:00
2022-09-28 18:14:46 +03:00
await bucket.putObject(key, contents)
await bucket.putObject(`${key}.sha256sum`, body)
2022-09-08 23:40:35 +03:00
2022-09-27 23:42:26 +03:00
console.log({ uploaded: url })
2022-09-08 23:40:35 +03:00
}
}
async function checksum(url: string) {
const rsp = await fetch(url)
if (!rsp.ok) throw new Error(`404-not-found: ${url}`)
const rdr = rsp.body?.getReader()
if (!rdr) throw new Error(`Couldnt read: ${url}`)
const r = await readAll(readerFromStreamReader(rdr))
return new TextDecoder().decode(r).split(' ')[0]
2022-09-27 22:11:35 +03:00
}
type RV = Package & {bottle: Path}