mirror of
https://github.com/ivabus/pantry
synced 2024-09-20 00:30:48 +03:00
making things relocatable (#122)
This commit is contained in:
parent
24d40b7c4f
commit
3226265d7c
|
@ -56,3 +56,11 @@ knowledge. Please keep it tidy.
|
|||
```sh
|
||||
scripts/ls.ts | xargs scripts/sort.ts | xargs scripts/build.ts
|
||||
```
|
||||
|
||||
## Typecheck
|
||||
|
||||
```sh
|
||||
for x in scripts/*.ts src/app.ts; do
|
||||
deno check --import-map=$SRCROOT/import-map.json $x
|
||||
done
|
||||
```
|
||||
|
|
|
@ -7,8 +7,8 @@ args:
|
|||
- --allow-net
|
||||
- --allow-run
|
||||
- --allow-env
|
||||
- --allow-read=/opt/
|
||||
- --allow-write=/opt/
|
||||
- --allow-read
|
||||
- --allow-write
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
--- */
|
||||
|
||||
|
@ -21,43 +21,48 @@ import useFlags from "hooks/useFlags.ts"
|
|||
import { crypto } from "deno/crypto/mod.ts"
|
||||
import { encodeToString } from "encodeToString"
|
||||
|
||||
useFlags()
|
||||
|
||||
const cellar = useCellar()
|
||||
const filesListName = 'files.txt'
|
||||
|
||||
const bottles: Path[] = []
|
||||
const fileLists: Path[] = []
|
||||
for (const pkg of Deno.args.map(parsePackageRequirement)) {
|
||||
console.log({ bottling: { pkg } })
|
||||
|
||||
const installation = await cellar.resolve(pkg)
|
||||
const path = await bottle(installation)
|
||||
const checksum = await sha256(path)
|
||||
//-------------------------------------------------------------------------- main
|
||||
|
||||
if (!path) throw new Error("wtf: bottle already exists")
|
||||
if (!checksum) throw new Error("failed to compute checksum")
|
||||
if (import.meta.main) {
|
||||
useFlags()
|
||||
|
||||
console.log({ bottled: { path } })
|
||||
const bottles: Path[] = []
|
||||
const fileLists: Path[] = []
|
||||
for (const pkg of Deno.args.map(parsePackageRequirement)) {
|
||||
console.log({ bottling: { pkg } })
|
||||
|
||||
bottles.push(path)
|
||||
bottles.push(checksum)
|
||||
fileLists.push(installation.path.join(filesListName))
|
||||
const installation = await cellar.resolve(pkg)
|
||||
const path = await bottle(installation)
|
||||
const checksum = await sha256(path)
|
||||
|
||||
if (!path) throw new Error("wtf: bottle already exists")
|
||||
if (!checksum) throw new Error("failed to compute checksum")
|
||||
|
||||
console.log({ bottled: { path } })
|
||||
|
||||
bottles.push(path)
|
||||
bottles.push(checksum)
|
||||
fileLists.push(installation.path.join(filesListName))
|
||||
}
|
||||
|
||||
if (bottles.length === 0) throw new Error("no input provided")
|
||||
|
||||
const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
|
||||
|
||||
const bottleList = bottles.map(x => x.string).join(" ")
|
||||
await Deno.stdout.write(encode(`::set-output name=bottles::${bottleList}\n`))
|
||||
|
||||
const paths = [...bottles, ...fileLists].map(x => x.string).join('%0A')
|
||||
await Deno.stdout.write(encode(`::set-output name=filenames::${paths}\n`))
|
||||
}
|
||||
|
||||
if (bottles.length === 0) throw new Error("no input provided")
|
||||
|
||||
const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
|
||||
|
||||
const bottleList = bottles.map(x => x.string).join(" ")
|
||||
await Deno.stdout.write(encode(`::set-output name=bottles::${bottleList}\n`))
|
||||
|
||||
const paths = [...bottles, ...fileLists].map(x => x.string).join('%0A')
|
||||
await Deno.stdout.write(encode(`::set-output name=filenames::${paths}\n`))
|
||||
|
||||
|
||||
//------------------------------------------------------------------------- funcs
|
||||
async function bottle({ path: kegdir, pkg }: Installation): Promise<Path> {
|
||||
export async function bottle({ path: kegdir, pkg }: Installation): Promise<Path> {
|
||||
|
||||
const files = await walk(kegdir, path => {
|
||||
/// HACK: `go` requires including the `src` dir
|
||||
|
@ -126,4 +131,4 @@ async function sha256(file: Path): Promise<Path> {
|
|||
await Deno.writeFile(checksum_file.string, checksum_contents)
|
||||
|
||||
return checksum_file
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ args:
|
|||
- run
|
||||
- --allow-net
|
||||
- --allow-run
|
||||
- --allow-read=/opt,/Library/Developer/CommandLineTools
|
||||
- --allow-read
|
||||
- --allow-write=/opt
|
||||
- --allow-env
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
|
@ -29,6 +29,7 @@ const pantry = usePantry()
|
|||
const cellar = useCellar()
|
||||
|
||||
const dry = Deno.args.map(parsePackageRequirement)
|
||||
const gha = !!Deno.env.get("GITHUB_ACTIONS")
|
||||
|
||||
const rv: Package[] = []
|
||||
for (const pkgrq of dry) {
|
||||
|
@ -37,7 +38,20 @@ for (const pkgrq of dry) {
|
|||
if (!version) throw "no-version-found"
|
||||
const pkg = { project: pkgrq.project, version }
|
||||
|
||||
console.log({ building: pkgrq.project })
|
||||
const installation = await cellar.isInstalled(pkg)
|
||||
if (installation) {
|
||||
console.log({ cleaning: installation.path })
|
||||
for await (const [path, {name}] of installation.path.ls()) {
|
||||
if (name == 'src') continue
|
||||
path.rm({ recursive: true })
|
||||
}
|
||||
}
|
||||
|
||||
if (gha) {
|
||||
console.log("::group::", `${pkg.project}@${pkg.version}`)
|
||||
} else {
|
||||
console.log({ building: pkgrq.project })
|
||||
}
|
||||
|
||||
// Get the source
|
||||
const prebuild = async () => {
|
||||
|
@ -66,6 +80,10 @@ for (const pkgrq of dry) {
|
|||
await link({ path, pkg })
|
||||
|
||||
rv.push(pkg)
|
||||
|
||||
if (gha) {
|
||||
console.log("::endgroup::")
|
||||
}
|
||||
}
|
||||
|
||||
const built_pkgs = rv.map(({ project, version }) => `${project}@${version}`).join(" ")
|
||||
|
|
|
@ -35,8 +35,8 @@ const wet = await hydrate(dry, get_deps)
|
|||
const gas = wet.pkgs.compactMap(({project}) => {
|
||||
if (Deno.args.includes('-i')) {
|
||||
return project
|
||||
} else {
|
||||
return explicit.has(project) || project
|
||||
} else if (!explicit.has(project)){
|
||||
return project
|
||||
}
|
||||
})
|
||||
|
||||
|
|
42
scripts/fixup-checksums.ts
Executable file
42
scripts/fixup-checksums.ts
Executable file
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env -S tea -E
|
||||
|
||||
/*---
|
||||
args:
|
||||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,S3_BUCKET
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
||||
import { S3 } from "s3";
|
||||
import { crypto } from "deno/crypto/mod.ts";
|
||||
import { readerFromStreamReader, readAll } from "deno/streams/conversion.ts";
|
||||
import { encodeToString } from "encodeToString";
|
||||
|
||||
const s3 = new S3({
|
||||
accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
|
||||
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
|
||||
region: "us-east-1",
|
||||
});
|
||||
|
||||
const bucket = s3.getBucket(Deno.env.get("S3_BUCKET")!);
|
||||
|
||||
for await (const pkg of bucket.listAllObjects({ batchSize: 200 })) {
|
||||
if (!pkg.key?.endsWith('.tar.gz')) { continue }
|
||||
console.log({ checking: pkg.key });
|
||||
|
||||
if (!await bucket.headObject(`${pkg.key}.sha256sum`)) {
|
||||
console.log({ missingChecksum: pkg.key })
|
||||
const reader = (await bucket.getObject(pkg.key))!.body.getReader()
|
||||
const contents = await readAll(readerFromStreamReader(reader))
|
||||
|
||||
const basename = pkg.key.split("/").pop()
|
||||
const sha256sum = encodeToString(new Uint8Array(await crypto.subtle.digest("SHA-256", contents)))
|
||||
const body = new TextEncoder().encode(`${sha256sum} ${basename}`)
|
||||
|
||||
await bucket.putObject(`${pkg.key}.sha256sum`, body);
|
||||
|
||||
console.log({ uploaded: `${pkg.key}.sha256sum` });
|
||||
}
|
||||
}
|
78
scripts/inventory.ts
Executable file
78
scripts/inventory.ts
Executable file
|
@ -0,0 +1,78 @@
|
|||
#!/usr/bin/env -S tea -E
|
||||
|
||||
/*---
|
||||
args:
|
||||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,S3_BUCKET
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
||||
import { S3 } from "s3";
|
||||
import { stringify as yaml } from "deno/encoding/yaml.ts"
|
||||
import { stringify as csv } from "deno/encoding/csv.ts"
|
||||
import { Inventory } from "../src/hooks/useInventory.ts";
|
||||
|
||||
const s3 = new S3({
|
||||
accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
|
||||
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
|
||||
region: "us-east-1",
|
||||
});
|
||||
|
||||
const bucket = s3.getBucket(Deno.env.get("S3_BUCKET")!);
|
||||
|
||||
const inventory: Inventory = {}
|
||||
const flat = []
|
||||
|
||||
for await (const pkg of bucket.listAllObjects({ batchSize: 200 })) {
|
||||
if (!pkg.key?.endsWith('.tar.gz')) { continue }
|
||||
|
||||
const matches = pkg.key.match(new RegExp("^(.*)/(.*)/(.*)/v([0-9]+\.[0-9]+\.[0-9]+)\.tar\.gz$"))
|
||||
|
||||
if (!matches) { continue }
|
||||
|
||||
const [_, project, platform, arch, version] = matches
|
||||
|
||||
if (!inventory[project]) inventory[project] = {}
|
||||
if (!inventory[project][platform]) inventory[project][platform] = {}
|
||||
if (!inventory[project][platform]) inventory[project][platform] = {}
|
||||
inventory[project][platform][arch] = [...(inventory[project]?.[platform]?.[arch] ?? []), version]
|
||||
flat.push({ project, platform, arch, version })
|
||||
}
|
||||
|
||||
/// For ultimate user-friendliness, we store this data 4 ways:
|
||||
/// YAML, JSON, CSV, flat text
|
||||
|
||||
const te = new TextEncoder()
|
||||
|
||||
// YAML: type Inventory
|
||||
|
||||
const yml = te.encode(yaml(inventory))
|
||||
|
||||
bucket.putObject("versions.yml", yml)
|
||||
|
||||
// JSON: type Inventory
|
||||
|
||||
const json = te.encode(JSON.stringify(inventory))
|
||||
|
||||
bucket.putObject("versions.json", json)
|
||||
|
||||
// CSV: project,platform,arch,version
|
||||
|
||||
const csvData = te.encode(await csv(flat, ["project", "platform", "arch", "version"]))
|
||||
|
||||
bucket.putObject("versions.csv", csvData)
|
||||
|
||||
// TXT: per project/platform/arch, newline-delimited
|
||||
|
||||
for(const [project, platforms] of Object.entries(inventory)) {
|
||||
for (const [platform, archs] of Object.entries(platforms)) {
|
||||
for (const [arch, versions] of Object.entries(archs)) {
|
||||
const txt = te.encode(versions.join("\n"))
|
||||
bucket.putObject(`${project}/${platform}/${arch}/versions.txt`, txt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//end
|
|
@ -7,24 +7,28 @@ args:
|
|||
- deno
|
||||
- run
|
||||
- --allow-env
|
||||
- --allow-read=/opt/tea.xyz/var/pantry
|
||||
- --allow-read
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
||||
import { Path } from "types"
|
||||
import useFlags from "hooks/useFlags.ts"
|
||||
import useCellar from "hooks/useCellar.ts"
|
||||
|
||||
const flags = useFlags()
|
||||
const prefix = new Path(`${useCellar().prefix}/tea.xyz/var/pantry/projects`)
|
||||
|
||||
interface Entry {
|
||||
project: string
|
||||
path: Path
|
||||
}
|
||||
|
||||
|
||||
const prefix = new Path('/opt/tea.xyz/var/pantry/projects')
|
||||
|
||||
//FIXME unfortunately importing executes the script below
|
||||
//------------------------------------------------------------------------- funcs
|
||||
export async function* ls(): AsyncGenerator<Entry> {
|
||||
for await (const path of _ls_pantry(prefix)) {
|
||||
yield {
|
||||
name: path.parent().relative({ to: prefix }),
|
||||
path: path.string
|
||||
project: path.parent().relative({ to: prefix }),
|
||||
path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -43,22 +47,23 @@ async function* _ls_pantry(dir: Path): AsyncGenerator<Path> {
|
|||
}
|
||||
}
|
||||
|
||||
interface Entry {
|
||||
name: string
|
||||
path: string
|
||||
}
|
||||
//-------------------------------------------------------------------------- main
|
||||
if (import.meta.main) {
|
||||
const flags = useFlags()
|
||||
|
||||
const rv: Entry[] = []
|
||||
for await (const item of ls()) {
|
||||
rv.push(item)
|
||||
}
|
||||
const rv: Entry[] = []
|
||||
for await (const item of ls()) {
|
||||
rv.push(item)
|
||||
}
|
||||
|
||||
if (Deno.env.get("GITHUB_ACTIONS")) {
|
||||
const projects = rv.map(x => x.name).join(":")
|
||||
console.log(`::set-output name=projects::${projects}`)
|
||||
} else if (flags.json) {
|
||||
const output = JSON.stringify(rv, null, 2)
|
||||
console.log(output)
|
||||
} else {
|
||||
console.log(rv.map(x => x.name).join("\n"))
|
||||
if (Deno.env.get("GITHUB_ACTIONS")) {
|
||||
const projects = rv.map(x => x.project).join(":")
|
||||
console.log(`::set-output name=projects::${projects}`)
|
||||
} else if (flags.json) {
|
||||
const obj = rv.map(({ path, project }) => ({ path: path.string, project }))
|
||||
const out = JSON.stringify(obj, null, 2)
|
||||
console.log(out)
|
||||
} else {
|
||||
console.log(rv.map(x => x.project).join("\n"))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,8 +6,8 @@ args:
|
|||
- run
|
||||
- --allow-net
|
||||
- --allow-run
|
||||
- --allow-read=/opt,/Library/Developer/CommandLineTools
|
||||
- --allow-write=/opt
|
||||
- --allow-read
|
||||
- --allow-write={{ tea.prefix }}
|
||||
- --allow-env
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
|
|
@ -29,7 +29,10 @@ const pantry = usePantry()
|
|||
|
||||
const pkg = await (async () => {
|
||||
if (magic) {
|
||||
const i = await cellar.resolve(parsePackageRequirement(Deno.args[0]))
|
||||
const project = Deno.args[0]
|
||||
const match = project.match(/projects\/(.+)\/package.yml/)
|
||||
const parsable = match ? match[1] : project
|
||||
const i = await cellar.resolve(parsePackageRequirement(parsable))
|
||||
return i.pkg
|
||||
} else {
|
||||
return parsePackage(Deno.args[0])
|
||||
|
|
60
scripts/upload-sync.ts
Executable file
60
scripts/upload-sync.ts
Executable file
|
@ -0,0 +1,60 @@
|
|||
#!/usr/bin/env -S tea -E
|
||||
|
||||
/*---
|
||||
args:
|
||||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-read={{ tea.prefix }}/tea.xyz/var/www
|
||||
- --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,S3_BUCKET
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
||||
import { S3 } from "s3";
|
||||
import { crypto } from "deno/crypto/mod.ts";
|
||||
import useCache from "hooks/useCache.ts";
|
||||
import { encodeToString } from "encodeToString";
|
||||
import { readAll, readerFromStreamReader } from "deno/streams/mod.ts";
|
||||
|
||||
const s3 = new S3({
|
||||
accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
|
||||
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
|
||||
region: "us-east-1",
|
||||
});
|
||||
|
||||
const bucket = s3.getBucket(Deno.env.get("S3_BUCKET")!);
|
||||
|
||||
for (const pkg of await useCache().ls()) {
|
||||
const key = useCache().s3Key(pkg)
|
||||
const bottle = useCache().bottle(pkg)
|
||||
|
||||
console.log({ checking: key });
|
||||
|
||||
const inRepo = await bucket.headObject(key)
|
||||
const repoChecksum = inRepo ? await checksum(`https://dist.tea.xyz/${key}.sha256sum`) : undefined
|
||||
|
||||
// path.read() returns a string; this is easier to get a UInt8Array
|
||||
const contents = await Deno.readFile(bottle.string);
|
||||
const sha256sum = encodeToString(new Uint8Array(await crypto.subtle.digest("SHA-256", contents)))
|
||||
|
||||
if (!inRepo || repoChecksum !== sha256sum) {
|
||||
const basename = key.split("/").pop()
|
||||
const body = new TextEncoder().encode(`${sha256sum} ${basename}`)
|
||||
|
||||
console.log({ uploading: key });
|
||||
|
||||
await bucket.putObject(key, contents);
|
||||
await bucket.putObject(`${key}.sha256sum`, body);
|
||||
|
||||
console.log({ uploaded: key });
|
||||
}
|
||||
}
|
||||
|
||||
async function checksum(url: string) {
|
||||
const rsp = await fetch(url)
|
||||
if (!rsp.ok) throw new Error(`404-not-found: ${url}`)
|
||||
const rdr = rsp.body?.getReader()
|
||||
if (!rdr) throw new Error(`Couldn’t read: ${url}`)
|
||||
const r = await readAll(readerFromStreamReader(rdr))
|
||||
return new TextDecoder().decode(r).split(' ')[0]
|
||||
}
|
|
@ -5,8 +5,7 @@ args:
|
|||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-read=/opt
|
||||
- --allow-write=/opt/tea.xyz/var/www
|
||||
- --allow-read
|
||||
- --allow-env
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
@ -47,8 +46,11 @@ for (const filename of Deno.args) {
|
|||
|
||||
const req = parsePackageRequirement(`${match[1]}@${match[2]}`)
|
||||
|
||||
if (path.basename().match(/\.sha256sum$/)) { checksums.add(`${req.project}@${req.constraint.raw}`) }
|
||||
else { bottles.add(req) }
|
||||
if (path.basename().match(/\.sha256sum$/)) {
|
||||
checksums.add(`${req.project}@${req.constraint.raw}`)
|
||||
} else {
|
||||
bottles.add(req)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure our sets are the same:
|
||||
|
|
Loading…
Reference in a new issue