mirror of
https://github.com/ivabus/pantry
synced 2024-11-22 08:25:07 +03:00
infra
This commit is contained in:
parent
dd4e1bfba5
commit
5880b26de2
8 changed files with 410 additions and 0 deletions
48
README.md
Normal file
48
README.md
Normal file
|
@ -0,0 +1,48 @@
|
|||
![tea](https://tea.xyz/banner.png)
|
||||
|
||||
tea is a decentralized package manager—this requires a decentralized package
|
||||
registry. We’re releasing our testnet later this year. In the meantime the
|
||||
pantry is our stop-gap solution.
|
||||
|
||||
# Entry Requirements
|
||||
|
||||
This pantry only accepts devtools that we feel confident we can maintain.
|
||||
Quality and robustness are our goals. If you want other tools you can maintain
|
||||
your own pantry and we’ll build the binaries.
|
||||
|
||||
# Philosophy
|
||||
|
||||
Fundamentally we're coming at this from the perspective that the maintainer
|
||||
should decide how their software is distributed and we’re making the tools so
|
||||
they can do that in cross platform way.
|
||||
|
||||
This repo is a bootstrap and is stubs.
|
||||
|
||||
# Naming
|
||||
|
||||
We use fully-qualified names. Naming is hard, and the world has spent a while
|
||||
trying to get it right. In this kind of domain the *correct choice* is
|
||||
to namespace.
|
||||
|
||||
# Coming Soon
|
||||
|
||||
## Maintaining Your Own Pantry
|
||||
|
||||
We will build binaries for forks of this repository and then surface the
|
||||
`package.yml`s you maintain to users of tea/cli. This feature is coming
|
||||
soon and will require signed commits and that you come to our Discord and say
|
||||
hi.
|
||||
|
||||
## Hosting and Maintaining Your Own `package.yml`
|
||||
|
||||
If you have a website you can host your own `package.yml` there and we will
|
||||
build binaries for you. This feature is coming soon and will require
|
||||
signed, versioned tags and signed source tarballs.
|
||||
|
||||
|
||||
# Dependencies
|
||||
|
||||
| Project | Version |
|
||||
|-------------|---------|
|
||||
| deno.land | ^1.18 |
|
||||
| tea.xyz | ^0 |
|
1
import-map.json
Symbolic link
1
import-map.json
Symbolic link
|
@ -0,0 +1 @@
|
|||
../cli/import-map.json
|
105
scripts/bottle.ts
Executable file
105
scripts/bottle.ts
Executable file
|
@ -0,0 +1,105 @@
|
|||
#!/usr/bin/env -S tea -E
|
||||
|
||||
/* ---
|
||||
args:
|
||||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-run
|
||||
- --allow-read=/opt/
|
||||
- --allow-write=/opt/
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
--- */
|
||||
|
||||
import { Installation, parsePackageRequirement } from "types"
|
||||
import { Path } from "types"
|
||||
import useCellar from "hooks/useCellar.ts"
|
||||
import { run } from "utils"
|
||||
import useCache from "hooks/useCache.ts"
|
||||
|
||||
const cellar = useCellar()
|
||||
const filesListName = 'files.txt'
|
||||
|
||||
const rv: Path[] = []
|
||||
for (const pkg of Deno.args.map(parsePackageRequirement)) {
|
||||
console.log({ bottling: { pkg } })
|
||||
|
||||
const installation = await cellar.resolve(pkg)
|
||||
const path = await bottle(installation)
|
||||
|
||||
if (!path) throw new Error("wtf: bottle already exists")
|
||||
|
||||
console.log({ bottled: { path } })
|
||||
|
||||
rv.push(path)
|
||||
rv.push(installation.path.join(filesListName))
|
||||
}
|
||||
|
||||
if (rv.length === 0) throw new Error("no input provided")
|
||||
|
||||
const paths = rv.map(x => x.string).join('%0A')
|
||||
const txt = `::set-output name=filenames::${paths}\n`
|
||||
await Deno.stdout.write(new TextEncoder().encode(txt))
|
||||
|
||||
|
||||
//------------------------------------------------------------------------- funcs
|
||||
async function bottle({ path: kegdir, pkg }: Installation): Promise<Path> {
|
||||
|
||||
const files = await walk(kegdir, path => {
|
||||
/// HACK: `go` requires including the `src` dir
|
||||
const isGo = kegdir.string.match(/\/go.dev\//)
|
||||
switch (path.relative({ to: kegdir })) {
|
||||
case 'src':
|
||||
return isGo ? 'accumulate' : 'skip'
|
||||
case 'build.sh':
|
||||
case filesListName:
|
||||
return 'skip'
|
||||
default:
|
||||
return 'accumulate'
|
||||
}
|
||||
})
|
||||
const relativePaths = files.map(x => x.relative({ to: cellar.prefix }))
|
||||
const filelist = kegdir
|
||||
.join(filesListName)
|
||||
.write({
|
||||
text: relativePaths.join("\n")
|
||||
})
|
||||
const tarball = useCache().bottle(pkg)
|
||||
|
||||
await run({
|
||||
cmd: [
|
||||
"tar", "zcf", tarball, "--files-from", filelist
|
||||
],
|
||||
cwd: cellar.prefix
|
||||
})
|
||||
|
||||
return tarball
|
||||
}
|
||||
|
||||
// using our own because of: https://github.com/denoland/deno_std/issues/1359
|
||||
// but frankly this also is more suitable for our needs here
|
||||
type Continuation = 'accumulate' | 'skip'
|
||||
|
||||
export async function walk(root: Path, body: (entry: Path) => Continuation): Promise<Path[]> {
|
||||
const rv: Path[] = []
|
||||
const stack: Path[] = [root]
|
||||
|
||||
do {
|
||||
root = stack.pop()!
|
||||
for await (const [path, entry] of root.ls()) {
|
||||
switch (body(path)) {
|
||||
case 'accumulate':
|
||||
if (entry.isDirectory) {
|
||||
stack.push(path)
|
||||
} else {
|
||||
rv.push(path)
|
||||
}
|
||||
break
|
||||
case 'skip':
|
||||
continue
|
||||
}
|
||||
}
|
||||
} while (stack.length > 0)
|
||||
|
||||
return rv
|
||||
}
|
104
scripts/build.ts
Executable file
104
scripts/build.ts
Executable file
|
@ -0,0 +1,104 @@
|
|||
#!/usr/bin/env -S tea -E
|
||||
|
||||
/*---
|
||||
args:
|
||||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-run
|
||||
- --allow-read=/opt,/Library/Developer/CommandLineTools
|
||||
- --allow-write=/opt
|
||||
- --allow-env
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
||||
import useSourceUnarchiver from "hooks/useSourceUnarchiver.ts"
|
||||
import useCellar from "hooks/useCellar.ts"
|
||||
import usePantry from "hooks/usePantry.ts"
|
||||
import useCache from "hooks/useCache.ts"
|
||||
import { lvl1 as link } from "prefab/link.ts"
|
||||
import install from "prefab/install.ts"
|
||||
import build from "prefab/build.ts"
|
||||
import { semver, PackageRequirement, Package } from "types"
|
||||
import { parsePackageRequirement } from "types"
|
||||
import hydrate from "prefab/hydrate.ts"
|
||||
import resolve from "prefab/resolve.ts"
|
||||
|
||||
const pantry = usePantry()
|
||||
const cellar = useCellar()
|
||||
|
||||
const dry = Deno.args.map(project => {
|
||||
const match = project.match(/projects\/(.*)\/package.yml/)
|
||||
return match ? match[1] : project
|
||||
}).map(parsePackageRequirement)
|
||||
|
||||
//FIXME this isn’t as specific as we are above
|
||||
const explicit_deps = new Set(dry.map(({ project }) => project))
|
||||
|
||||
const build_deps = await (async () => {
|
||||
const rv: PackageRequirement[] = []
|
||||
for (const pkg of dry) {
|
||||
const foo = await pantry.getDeps(pkg)
|
||||
rv.push(...foo.build)
|
||||
}
|
||||
return rv
|
||||
})()
|
||||
|
||||
const wet = [...await hydrate(dry), ...build_deps]
|
||||
const gas = await resolve(await (async () => {
|
||||
const rv: PackageRequirement[] = []
|
||||
for (const pkg of wet) {
|
||||
if (await cellar.isInstalled(pkg)) continue
|
||||
if (explicit_deps.has(pkg.project)) continue
|
||||
rv.push(pkg)
|
||||
}
|
||||
return rv
|
||||
})())
|
||||
|
||||
for await (const pkg of gas) {
|
||||
console.log({ installing: pkg.project })
|
||||
const installation = install(pkg)
|
||||
await link(installation)
|
||||
}
|
||||
|
||||
const rv: Package[] = []
|
||||
for await (const pkg of dry) {
|
||||
console.log({ building: pkg.project })
|
||||
|
||||
const versions = await pantry.getVersions(pkg)
|
||||
const version = semver.maxSatisfying(versions, pkg.constraint)
|
||||
if (!version) throw "no-version-found"
|
||||
await b({ project: pkg.project, version })
|
||||
|
||||
rv.push({ project: pkg.project, version })
|
||||
}
|
||||
|
||||
const built_pkgs = rv.map(({ project, version }) => `${project}@${version}`).join(" ")
|
||||
const txt = `::set-output name=pkgs::${built_pkgs}\n`
|
||||
await Deno.stdout.write(new TextEncoder().encode(txt))
|
||||
|
||||
//end
|
||||
|
||||
|
||||
|
||||
async function b(pkg: Package) {
|
||||
|
||||
// Get the source
|
||||
const prebuild = async () => {
|
||||
const dstdir = useCellar().mkpath(pkg).join("src")
|
||||
const { url, stripComponents } = await pantry.getDistributable(pkg)
|
||||
const { download } = useCache()
|
||||
const zip = await download({ pkg, url, type: 'src' })
|
||||
await useSourceUnarchiver().unarchive({
|
||||
dstdir,
|
||||
zipfile: zip,
|
||||
stripComponents
|
||||
})
|
||||
}
|
||||
|
||||
const deps = await pantry.getDeps(pkg)
|
||||
|
||||
// Build and link
|
||||
const path = await build({ pkg, deps, prebuild })
|
||||
await link({ path, pkg })
|
||||
}
|
78
scripts/test.ts
Executable file
78
scripts/test.ts
Executable file
|
@ -0,0 +1,78 @@
|
|||
#!/usr/bin/env -S tea -E
|
||||
|
||||
/*---
|
||||
args:
|
||||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-run
|
||||
- --allow-read
|
||||
- --allow-write=/opt/tea.xyz/tmp
|
||||
- --allow-env
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
||||
import { parsePackage, semver, Path, PackageRequirement, PlainObject } from "types"
|
||||
import usePantry from "hooks/usePantry.ts"
|
||||
import useShellEnv, { expand } from "hooks/useShellEnv.ts"
|
||||
import { run, undent, isPlainObject, isString } from "utils"
|
||||
import { validatePackageRequirement } from "utils/lvl2.ts"
|
||||
|
||||
//TODO install any other deps
|
||||
|
||||
const pantry = usePantry()
|
||||
const pkg = parsePackage(Deno.args[0])
|
||||
const self = {
|
||||
project: pkg.project,
|
||||
constraint: new semver.Range(pkg.version.toString())
|
||||
}
|
||||
const [yml] = await pantry.getYAML(pkg)
|
||||
const deps: PackageRequirement[] = [self, ...get_deps()]
|
||||
|
||||
const env = await useShellEnv(deps)
|
||||
|
||||
let text = undent`
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
set -x
|
||||
|
||||
${expand(env.vars)}
|
||||
|
||||
`
|
||||
|
||||
const tmp = Path.mktemp()
|
||||
|
||||
try {
|
||||
if (yml.test.fixture) {
|
||||
const fixture = tmp.join("fixture.tea").write({ text: yml.test.fixture.toString() })
|
||||
text += `export FIXTURE="${fixture}"\n\n`
|
||||
}
|
||||
|
||||
text += await pantry.getScript(pkg, 'test')
|
||||
text += "\n"
|
||||
|
||||
const cmd = tmp
|
||||
.join("test.sh")
|
||||
.write({ text, force: true })
|
||||
.chmod(0o500)
|
||||
await run({ cmd })
|
||||
} finally {
|
||||
tmp.rm({ recursive: true })
|
||||
}
|
||||
|
||||
function get_deps() {
|
||||
const rv: PackageRequirement[] = []
|
||||
attempt(yml.dependencies)
|
||||
attempt(yml.test.dependencies)
|
||||
return rv
|
||||
|
||||
function attempt(obj: PlainObject) {
|
||||
if (isPlainObject(obj))
|
||||
for (const [project, constraint] of Object.entries(obj)) {
|
||||
const pkg = validatePackageRequirement({ project, constraint })
|
||||
if (pkg) rv.push(pkg)
|
||||
}
|
||||
}
|
||||
}
|
63
scripts/upload.ts
Executable file
63
scripts/upload.ts
Executable file
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env -S tea -E
|
||||
|
||||
/*---
|
||||
args:
|
||||
- deno
|
||||
- run
|
||||
- --allow-net
|
||||
- --allow-read=/opt
|
||||
- --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_S3
|
||||
- --import-map={{ srcroot }}/import-map.json
|
||||
---*/
|
||||
|
||||
import { S3 } from "s3"
|
||||
import { crypto } from "deno/crypto/mod.ts"
|
||||
import useCache from "hooks/useCache.ts"
|
||||
import useCellar from "hooks/useCellar.ts"
|
||||
import { encodeToString } from "encodeToString"
|
||||
import { Package, parsePackageRequirement, SemVer, semver } from "types"
|
||||
|
||||
if (Deno.args.length === 0) throw new Error("no args supplied")
|
||||
|
||||
const s3 = new S3({
|
||||
accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
|
||||
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
|
||||
region: "us-east-1",
|
||||
})
|
||||
|
||||
const bucket = s3.getBucket(Deno.env.get("AWS_S3")!)
|
||||
const cellar = useCellar()
|
||||
|
||||
for (const rq of Deno.args.map(parsePackageRequirement)) {
|
||||
const {pkg} = await cellar.resolve(rq)
|
||||
const key = useCache().s3Key(pkg)
|
||||
const bottle = useCache().bottle(pkg)
|
||||
|
||||
console.log({ key });
|
||||
|
||||
//FIXME stream it to S3
|
||||
const bottle_contents = await Deno.readFile(bottle.string)
|
||||
const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
|
||||
const [basename, dirname] = (split => [split.pop(), split.join("/")])(key.split("/"))
|
||||
const checksum = encodeToString(new Uint8Array(await crypto.subtle.digest("SHA-256", bottle_contents)))
|
||||
const versions = await get_versions(pkg)
|
||||
|
||||
await bucket.putObject(key, bottle_contents)
|
||||
await bucket.putObject(`${key}.sha256sum`, encode(`${checksum} ${basename}`))
|
||||
await bucket.putObject(`${dirname}/versions.txt`, encode(versions.join("\n")))
|
||||
|
||||
console.log({ uploaded: key })
|
||||
}
|
||||
|
||||
async function get_versions(pkg: Package): Promise<SemVer[]> {
|
||||
const prefix = useCache().s3Key(pkg)
|
||||
const rsp = await bucket.listObjects({ prefix })
|
||||
|
||||
//FIXME? API isn’t clear if these nulls indicate failure or not
|
||||
//NOTE if this is a new package then some empty results is expected
|
||||
return rsp
|
||||
?.contents
|
||||
?.compactMap(x => x.key?.split("/").pop())
|
||||
.compactMap(semver.coerce) //FIXME coerce is too loose
|
||||
.sort() ?? []
|
||||
}
|
1
src
Symbolic link
1
src
Symbolic link
|
@ -0,0 +1 @@
|
|||
../cli/src
|
10
tsconfig.json
Normal file
10
tsconfig.json
Normal file
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"allowJs": false,
|
||||
"target": "ESNext",
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"module": "ESNext",
|
||||
"strict": true,
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue