This commit is contained in:
Max Howell 2022-08-01 15:43:40 -04:00
parent dd4e1bfba5
commit 5880b26de2
8 changed files with 410 additions and 0 deletions

48
README.md Normal file
View file

@ -0,0 +1,48 @@
![tea](https://tea.xyz/banner.png)
tea is a decentralized package manager—this requires a decentralized package
registry. Were releasing our testnet later this year. In the meantime the
pantry is our stop-gap solution.
# Entry Requirements
This pantry only accepts devtools that we feel confident we can maintain.
Quality and robustness are our goals. If you want other tools you can maintain
your own pantry and well build the binaries.
# Philosophy
Fundamentally we're coming at this from the perspective that the maintainer
should decide how their software is distributed and were making the tools so
they can do that in cross platform way.
This repo is a bootstrap and is stubs.
# Naming
We use fully-qualified names. Naming is hard, and the world has spent a while
trying to get it right. In this kind of domain the *correct choice* is
to namespace.
# Coming Soon
## Maintaining Your Own Pantry
We will build binaries for forks of this repository and then surface the
`package.yml`s you maintain to users of tea/cli. This feature is coming
soon and will require signed commits and that you come to our Discord and say
hi.
## Hosting and Maintaining Your Own `package.yml`
If you have a website you can host your own `package.yml` there and we will
build binaries for you. This feature is coming soon and will require
signed, versioned tags and signed source tarballs.
# Dependencies
| Project | Version |
|-------------|---------|
| deno.land | ^1.18 |
| tea.xyz | ^0 |

1
import-map.json Symbolic link
View file

@ -0,0 +1 @@
../cli/import-map.json

105
scripts/bottle.ts Executable file
View file

@ -0,0 +1,105 @@
#!/usr/bin/env -S tea -E
/* ---
args:
- deno
- run
- --allow-net
- --allow-run
- --allow-read=/opt/
- --allow-write=/opt/
- --import-map={{ srcroot }}/import-map.json
--- */
import { Installation, parsePackageRequirement } from "types"
import { Path } from "types"
import useCellar from "hooks/useCellar.ts"
import { run } from "utils"
import useCache from "hooks/useCache.ts"
const cellar = useCellar()
const filesListName = 'files.txt'
const rv: Path[] = []
for (const pkg of Deno.args.map(parsePackageRequirement)) {
console.log({ bottling: { pkg } })
const installation = await cellar.resolve(pkg)
const path = await bottle(installation)
if (!path) throw new Error("wtf: bottle already exists")
console.log({ bottled: { path } })
rv.push(path)
rv.push(installation.path.join(filesListName))
}
if (rv.length === 0) throw new Error("no input provided")
const paths = rv.map(x => x.string).join('%0A')
const txt = `::set-output name=filenames::${paths}\n`
await Deno.stdout.write(new TextEncoder().encode(txt))
//------------------------------------------------------------------------- funcs
async function bottle({ path: kegdir, pkg }: Installation): Promise<Path> {
const files = await walk(kegdir, path => {
/// HACK: `go` requires including the `src` dir
const isGo = kegdir.string.match(/\/go.dev\//)
switch (path.relative({ to: kegdir })) {
case 'src':
return isGo ? 'accumulate' : 'skip'
case 'build.sh':
case filesListName:
return 'skip'
default:
return 'accumulate'
}
})
const relativePaths = files.map(x => x.relative({ to: cellar.prefix }))
const filelist = kegdir
.join(filesListName)
.write({
text: relativePaths.join("\n")
})
const tarball = useCache().bottle(pkg)
await run({
cmd: [
"tar", "zcf", tarball, "--files-from", filelist
],
cwd: cellar.prefix
})
return tarball
}
// using our own because of: https://github.com/denoland/deno_std/issues/1359
// but frankly this also is more suitable for our needs here
type Continuation = 'accumulate' | 'skip'
export async function walk(root: Path, body: (entry: Path) => Continuation): Promise<Path[]> {
const rv: Path[] = []
const stack: Path[] = [root]
do {
root = stack.pop()!
for await (const [path, entry] of root.ls()) {
switch (body(path)) {
case 'accumulate':
if (entry.isDirectory) {
stack.push(path)
} else {
rv.push(path)
}
break
case 'skip':
continue
}
}
} while (stack.length > 0)
return rv
}

104
scripts/build.ts Executable file
View file

@ -0,0 +1,104 @@
#!/usr/bin/env -S tea -E
/*---
args:
- deno
- run
- --allow-net
- --allow-run
- --allow-read=/opt,/Library/Developer/CommandLineTools
- --allow-write=/opt
- --allow-env
- --import-map={{ srcroot }}/import-map.json
---*/
import useSourceUnarchiver from "hooks/useSourceUnarchiver.ts"
import useCellar from "hooks/useCellar.ts"
import usePantry from "hooks/usePantry.ts"
import useCache from "hooks/useCache.ts"
import { lvl1 as link } from "prefab/link.ts"
import install from "prefab/install.ts"
import build from "prefab/build.ts"
import { semver, PackageRequirement, Package } from "types"
import { parsePackageRequirement } from "types"
import hydrate from "prefab/hydrate.ts"
import resolve from "prefab/resolve.ts"
const pantry = usePantry()
const cellar = useCellar()
const dry = Deno.args.map(project => {
const match = project.match(/projects\/(.*)\/package.yml/)
return match ? match[1] : project
}).map(parsePackageRequirement)
//FIXME this isnt as specific as we are above
const explicit_deps = new Set(dry.map(({ project }) => project))
const build_deps = await (async () => {
const rv: PackageRequirement[] = []
for (const pkg of dry) {
const foo = await pantry.getDeps(pkg)
rv.push(...foo.build)
}
return rv
})()
const wet = [...await hydrate(dry), ...build_deps]
const gas = await resolve(await (async () => {
const rv: PackageRequirement[] = []
for (const pkg of wet) {
if (await cellar.isInstalled(pkg)) continue
if (explicit_deps.has(pkg.project)) continue
rv.push(pkg)
}
return rv
})())
for await (const pkg of gas) {
console.log({ installing: pkg.project })
const installation = install(pkg)
await link(installation)
}
const rv: Package[] = []
for await (const pkg of dry) {
console.log({ building: pkg.project })
const versions = await pantry.getVersions(pkg)
const version = semver.maxSatisfying(versions, pkg.constraint)
if (!version) throw "no-version-found"
await b({ project: pkg.project, version })
rv.push({ project: pkg.project, version })
}
const built_pkgs = rv.map(({ project, version }) => `${project}@${version}`).join(" ")
const txt = `::set-output name=pkgs::${built_pkgs}\n`
await Deno.stdout.write(new TextEncoder().encode(txt))
//end
async function b(pkg: Package) {
// Get the source
const prebuild = async () => {
const dstdir = useCellar().mkpath(pkg).join("src")
const { url, stripComponents } = await pantry.getDistributable(pkg)
const { download } = useCache()
const zip = await download({ pkg, url, type: 'src' })
await useSourceUnarchiver().unarchive({
dstdir,
zipfile: zip,
stripComponents
})
}
const deps = await pantry.getDeps(pkg)
// Build and link
const path = await build({ pkg, deps, prebuild })
await link({ path, pkg })
}

78
scripts/test.ts Executable file
View file

@ -0,0 +1,78 @@
#!/usr/bin/env -S tea -E
/*---
args:
- deno
- run
- --allow-net
- --allow-run
- --allow-read
- --allow-write=/opt/tea.xyz/tmp
- --allow-env
- --import-map={{ srcroot }}/import-map.json
---*/
import { parsePackage, semver, Path, PackageRequirement, PlainObject } from "types"
import usePantry from "hooks/usePantry.ts"
import useShellEnv, { expand } from "hooks/useShellEnv.ts"
import { run, undent, isPlainObject, isString } from "utils"
import { validatePackageRequirement } from "utils/lvl2.ts"
//TODO install any other deps
const pantry = usePantry()
const pkg = parsePackage(Deno.args[0])
const self = {
project: pkg.project,
constraint: new semver.Range(pkg.version.toString())
}
const [yml] = await pantry.getYAML(pkg)
const deps: PackageRequirement[] = [self, ...get_deps()]
const env = await useShellEnv(deps)
let text = undent`
#!/bin/bash
set -e
set -o pipefail
set -x
${expand(env.vars)}
`
const tmp = Path.mktemp()
try {
if (yml.test.fixture) {
const fixture = tmp.join("fixture.tea").write({ text: yml.test.fixture.toString() })
text += `export FIXTURE="${fixture}"\n\n`
}
text += await pantry.getScript(pkg, 'test')
text += "\n"
const cmd = tmp
.join("test.sh")
.write({ text, force: true })
.chmod(0o500)
await run({ cmd })
} finally {
tmp.rm({ recursive: true })
}
function get_deps() {
const rv: PackageRequirement[] = []
attempt(yml.dependencies)
attempt(yml.test.dependencies)
return rv
function attempt(obj: PlainObject) {
if (isPlainObject(obj))
for (const [project, constraint] of Object.entries(obj)) {
const pkg = validatePackageRequirement({ project, constraint })
if (pkg) rv.push(pkg)
}
}
}

63
scripts/upload.ts Executable file
View file

@ -0,0 +1,63 @@
#!/usr/bin/env -S tea -E
/*---
args:
- deno
- run
- --allow-net
- --allow-read=/opt
- --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_S3
- --import-map={{ srcroot }}/import-map.json
---*/
import { S3 } from "s3"
import { crypto } from "deno/crypto/mod.ts"
import useCache from "hooks/useCache.ts"
import useCellar from "hooks/useCellar.ts"
import { encodeToString } from "encodeToString"
import { Package, parsePackageRequirement, SemVer, semver } from "types"
if (Deno.args.length === 0) throw new Error("no args supplied")
const s3 = new S3({
accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
region: "us-east-1",
})
const bucket = s3.getBucket(Deno.env.get("AWS_S3")!)
const cellar = useCellar()
for (const rq of Deno.args.map(parsePackageRequirement)) {
const {pkg} = await cellar.resolve(rq)
const key = useCache().s3Key(pkg)
const bottle = useCache().bottle(pkg)
console.log({ key });
//FIXME stream it to S3
const bottle_contents = await Deno.readFile(bottle.string)
const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
const [basename, dirname] = (split => [split.pop(), split.join("/")])(key.split("/"))
const checksum = encodeToString(new Uint8Array(await crypto.subtle.digest("SHA-256", bottle_contents)))
const versions = await get_versions(pkg)
await bucket.putObject(key, bottle_contents)
await bucket.putObject(`${key}.sha256sum`, encode(`${checksum} ${basename}`))
await bucket.putObject(`${dirname}/versions.txt`, encode(versions.join("\n")))
console.log({ uploaded: key })
}
async function get_versions(pkg: Package): Promise<SemVer[]> {
const prefix = useCache().s3Key(pkg)
const rsp = await bucket.listObjects({ prefix })
//FIXME? API isnt clear if these nulls indicate failure or not
//NOTE if this is a new package then some empty results is expected
return rsp
?.contents
?.compactMap(x => x.key?.split("/").pop())
.compactMap(semver.coerce) //FIXME coerce is too loose
.sort() ?? []
}

1
src Symbolic link
View file

@ -0,0 +1 @@
../cli/src

10
tsconfig.json Normal file
View file

@ -0,0 +1,10 @@
{
"compilerOptions": {
"allowJs": false,
"target": "ESNext",
"moduleResolution": "node",
"esModuleInterop": true,
"module": "ESNext",
"strict": true,
}
}