infra

.github/workflows/ci.yml

@@ -0,0 +1,112 @@
+name: ci
+on: pull_request
+
+concurrency:
+  #TODO group: ${{ github.ref }}
+  group: only-one-due-to-versions.txt-generation-requiring-serial-ness
+  cancel-in-progress: true
+
+jobs:
+  get-diff:
+    # separate step since our image has no `git`
+    runs-on: ubuntu-latest
+    outputs:
+      diff: ${{ steps.diff.outputs.diff }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
+        id: diff
+        with:
+          PATTERNS: projects/**/package.yml
+
+  build:
+    needs: [get-diff]
+    runs-on: ${{ matrix.os}}
+    strategy:
+      matrix:
+        include:
+          - os: macos-11
+            container: ~
+          - os: ubuntu-latest
+            container: ghcr.io/teaxyz/infuser:main
+    container: ${{ matrix.container }}
+    defaults:
+      run:
+        working-directory: pantry
+    steps:
+      - name: co pantry
+        uses: actions/checkout@v3
+        with:
+          path: pantry
+
+      - name: co cli
+        uses: actions/checkout@v3
+        with:
+          path: cli
+          repository: teaxyz/cli
+          token: ${{ secrets.TEMP_JACOBS_GITHUB_PAT }}
+
+      - name: HACKS
+        run: |
+          case ${{ matrix.os }} in
+          ubuntu-latest)
+            rm -rf /opt/tea.xyz/var/pantry
+            ln -s $GITHUB_WORKSPACE/pantry /opt/tea.xyz/var/pantry
+            mkdir .git  # no git in our image
+            ;;
+          macos-11)
+            sudo mkdir -p /opt/tea.xyz/var
+            sudo chown -R $(whoami):staff /opt
+            ln -s $GITHUB_WORKSPACE/pantry /opt/tea.xyz/var/pantry
+            mkdir -p /opt/deno.land/v1.23.3/.tricksy
+
+            # HACKs for teaxyz/setup since it currently requires the working dir to be a srcroot
+            cp README.md ..
+            mkdir ../.git
+            ;;
+          *)
+            exit 1
+          esac
+
+          mkdir -p projects/deno.land
+          echo 'versions: [1.23.3]' > projects/deno.land/package.yml
+
+      - name: HACKS
+        uses: denoland/setup-deno@v1
+        if: ${{ matrix.os == 'macos-11' }}
+
+      - uses: teaxyz/setup@v0
+        env:
+          TEA_SECRET: ${{ secrets.TEA_SECRET }}
+        if: ${{ matrix.os == 'macos-11' }}
+
+      - name: Build
+        run: |
+          ./scripts/build.ts ${{ needs.get-diff.outputs.diff }}
+        id: build
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Test
+        run: ./scripts/test.ts ${{ steps.build.outputs.pkgs }}
+
+      - name: Bottle
+        run: ./scripts/bottle.ts ${{ steps.build.outputs.pkgs }}
+        id: bottle
+
+      - name: Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.os }}
+          path: ${{ steps.bottle.outputs.filenames }}
+          if-no-files-found: error
+
+      # TODO only upload if all jobs succeed
+      # TODO only upload when we merge
+      # TODO upload to a staging location until we release new pantry versions
+      - name: Upload
+        run: ./scripts/upload.ts ${{ steps.build.outputs.pkgs }}
+        env:
+          AWS_S3: ${{ secrets.AWS_S3 }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+  "deno.enable": true,
+  "deno.lint": true,
+  "deno.unstable": true,
+  "deno.importMap": "import-map.json",
+  "deno.config": "tsconfig.json"
+}

README.md

@@ -0,0 +1,48 @@
+![tea](https://tea.xyz/banner.png)
+
+tea is a decentralized package manager—this requires a decentralized package
+registry. We’re releasing our testnet later this year. In the meantime the
+pantry is our stop-gap solution.
+
+# Entry Requirements
+
+This pantry only accepts devtools that we feel confident we can maintain.
+Quality and robustness are our goals. If you want other tools you can maintain
+your own pantry and we’ll build the binaries.
+
+# Philosophy
+
+Fundamentally we're coming at this from the perspective that the maintainer
+should decide how their software is distributed and we’re making the tools so
+they can do that in cross platform way.
+
+This repo is a bootstrap and is stubs.
+
+# Naming
+
+We use fully-qualified names. Naming is hard, and the world has spent a while
+trying to get it right. In this kind of domain the *correct choice* is
+to namespace.
+
+# Coming Soon
+
+## Maintaining Your Own Pantry
+
+We will build binaries for forks of this repository and then surface the
+`package.yml`s you maintain to users of tea/cli. This feature is coming
+soon and will require signed commits and that you come to our Discord and say
+hi.
+
+## Hosting and Maintaining Your Own `package.yml`
+
+If you have a website you can host your own `package.yml` there and we will
+build binaries for you. This feature is coming soon and will require
+signed, versioned tags and signed source tarballs.
+
+
+# Dependencies
+
+|   Project   | Version |
+|-------------|---------|
+| deno.land   | ^1.18   |
+| tea.xyz     | ^0      |

import-map.json

@@ -0,0 +1 @@
+../cli/import-map.json

scripts/bottle.ts

@@ -0,0 +1,105 @@
+#!/usr/bin/env -S tea -E
+
+/* ---
+args:
+  - deno
+  - run
+  - --allow-net
+  - --allow-run
+  - --allow-read=/opt/
+  - --allow-write=/opt/
+  - --import-map={{ srcroot }}/import-map.json
+--- */
+
+import { Installation, parsePackageRequirement } from "types"
+import { Path } from "types"
+import useCellar from "hooks/useCellar.ts"
+import { run } from "utils"
+import useCache from "hooks/useCache.ts"
+
+const cellar = useCellar()
+const filesListName = 'files.txt'
+
+const rv: Path[] = []
+for (const pkg of Deno.args.map(parsePackageRequirement)) {
+  console.log({ bottling: { pkg } })
+
+  const installation = await cellar.resolve(pkg)
+  const path = await bottle(installation)
+
+  if (!path) throw new Error("wtf: bottle already exists")
+
+  console.log({ bottled: { path } })
+
+  rv.push(path)
+  rv.push(installation.path.join(filesListName))
+}
+
+if (rv.length === 0) throw new Error("no input provided")
+
+const paths = rv.map(x => x.string).join('%0A')
+const txt = `::set-output name=filenames::${paths}\n`
+await Deno.stdout.write(new TextEncoder().encode(txt))
+
+
+//------------------------------------------------------------------------- funcs
+async function bottle({ path: kegdir, pkg }: Installation): Promise<Path> {
+
+  const files = await walk(kegdir, path => {
+    /// HACK: `go` requires including the `src` dir
+    const isGo = kegdir.string.match(/\/go.dev\//)
+    switch (path.relative({ to: kegdir })) {
+    case 'src':
+      return isGo ? 'accumulate' : 'skip'
+    case 'build.sh':
+    case filesListName:
+      return 'skip'
+    default:
+      return 'accumulate'
+    }
+  })
+  const relativePaths = files.map(x => x.relative({ to: cellar.prefix }))
+  const filelist = kegdir
+    .join(filesListName)
+    .write({
+      text: relativePaths.join("\n")
+    })
+  const tarball = useCache().bottle(pkg)
+
+  await run({
+    cmd: [
+      "tar", "zcf", tarball, "--files-from", filelist
+    ],
+    cwd: cellar.prefix
+  })
+
+  return tarball
+}
+
+// using our own because of: https://github.com/denoland/deno_std/issues/1359
+// but frankly this also is more suitable for our needs here
+type Continuation = 'accumulate' | 'skip'
+
+export async function walk(root: Path, body: (entry: Path) => Continuation): Promise<Path[]> {
+  const rv: Path[] = []
+  const stack: Path[] = [root]
+
+  do {
+    root = stack.pop()!
+    for await (const [path, entry] of root.ls()) {
+      switch (body(path)) {
+      case 'accumulate':
+        if (entry.isDirectory) {
+          stack.push(path)
+        } else {
+          rv.push(path)
+        }
+        break
+      case 'skip':
+        continue
+      }
+    }
+  } while (stack.length > 0)
+
+  return rv
+}

scripts/build.ts

@@ -0,0 +1,104 @@
+#!/usr/bin/env -S tea -E
+
+/*---
+args:
+  - deno
+  - run
+  - --allow-net
+  - --allow-run
+  - --allow-read=/opt,/Library/Developer/CommandLineTools
+  - --allow-write=/opt
+  - --allow-env
+  - --import-map={{ srcroot }}/import-map.json
+---*/
+
+import useSourceUnarchiver from "hooks/useSourceUnarchiver.ts"
+import useCellar from "hooks/useCellar.ts"
+import usePantry from "hooks/usePantry.ts"
+import useCache from "hooks/useCache.ts"
+import { lvl1 as link } from "prefab/link.ts"
+import install from "prefab/install.ts"
+import build from "prefab/build.ts"
+import { semver, PackageRequirement, Package } from "types"
+import { parsePackageRequirement } from "types"
+import hydrate from "prefab/hydrate.ts"
+import resolve from "prefab/resolve.ts"
+
+const pantry = usePantry()
+const cellar = useCellar()
+
+const dry = Deno.args.map(project => {
+  const match = project.match(/projects\/(.*)\/package.yml/)
+  return match ? match[1] : project
+}).map(parsePackageRequirement)
+
+//FIXME this isn’t as specific as we are above
+const explicit_deps = new Set(dry.map(({ project }) => project))
+
+const build_deps = await (async () => {
+  const rv: PackageRequirement[] = []
+  for (const pkg of dry) {
+    const foo = await pantry.getDeps(pkg)
+    rv.push(...foo.build)
+  }
+  return rv
+})()
+
+const wet = [...await hydrate(dry), ...build_deps]
+const gas = await resolve(await (async () => {
+  const rv: PackageRequirement[] = []
+  for (const pkg of wet) {
+    if (await cellar.isInstalled(pkg)) continue
+    if (explicit_deps.has(pkg.project)) continue
+    rv.push(pkg)
+  }
+  return rv
+})())
+
+for await (const pkg of gas) {
+  console.log({ installing: pkg.project })
+  const installation = install(pkg)
+  await link(installation)
+}
+
+const rv: Package[] = []
+for await (const pkg of dry) {
+  console.log({ building: pkg.project })
+
+  const versions = await pantry.getVersions(pkg)
+  const version = semver.maxSatisfying(versions, pkg.constraint)
+  if (!version) throw "no-version-found"
+  await b({ project: pkg.project, version })
+
+  rv.push({ project: pkg.project, version })
+}
+
+const built_pkgs = rv.map(({ project, version }) => `${project}@${version}`).join(" ")
+const txt = `::set-output name=pkgs::${built_pkgs}\n`
+await Deno.stdout.write(new TextEncoder().encode(txt))
+
+//end
+
+
+
+async function b(pkg: Package) {
+
+  // Get the source
+  const prebuild = async () => {
+    const dstdir = useCellar().mkpath(pkg).join("src")
+    const { url, stripComponents } = await pantry.getDistributable(pkg)
+    const { download } = useCache()
+    const zip = await download({ pkg, url, type: 'src' })
+    await useSourceUnarchiver().unarchive({
+      dstdir,
+      zipfile: zip,
+      stripComponents
+    })
+  }
+
+  const deps = await pantry.getDeps(pkg)
+
+  // Build and link
+  const path = await build({ pkg, deps, prebuild })
+  await link({ path, pkg })
+}

scripts/test.ts

@@ -0,0 +1,78 @@
+#!/usr/bin/env -S tea -E
+
+/*---
+args:
+  - deno
+  - run
+  - --allow-net
+  - --allow-run
+  - --allow-read
+  - --allow-write=/opt/tea.xyz/tmp
+  - --allow-env
+  - --import-map={{ srcroot }}/import-map.json
+---*/
+
+import { parsePackage, semver, Path, PackageRequirement, PlainObject } from "types"
+import usePantry from "hooks/usePantry.ts"
+import useShellEnv, { expand } from "hooks/useShellEnv.ts"
+import { run, undent, isPlainObject, isString } from "utils"
+import { validatePackageRequirement } from "utils/lvl2.ts"
+
+//TODO install any other deps
+
+const pantry = usePantry()
+const pkg = parsePackage(Deno.args[0])
+const self = {
+  project: pkg.project,
+  constraint: new semver.Range(pkg.version.toString())
+}
+const [yml] = await pantry.getYAML(pkg)
+const deps: PackageRequirement[] = [self, ...get_deps()]
+
+const env = await useShellEnv(deps)
+
+let text = undent`
+  #!/bin/bash
+
+  set -e
+  set -o pipefail
+  set -x
+
+  ${expand(env.vars)}
+
+  `
+
+const tmp = Path.mktemp()
+
+try {
+  if (yml.test.fixture) {
+    const fixture = tmp.join("fixture.tea").write({ text: yml.test.fixture.toString() })
+    text += `export FIXTURE="${fixture}"\n\n`
+  }
+
+  text += await pantry.getScript(pkg, 'test')
+  text += "\n"
+
+  const cmd = tmp
+    .join("test.sh")
+    .write({ text, force: true })
+    .chmod(0o500)
+  await run({ cmd })
+} finally {
+  tmp.rm({ recursive: true })
+}
+
+function get_deps() {
+  const rv: PackageRequirement[] = []
+  attempt(yml.dependencies)
+  attempt(yml.test.dependencies)
+  return rv
+
+  function attempt(obj: PlainObject) {
+    if (isPlainObject(obj))
+    for (const [project, constraint] of Object.entries(obj)) {
+      const pkg = validatePackageRequirement({ project, constraint })
+      if (pkg) rv.push(pkg)
+    }
+  }
+}

scripts/upload.ts

@@ -0,0 +1,63 @@
+#!/usr/bin/env -S tea -E
+
+/*---
+args:
+  - deno
+  - run
+  - --allow-net
+  - --allow-read=/opt
+  - --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_S3
+  - --import-map={{ srcroot }}/import-map.json
+---*/
+
+import { S3 } from "s3"
+import { crypto } from "deno/crypto/mod.ts"
+import useCache from "hooks/useCache.ts"
+import useCellar from "hooks/useCellar.ts"
+import { encodeToString } from "encodeToString"
+import { Package, parsePackageRequirement, SemVer, semver } from "types"
+
+if (Deno.args.length === 0) throw new Error("no args supplied")
+
+const s3 = new S3({
+  accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
+  secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
+  region: "us-east-1",
+})
+
+const bucket = s3.getBucket(Deno.env.get("AWS_S3")!)
+const cellar = useCellar()
+
+for (const rq of Deno.args.map(parsePackageRequirement)) {
+  const {pkg} = await cellar.resolve(rq)
+  const key = useCache().s3Key(pkg)
+  const bottle = useCache().bottle(pkg)
+
+  console.log({ key });
+
+  //FIXME stream it to S3
+  const bottle_contents = await Deno.readFile(bottle.string)
+  const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
+  const [basename, dirname] = (split => [split.pop(), split.join("/")])(key.split("/"))
+  const checksum = encodeToString(new Uint8Array(await crypto.subtle.digest("SHA-256", bottle_contents)))
+  const versions = await get_versions(pkg)
+
+  await bucket.putObject(key, bottle_contents)
+  await bucket.putObject(`${key}.sha256sum`, encode(`${checksum}  ${basename}`))
+  await bucket.putObject(`${dirname}/versions.txt`, encode(versions.join("\n")))
+
+  console.log({ uploaded: key })
+}
+
+async function get_versions(pkg: Package): Promise<SemVer[]> {
+  const prefix = useCache().s3Key(pkg)
+  const rsp = await bucket.listObjects({ prefix })
+
+  //FIXME? API isn’t clear if these nulls indicate failure or not
+  //NOTE if this is a new package then some empty results is expected
+  return rsp
+    ?.contents
+    ?.compactMap(x => x.key?.split("/").pop())
+    .compactMap(semver.coerce) //FIXME coerce is too loose
+    .sort() ?? []
+}

src

@@ -0,0 +1 @@
+../cli/src

tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "allowJs": false,
+    "target": "ESNext",
+    "moduleResolution": "node",
+    "esModuleInterop": true,
+    "module": "ESNext",
+    "strict": true,
+  }
+}