From 6d48c72aca6f0987c6ba1dd354804f5cf8d895e5 Mon Sep 17 00:00:00 2001 From: Max Howell Date: Fri, 24 Feb 2023 15:50:10 -0500 Subject: [PATCH] Actions Refactor (#336) --- .github/actions/bottle/action.yml | 26 ++++++ {scripts => .github/actions/bottle}/bottle.ts | 4 +- .github/actions/cache/action.yml | 28 ++++++ .../{apple-signing => codesign}/action.yml | 22 ++--- .github/actions/fetch-pr-artifacts/action.yml | 35 ++++++++ .../fetch-pr-artifacts}/fetch-pr-artifacts.ts | 0 .github/actions/get-platform/action.yml | 39 ++++++++ .../actions/get-platform}/get-platform.ts | 13 +-- .github/actions/has-artifacts/action.yml | 39 ++++++++ .../actions/has-artifacts}/has-artifacts.ts | 6 +- .github/actions/setup-brewkit/action.yml | 39 ++++++++ .../actions/stage-build-artifacts/action.yml | 40 +++++++++ .../stage-build-artifacts}/cache-artifacts.ts | 2 +- .github/actions/upload/action.yml | 47 ++++++++++ {scripts => .github/actions/upload}/upload.ts | 4 +- deno.jsonc => .github/deno.jsonc | 9 +- .../scripts}/index-packages.ts | 12 +-- .../scripts}/map-projects-to-githubs.ts | 0 {scripts => .github/scripts}/utils/args.ts | 0 {scripts => .github/scripts}/utils/gha.ts | 1 - .github/workflows/bottle.yml | 90 ++++++------------- .github/workflows/build.yml | 70 +++------------ .github/workflows/cd.yml | 13 ++- .github/workflows/ci-scripts.yml | 12 ++- .github/workflows/ci.yml | 2 +- .github/workflows/cleanup.yml | 6 +- .github/workflows/complain.yml | 2 - .github/workflows/index-data.yml | 9 +- .github/workflows/new-version.yml | 2 +- .vscode/settings.json | 2 +- scripts/upload-sync.ts | 63 ------------- tea.yaml | 5 ++ 32 files changed, 383 insertions(+), 259 deletions(-) create mode 100644 .github/actions/bottle/action.yml rename {scripts => .github/actions/bottle}/bottle.ts (96%) create mode 100644 .github/actions/cache/action.yml rename .github/actions/{apple-signing => codesign}/action.yml (76%) create mode 100644 .github/actions/fetch-pr-artifacts/action.yml rename {scripts => .github/actions/fetch-pr-artifacts}/fetch-pr-artifacts.ts (100%) create mode 100644 .github/actions/get-platform/action.yml rename {scripts => .github/actions/get-platform}/get-platform.ts (85%) create mode 100644 .github/actions/has-artifacts/action.yml rename {scripts => .github/actions/has-artifacts}/has-artifacts.ts (89%) create mode 100644 .github/actions/setup-brewkit/action.yml create mode 100644 .github/actions/stage-build-artifacts/action.yml rename {scripts => .github/actions/stage-build-artifacts}/cache-artifacts.ts (95%) create mode 100644 .github/actions/upload/action.yml rename {scripts => .github/actions/upload}/upload.ts (97%) rename deno.jsonc => .github/deno.jsonc (56%) rename {scripts => .github/scripts}/index-packages.ts (89%) rename {scripts => .github/scripts}/map-projects-to-githubs.ts (100%) rename {scripts => .github/scripts}/utils/args.ts (100%) rename {scripts => .github/scripts}/utils/gha.ts (99%) delete mode 100755 scripts/upload-sync.ts create mode 100644 tea.yaml diff --git a/.github/actions/bottle/action.yml b/.github/actions/bottle/action.yml new file mode 100644 index 00000000..a405f3a6 --- /dev/null +++ b/.github/actions/bottle/action.yml @@ -0,0 +1,26 @@ +name: tea/pantry/bottle +description: internal tea.xyz specific at this time + +inputs: + gpg-key-id: + description: '' + required: true + gpg-key-passphrase: + description: '' + required: true + built: + description: '' + required: true + compression: + description: '' + required: true + +runs: + using: composite + steps: + - run: ${{ github.action_path }}/bottle.ts ${{ inputs.built }} + shell: sh + env: + COMPRESSION: ${{ inputs.compression }} + GPG_KEY_ID: ${{ inputs.gpg-key-id }} + GPG_PASSPHRASE: ${{ inputs.gpg-passphrase }} diff --git a/scripts/bottle.ts b/.github/actions/bottle/bottle.ts similarity index 96% rename from scripts/bottle.ts rename to .github/actions/bottle/bottle.ts index 4d82ad95..cbd403f7 100755 --- a/scripts/bottle.ts +++ b/.github/actions/bottle/bottle.ts @@ -22,8 +22,8 @@ import { backticks, panic, run } from "utils" import { crypto } from "deno/crypto/mod.ts" import { encode } from "deno/encoding/hex.ts" import { encode as base64Encode } from "deno/encoding/base64.ts" -import { set_output } from "./utils/gha.ts" -import * as ARGV from "./utils/args.ts" +import { set_output } from "../../scripts/utils/gha.ts" +import * as ARGV from "../../scripts/utils/args.ts" import Path from "path" const cellar = useCellar() diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml new file mode 100644 index 00000000..906549c5 --- /dev/null +++ b/.github/actions/cache/action.yml @@ -0,0 +1,28 @@ +name: tea.xyz/pantry/actions/cache +description: cache deno deps + +inputs: + cache-name: + description: name of the job to use on the cache key + required: true + +runs: + using: composite + steps: + - run: | + if test "$RUNNER_OS" = "macOS"; then + echo "cache=~/Library/Caches/deno" >> $GITHUB_OUTPUT + else + echo "cache=~/.cache/deno" >> $GITHUB_OUTPUT + fi + id: os-cache + shell: sh + + - uses: actions/cache@v3 + with: + path: | + ~/.deno + ${{ steps.os-cache.outputs.cache }} + # This isn't perfect (can't hash stuff outside github.workspace, and if the there scripts change, the hash won't) + # but it's good enough for now. It's slightly conservative, since it monitors all .ts files, but that's fine. + key: ${{ runner.os }}-deno-${{ inputs.cache-name }}-${{ hashFiles('**/deno.jsonc', '**/*.ts') }} diff --git a/.github/actions/apple-signing/action.yml b/.github/actions/codesign/action.yml similarity index 76% rename from .github/actions/apple-signing/action.yml rename to .github/actions/codesign/action.yml index e9b1bfd8..5d31fd62 100644 --- a/.github/actions/apple-signing/action.yml +++ b/.github/actions/codesign/action.yml @@ -1,5 +1,6 @@ -name: Apple signing -description: signs binaries for macOS +name: Apple Codesigning +description: Codesigns macOS binaries + inputs: p12-file-base64: description: Base64 encoded p12 file @@ -11,11 +12,11 @@ inputs: description: Identity to use for signing required: true paths: - description: Paths to search for files to sign + description: paths to sign required: true runs: - using: "composite" + using: composite steps: # Only runs on macOS - name: Check platform @@ -37,13 +38,9 @@ runs: p12-file-base64: ${{ inputs.p12-file-base64 }} p12-password: ${{ inputs.p12-password }} - - name: Codesign files shell: sh - run: | - find $PATHS -type f -print0 | \ - xargs -0 /usr/bin/codesign -s "$IDENTITY" --force -v \ - --timestamp || true + run: find $PATHS -type f -print0 | xargs -0 codesign -s "$IDENTITY" --force -v --timestamp || true env: PATHS: ${{ inputs.paths }} IDENTITY: ${{ inputs.identity }} @@ -51,10 +48,9 @@ runs: # This isn't very informative, but even a no-op is safer than none - name: Check codesigning shell: sh - run: | - # FIXME: `deno` compiled binaries don't currently pass validation. - # https://github.com/denoland/deno/issues/17753 - find $PATHS -type f ! -name tea -print0 | xargs -0 codesign -vvv --strict + # FIXME: `deno` compiled binaries don't currently pass validation. + # https://github.com/denoland/deno/issues/17753 + run: find $PATHS -type f ! -name tea -print0 | xargs -0 codesign -vvv --strict env: PATHS: ${{ inputs.paths }} diff --git a/.github/actions/fetch-pr-artifacts/action.yml b/.github/actions/fetch-pr-artifacts/action.yml new file mode 100644 index 00000000..abfa6a93 --- /dev/null +++ b/.github/actions/fetch-pr-artifacts/action.yml @@ -0,0 +1,35 @@ +name: tea/pantry/fetch-pr-artifacts +description: internal tea.xyz specific at this time + +inputs: + platform: + description: platform+arch to fetch + required: true + token: + description: github token + default: ${{ github.token }} + required: true + AWS_S3_BUCKET: + description: AWS S3 bucket to use for cache + required: true + AWS_ACCESS_KEY_ID: + description: AWS access key id + required: true + AWS_SECRET_ACCESS_KEY: + description: AWS secret access key + required: true + +runs: + using: composite + steps: + - run: + ${{ github.action_path }}/fetch-pr-artifacts.ts + ${{ github.repository }} + ${{ github.sha }} + ${{ inputs.platform }} >>$GITHUB_ENV + shell: sh + env: + GITHUB_TOKEN: ${{ inputs.token }} + AWS_S3_CACHE: ${{ inputs.AWS_S3_CACHE }} + AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} diff --git a/scripts/fetch-pr-artifacts.ts b/.github/actions/fetch-pr-artifacts/fetch-pr-artifacts.ts similarity index 100% rename from scripts/fetch-pr-artifacts.ts rename to .github/actions/fetch-pr-artifacts/fetch-pr-artifacts.ts diff --git a/.github/actions/get-platform/action.yml b/.github/actions/get-platform/action.yml new file mode 100644 index 00000000..bdb499d3 --- /dev/null +++ b/.github/actions/get-platform/action.yml @@ -0,0 +1,39 @@ +name: tea/pantry/get-platform +description: Outputs the platform spec we need for builds + +inputs: + platform: + description: > + The platform+arch to get specs for + required: true + +outputs: + os: + description: the OS for general tasks + value: ${{ steps.platform.outputs.os }} + build-os: + description: the OS for build tasks + value: ${{ steps.platform.outputs.build-os }} + container: + description: the container for build tasks + value: ${{ steps.platform.outputs.container }} + test-matrix: + description: the matrix of os/containers for test tasks + value: ${{ steps.platform.outputs.test-matrix }} + +runs: + using: composite + steps: + - uses: teaxyz/setup@v0 + with: + srcroot: null + + - uses: teaxyz/pantry.core/.github/actions/cache@main + with: + cache-name: get-platform + + - run: ${{github.action_path}}/get-platform.ts + shell: sh + id: platform + env: + PLATFORM: ${{ inputs.platform }} diff --git a/scripts/get-platform.ts b/.github/actions/get-platform/get-platform.ts similarity index 85% rename from scripts/get-platform.ts rename to .github/actions/get-platform/get-platform.ts index dbc1114d..0096212d 100755 --- a/scripts/get-platform.ts +++ b/.github/actions/get-platform/get-platform.ts @@ -25,7 +25,6 @@ type Output = { buildOs: OS, container?: string, testMatrix: { os: OS, container?: string }[] - cacheSet: string } type OS = string | string[] @@ -33,11 +32,6 @@ type OS = string | string[] const platform = Deno.env.get("PLATFORM") ?? panic("$PLATFORM not set") -const cacheSets = { - "darwin": `~/.deno\n~/Library/Caches/deno/deps/https/`, - "linux": `~/.deno\n~/.cache/deno/deps/https/` -} - const output: Output = (() => { switch(platform) { case "darwin+x86-64": { @@ -46,7 +40,6 @@ const output: Output = (() => { os, buildOs: ["self-hosted", "macOS", "X64"], testMatrix: [{ os }], - cacheSet: cacheSets["darwin"] } } case "darwin+aarch64": { @@ -55,7 +48,6 @@ const output: Output = (() => { os, buildOs: os, testMatrix: [{ os }], - cacheSet: cacheSets["darwin"] } } case "linux+aarch64": { @@ -64,7 +56,6 @@ const output: Output = (() => { os, buildOs: os, testMatrix: [{ os }], - cacheSet: cacheSets["linux"] } } case "linux+x86-64": { @@ -80,7 +71,6 @@ const output: Output = (() => { { os, container }, { os, container: "debian:buster-slim" } ], - cacheSet: cacheSets["linux"] } } default: @@ -90,8 +80,7 @@ const output: Output = (() => { const rv = `os=${JSON.stringify(output.os)}\n` + `build-os=${JSON.stringify(output.buildOs)}\n` + `container=${JSON.stringify(output.container)}\n` + - `test-matrix=${JSON.stringify(output.testMatrix)}\n` + - `cache-set< + The platform+arch to get specs for + required: true + +outputs: + os: + description: the OS for general tasks + value: ${{ steps.platform.outputs.os }} + build-os: + description: the OS for build tasks + value: ${{ steps.platform.outputs.build-os }} + container: + description: the container for build tasks + value: ${{ steps.platform.outputs.container }} + test-matrix: + description: the matrix of os/containers for test tasks + value: ${{ steps.platform.outputs.test-matrix }} + +runs: + using: composite + steps: + - uses: teaxyz/setup@v0 + with: + srcroot: null + + - uses: teaxyz/pantry.core/.github/actions/cache@main + with: + cache-name: has-artifacts + + - run: ${{github.action_path}}/get-platform.ts + shell: sh + id: platform + env: + PLATFORM: ${{ inputs.platform }} diff --git a/scripts/has-artifacts.ts b/.github/actions/has-artifacts/has-artifacts.ts similarity index 89% rename from scripts/has-artifacts.ts rename to .github/actions/has-artifacts/has-artifacts.ts index 219f63fb..42ad1c62 100755 --- a/scripts/has-artifacts.ts +++ b/.github/actions/has-artifacts/has-artifacts.ts @@ -11,9 +11,9 @@ args: /// Test /// ./scripts/has-artifacts.ts e582b03fe6efedde80f9569403555f4513dbec91 -import { S3 } from "s3"; -import { panic } from "utils"; -import { find_pr } from "./fetch-pr-artifacts.ts"; +import { S3 } from "s3" +import { panic } from "utils" +import { find_pr } from "../fetch-pr-artifacts/fetch-pr-artifacts.ts" /// Main /// ------------------------------------------------------------------------------- diff --git a/.github/actions/setup-brewkit/action.yml b/.github/actions/setup-brewkit/action.yml new file mode 100644 index 00000000..a66571ab --- /dev/null +++ b/.github/actions/setup-brewkit/action.yml @@ -0,0 +1,39 @@ +name: +tea.xyz/brewkit +description: sets up tea, tea.xyz/brewkit & caching + +# inputs and outputs are `teaxyz/setup` passthrough + +inputs: + prefix: + description: > + Where tea stows its packages. + Defaults to `$HOME/.tea`. + required: false +outputs: + version: + description: Your project’s version. + value: ${{ steps.tea.outputs.version }} + prefix: + description: The prefix you specified. + value: ${{ steps.tea.outputs.prefix }} + +runs: + using: composite + steps: + - uses: teaxyz/setup@v0 + id: tea + with: + prefix: ${{ inputs.prefix }} + +: tea.xyz/brewkit + # prevent pantry from reassigning TEA_PREFIX etc. + srcroot: null + + - uses: teaxyz/pantry.core/.github/actions/cache@main + with: + cache-name: setup + + - run: | + # if test -d "{{ github.workspace }}"/projects; then + echo "TEA_PANTRY_PATH=${{ github.workspace }}" >> $GITHUB_ENV + # fi + shell: sh diff --git a/.github/actions/stage-build-artifacts/action.yml b/.github/actions/stage-build-artifacts/action.yml new file mode 100644 index 00000000..36aff0de --- /dev/null +++ b/.github/actions/stage-build-artifacts/action.yml @@ -0,0 +1,40 @@ +name: tea/pantry/stage-build-artifacts +description: internal tea.xyz specific at this time + +inputs: + platform: + description: '' + required: true + AWS_S3_BUCKET: + description: '' + required: true + AWS_ACCESS_KEY_ID: + description: '' + required: true + AWS_SECRET_ACCESS_KEY: + description: '' + required: true + +runs: + using: composite + steps: + - uses: actions/download-artifact@v3 + with: + name: ${{ inputs.platform }} + + - uses: teaxyz/setup@v0 + + - uses: teaxyz/pantry.core/.github/actions/cache@main + with: + cache-name: stage + + - run: ${{ github.action_path }}/cache-artifacts.ts + ${{github.repository}} + ${{github.ref}} + ${{inputs.platform}} + artifacts.tgz + shell: sh + env: + AWS_S3_BUCKET: ${{ inputs.AWS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} diff --git a/scripts/cache-artifacts.ts b/.github/actions/stage-build-artifacts/cache-artifacts.ts similarity index 95% rename from scripts/cache-artifacts.ts rename to .github/actions/stage-build-artifacts/cache-artifacts.ts index 22ebeba6..b8be79ce 100755 --- a/scripts/cache-artifacts.ts +++ b/.github/actions/stage-build-artifacts/cache-artifacts.ts @@ -34,7 +34,7 @@ const s3 = new S3({ secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, region: "us-east-1", }) -const bucket = s3.getBucket(Deno.env.get("AWS_S3_CACHE")!) +const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!) const key = `pull-request/${repo.split("/")[1]}/${pr}/${dest}` const body = await Deno.readFile(file.string) diff --git a/.github/actions/upload/action.yml b/.github/actions/upload/action.yml new file mode 100644 index 00000000..9022fbfe --- /dev/null +++ b/.github/actions/upload/action.yml @@ -0,0 +1,47 @@ +name: tea/pantry/upload +description: internal tea.xyz specific at this time + +inputs: + pkgs: + description: '' + required: true + srcs: + description: '' + required: true + bottles: + description: '' + required: true + checksums: + description: '' + required: true + signatures: + description: '' + required: true + AWS_S3_BUCKET: + description: '' + required: true + AWS_ACCESS_KEY_ID: + description: '' + required: true + AWS_SECRET_ACCESS_KEY: + description: '' + required: true + +runs: + using: composite + steps: + - uses: teaxyz/pantry.core/.github/actions/cache@main + with: + cache-name: upload + + - run: ${{ github.action_path }}/upload.ts + --pkgs ${{ inputs.pkgs }} + --srcs ${{ inputs.srcs }} + --bottles ${{ inputs.bottles }} + --checksums ${{ inputs.checksums }} + --signatures ${{ inputs.signatures }} + shell: sh + env: + AWS_S3_BUCKET: ${{ inputs.AWS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} diff --git a/scripts/upload.ts b/.github/actions/upload/upload.ts similarity index 97% rename from scripts/upload.ts rename to .github/actions/upload/upload.ts index 621842df..d5f586ec 100755 --- a/scripts/upload.ts +++ b/.github/actions/upload/upload.ts @@ -18,8 +18,8 @@ import { basename, dirname } from "deno/path/mod.ts" import { retry } from "deno/async/retry.ts" import { decode as base64Decode } from "deno/encoding/base64.ts" import Path from "path" -import { set_output } from "./utils/gha.ts" -import { sha256 } from "./bottle.ts" +import { set_output } from "../../scripts/utils/gha.ts" +import { sha256 } from "../bottle/bottle.ts" //------------------------------------------------------------------------- funcs function args_get(key: string): string[] { diff --git a/deno.jsonc b/.github/deno.jsonc similarity index 56% rename from deno.jsonc rename to .github/deno.jsonc index ddecc884..a2fac0a3 100644 --- a/deno.jsonc +++ b/.github/deno.jsonc @@ -16,14 +16,7 @@ }, "tea": { "dependencies": { - "deno.land": "^1.30", - "tea.xyz/brewkit": "^0.3" - }, - "env": { - // if your primary tea.prefix is somewhere else then you’ll - // need to `tea --sync` in this dev-env - "TEA_PANTRY_PATH": "{{srcroot}}:{{home}}/.tea/tea.xyz/var/pantry", - "TEA_PREFIX": "{{srcroot}}/tea.out" + "deno.land": "^1.30" } }, "importMap": "https://raw.githubusercontent.com/teaxyz/cli/v0.23/import-map.json" diff --git a/scripts/index-packages.ts b/.github/scripts/index-packages.ts similarity index 89% rename from scripts/index-packages.ts rename to .github/scripts/index-packages.ts index a9896f15..a4f8d395 100755 --- a/scripts/index-packages.ts +++ b/.github/scripts/index-packages.ts @@ -1,14 +1,4 @@ -#!/usr/bin/env tea - -/*--- -args: - - deno - - run - - --allow-read - - --allow-env - - --allow-net - - --allow-sys ----*/ +#!/usr/bin/env -S tea -E deno run --allow-read --allow-env --allow-net --allow-sys import { usePantry } from "hooks" import * as ARGV from "./utils/args.ts" diff --git a/scripts/map-projects-to-githubs.ts b/.github/scripts/map-projects-to-githubs.ts similarity index 100% rename from scripts/map-projects-to-githubs.ts rename to .github/scripts/map-projects-to-githubs.ts diff --git a/scripts/utils/args.ts b/.github/scripts/utils/args.ts similarity index 100% rename from scripts/utils/args.ts rename to .github/scripts/utils/args.ts diff --git a/scripts/utils/gha.ts b/.github/scripts/utils/gha.ts similarity index 99% rename from scripts/utils/gha.ts rename to .github/scripts/utils/gha.ts index 3f7926d3..e5b22c13 100644 --- a/scripts/utils/gha.ts +++ b/.github/scripts/utils/gha.ts @@ -1,4 +1,3 @@ - const e = new TextEncoder() const encode = e.encode.bind(e) diff --git a/.github/workflows/bottle.yml b/.github/workflows/bottle.yml index 5472ffa7..1786f448 100644 --- a/.github/workflows/bottle.yml +++ b/.github/workflows/bottle.yml @@ -22,20 +22,10 @@ jobs: os: ${{ steps.platform.outputs.os }} cache-set: ${{ steps.platform.outputs.cache-set }} steps: - - uses: actions/checkout@v3 - with: - repository: teaxyz/pantry.core - - uses: actions/cache@v3 - with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-get-platform-${{ hashFiles('deno.jsonc')}} - - uses: teaxyz/setup@v0 - - run: scripts/get-platform.ts + - uses: teaxyz/pantry.core/.github/actions/get-platform@main id: platform - env: - PLATFORM: ${{ inputs.platform }} + with: + platform: ${{ inputs.platform }} bottle: needs: [get-platform] @@ -45,29 +35,17 @@ jobs: built: ${{ env.built }} pr: ${{ env.PR }} steps: - - uses: teaxyz/setup@v0 - id: tea - with: - srcroot: null - prefix: ${{ github.workspace }} - - - uses: actions/cache@v3 - with: - path: ${{ needs.get-platform.outputs.cache-set }} - key: ${{ runner.os }}-deno-bottle-${{ hashFiles('tea.xyz/var/pantry/deno.jsonc')}} - - - name: configure scripts PATH - run: echo "$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH + - uses: teaxyz/pantry.core/.github/actions/setup-brewkit@main - uses: actions/download-artifact@v3 if: ${{ inputs.new-version }} with: name: ${{ inputs.platform }} - - run: fetch-pr-artifacts.ts ${{ github.repository }} ${{ github.sha }} ${{ inputs.platform }} >>$GITHUB_ENV + - uses: teaxyz/pantry.core/.github/actions/fetch-pr-artifacts@main if: ${{ !inputs.new-version }} - env: - GITHUB_TOKEN: ${{github.token}} + with: + token: ${{ github.token }} AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -87,19 +65,21 @@ jobs: env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - - run: bottle.ts $built + - uses: teaxyz/pantry.core/.github/actions/bottle@main id: bottle-xz - env: - COMPRESSION: xz - GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + with: + built: ${{ env.built }} + compresson: xz + gpg-key-id: ${{ secrets.GPG_KEY_ID }} + gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }} - - run: bottle.ts $built + - uses: teaxyz/pantry.core/.github/actions/bottle@main id: bottle-gz - env: - COMPRESSION: gz - GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + with: + built: ${{ env.built }} + compresson: gz + gpg-key-id: ${{ secrets.GPG_KEY_ID }} + gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }} - run: | echo ${{ steps.bottle-gz.outputs.bottles }} ${{ steps.bottle-xz.outputs.bottles }} >bottles @@ -125,22 +105,6 @@ jobs: needs: [bottle] runs-on: ubuntu-latest steps: - - uses: teaxyz/setup@v0 - id: tea - with: - srcroot: null - prefix: ${{ github.workspace }} - - - name: configure scripts PATH - run: echo "${{ steps.tea.outputs.prefix }}/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH - - - uses: actions/cache@v3 - with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-bottle-${{ hashFiles('tea.xyz/var/pantry/deno.jsonc')}} - - uses: actions/download-artifact@v3 with: name: ${{ inputs.platform }}-bottles @@ -152,15 +116,13 @@ jobs: echo "$file=$(cat $file)" >>$GITHUB_ENV done - - name: upload bottles - id: upload - run: upload.ts - --pkgs ${{ needs.bottle.outputs.built }} ${{ needs.bottle.outputs.built }} - --srcs ${{ needs.bottle.outputs.srcs }} ${{ needs.bottle.outputs.srcs }} - --bottles $bottles - --checksums $checksums - --signatures $signatures - env: + - uses: teaxyz/pantry.core/.github/actions/upload@main + with: + pkgs: ${{ needs.bottle.outputs.built }} ${{ needs.bottle.outputs.built }} + srcs: ${{ needs.bottle.outputs.srcs }} ${{ needs.bottle.outputs.srcs }} + bottles: ${{ env.bottles }} + checksums: ${{ env.checksums }} + signatures: ${{ env.signatures }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd956a8a..35bbd238 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,21 +24,10 @@ jobs: test-matrix: ${{ steps.platform.outputs.test-matrix }} cache-set: ${{ steps.platform.outputs.cache-set }} steps: - - uses: actions/checkout@v3 - with: - repository: teaxyz/pantry.core - - uses: actions/cache@v3 - with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-get-platform-${{ hashFiles('deno.jsonc')}} - - uses: teaxyz/setup@v0 - - run: scripts/get-platform.ts ${{ inputs.projects }} + - uses: teaxyz/pantry.core/.github/actions/get-platform@main id: platform - env: - PLATFORM: ${{ inputs.platform }} - TEA_PANTRY_PATH: ${{ github.workspace }} + with: + platform: ${{ inputs.platform }} build: runs-on: ${{ fromJson(needs.get-platform.outputs.build-os) }} @@ -46,27 +35,17 @@ jobs: needs: [get-platform] steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: ${{ needs.get-platform.outputs.cache-set }} - key: ${{ runner.os }}-deno-build-${{ hashFiles('deno.jsonc')}} - - uses: teaxyz/setup@v0 + - uses: teaxyz/pantry.core/.github/actions/setup-brewkit@main id: tea with: - srcroot: null prefix: /opt - name: sanitize macOS runners if: fromJson(needs.get-platform.outputs.build-os) == 'macos-11' run: sudo mv /usr/local/bin/* /tmp/ - - name: configure tea env - run: | - echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH - echo "TEA_PANTRY_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - - run: tea +tea.xyz/brewkit build ${{ inputs.projects }} + - run: pkg build ${{ inputs.projects }} id: build env: GITHUB_TOKEN: ${{ github.token }} @@ -82,7 +61,7 @@ jobs: TEA_PREFIX: ${{ steps.tea.outputs.prefix }} # sign macOS binaries - - uses: teaxyz/pantry.core/.github/actions/apple-signing@main + - uses: teaxyz/pantry.core/.github/actions/codesign@main if: startsWith(inputs.platform, 'darwin+') && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name with: p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} @@ -118,7 +97,6 @@ jobs: needs: [get-platform, build] runs-on: ${{ matrix.platform.os }} strategy: - fail-fast: false matrix: platform: ${{ fromJson(needs.get-platform.outputs.test-matrix) }} outputs: @@ -126,11 +104,7 @@ jobs: container: ${{ matrix.platform.container }} steps: - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ${{ needs.get-platform.outputs.cache-set }} - key: ${{ runner.os }}-deno-test-${{ hashFiles('deno.jsonc')}} + - uses: teaxyz/pantry.core/.github/actions/setup-brewkit@main - uses: teaxyz/setup@v0 with: @@ -143,12 +117,11 @@ jobs: - name: extract bottles run: tar xzvf artifacts.tgz -C $TEA_PREFIX - - run: tea +tea.xyz/brewkit test ${{ inputs.projects }} + - run: pkg test ${{ inputs.projects }} env: - TEA_PANTRY_PATH: ${{ github.workspace }} - GITHUB_TOKEN: ${{github.token}} + GITHUB_TOKEN: ${{ github.token }} - - name: post + - name: '[post]' run: echo "HAS_SECRETS=$HAS_SECRETS" >>$GITHUB_ENV env: @@ -178,28 +151,13 @@ jobs: if: startsWith(github.ref, 'refs/pull/') && startsWith(github.repository, 'teaxyz/pantry.') && needs.test.outputs.HAS_SECRETS == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-stage-${{ hashFiles('deno.jsonc')}} - - uses: teaxyz/setup@v0 - - - name: configure scripts PATH - run: echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH - - uses: actions/download-artifact@v3 with: name: ${{ inputs.platform }} - - run: cache-artifacts.ts - ${{github.repository}} - ${{github.ref}} - ${{inputs.platform}} - artifacts.tgz - env: - AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} + - uses: teaxyz/pantry.core/.github/actions/stage-build-artifacts@main + with: + platform: ${{ inputs.platform }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_CACHE }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index c2ae5490..3ce32492 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -5,23 +5,22 @@ on: branches: [main] jobs: - cd: # job is named poorly, all I can tell you about it is: it’s *not* CD + cd: runs-on: ubuntu-latest outputs: HAS_ARTIFACTS: ${{ env.HAS_ARTIFACTS }} steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-cd-${{ hashFiles('deno.jsonc')}} - - uses: teaxyz/setup@v0 + repo: teaxyz/pantry.core + + - uses: teaxyz/pantry.core/.github/actions/setup-brewkit@main + - run: scripts/map-projects-to-githubs.ts env: WATCHER_URL: ${{ secrets.WATCHER_URL }} TEA_API_TOKEN: ${{ secrets.TEA_API_TOKEN }} + - run: scripts/has-artifacts.ts ${{ github.repository }} ${{ github.sha }} >>$GITHUB_ENV env: GITHUB_TOKEN: ${{github.token}} diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml index 2d08f2df..06f78544 100644 --- a/.github/workflows/ci-scripts.yml +++ b/.github/workflows/ci-scripts.yml @@ -3,14 +3,18 @@ name: ci·scripts on: pull_request: paths: - - scripts/**/*.ts + - .github/**.ts jobs: typecheck: runs-on: ubuntu-latest - env: - TEA_SECRET: ${{ secrets.TEA_SECRET }} steps: - uses: actions/checkout@v3 - uses: teaxyz/setup@v0 - - run: deno check --unstable scripts/*.ts + with: + srcroot: .github + - uses: teaxyz/pantry.core/.github/actions/cache@main + with: + cache-name: ci-scripts + - run: deno check --unstable **/*.ts + working-directory: .github diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2368627a..68619405 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: RESULT="$RESULT $y" done echo "diff=$RESULT" >> $GITHUB_OUTPUT - build: + ci: strategy: fail-fast: false matrix: diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index c178a119..3597def2 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -1,4 +1,7 @@ -name: cleanup +# cleans up our S3 staging area if a PR is closed without merge + +name: teaxyz s3 cleanup + on: pull_request: types: [closed] @@ -14,6 +17,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 + - name: remove staged artifacts run: | REPO=$(echo ${{github.repository}} | sed -e 's_teaxyz/__') diff --git a/.github/workflows/complain.yml b/.github/workflows/complain.yml index c5f79f42..3caa57ae 100644 --- a/.github/workflows/complain.yml +++ b/.github/workflows/complain.yml @@ -26,8 +26,6 @@ jobs: SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} SLACK_MESSAGE: new-version:${{ inputs.projects }} (${{ inputs.platform }}) ${{ steps.status.outputs.status }} SLACK_COLOR: ${{ steps.status.outputs.status }} - - run: env | grep ^GITHUB - - run: echo "${{ github.ref_name == 'main'}}" - uses: actions/checkout@v3 if: github.ref_name == 'main' - uses: JasonEtco/create-an-issue@v2 diff --git a/.github/workflows/index-data.yml b/.github/workflows/index-data.yml index 27306507..badfac85 100644 --- a/.github/workflows/index-data.yml +++ b/.github/workflows/index-data.yml @@ -12,13 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-queue-detail-ingestion-${{ hashFiles('deno.jsonc')}} - uses: teaxyz/setup@v0 + with: + srcroot: null + - uses: teaxyz/pantry.core/.github/actions/cache@main - run: ./scripts/index-packages.ts ${{ inputs.projects }} env: TEA_PANTRY_PATH: ${{ github.workspace }} diff --git a/.github/workflows/new-version.yml b/.github/workflows/new-version.yml index 059d767e..78a097be 100644 --- a/.github/workflows/new-version.yml +++ b/.github/workflows/new-version.yml @@ -8,7 +8,6 @@ on: required: true type: string - jobs: build: strategy: @@ -25,6 +24,7 @@ jobs: projects: ${{ inputs.projects }} platform: ${{ matrix.platform }} secrets: inherit + index_data: needs: [build] if: success() diff --git a/.vscode/settings.json b/.vscode/settings.json index a25540ed..89c517a3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,6 @@ "deno.enable": true, "deno.lint": true, "deno.unstable": true, - "deno.config": "deno.jsonc", + "deno.config": ".github/deno.jsonc", "deno.importMap": "../cli/import-map.json" } \ No newline at end of file diff --git a/scripts/upload-sync.ts b/scripts/upload-sync.ts deleted file mode 100755 index 4cc4e217..00000000 --- a/scripts/upload-sync.ts +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env tea - -/*--- -args: - - deno - - run - - --allow-read - - --allow-net - - --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_S3_BUCKET,TEA_PREFIX ----*/ - -import { readAll, readerFromStreamReader } from "deno/streams/mod.ts" -import { useCache, useOffLicense } from "hooks" -import { Package } from "types" -import { crypto, toHashString } from "deno/crypto/mod.ts"; -import { S3 } from "s3" -import Path from "path" - -const s3 = new S3({ - accessKeyID: Deno.env.get("AWS_ACCESS_KEY_ID")!, - secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - region: "us-east-1", -}) - -const offy = useOffLicense('s3') -const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!) - -for (const stowed of await useCache().ls()) { - const url = offy.url(stowed) - const key = offy.key(stowed) - - console.log({ checking: url }) - - const inRepo = await bucket.headObject(key) - const repoChecksum = inRepo ? await checksum(`${url}.sha256sum`) : undefined - - // path.read() returns a string; this is easier to get a UInt8Array - const contents = await Deno.readFile(stowed.path.string) - const sha256sum = toHashString(await crypto.subtle.digest("SHA-256", contents)) - - if (!inRepo || repoChecksum !== sha256sum) { - const basename = url.path().basename() - const body = new TextEncoder().encode(`${sha256sum} ${basename}`) - - console.log({ uploading: url }) - - await bucket.putObject(key, contents) - await bucket.putObject(`${key}.sha256sum`, body) - - console.log({ uploaded: url }) - } -} - -async function checksum(url: string) { - const rsp = await fetch(url) - if (!rsp.ok) throw new Error(`404-not-found: ${url}`) - const rdr = rsp.body?.getReader() - if (!rdr) throw new Error(`Couldn’t read: ${url}`) - const r = await readAll(readerFromStreamReader(rdr)) - return new TextDecoder().decode(r).split(' ')[0] -} - -type RV = Package & {bottle: Path} diff --git a/tea.yaml b/tea.yaml new file mode 100644 index 00000000..e60dab06 --- /dev/null +++ b/tea.yaml @@ -0,0 +1,5 @@ +dependencies: + tea.xyz/brewkit: ^0.3 +env: + TEA_PANTRY_PATH: ${{srcroot}}:{{home}}/.tea/tea.xyz/var/pantry + TEA_PREFIX: ${{srcroot}}/tea.out