diff --git a/.github/actions/complain/action.yml b/.github/actions/complain/action.yml index 9feaacd9..3e7a4554 100644 --- a/.github/actions/complain/action.yml +++ b/.github/actions/complain/action.yml @@ -2,8 +2,8 @@ name: pkgx/pantry/complain description: creates an issue for failure conditions inputs: - projects: - description: projects to complain about + pkg: + description: project to complain about required: true platform: description: platform key @@ -12,12 +12,6 @@ inputs: description: github token required: true default: ${{ github.token }} - # slack-webhook: - # description: slack webhook - # required: false - # slack-channel: - # description: slack channel - # required: false runs: using: composite @@ -29,7 +23,7 @@ runs: actions: 'find-issues' token: ${{ inputs.token }} issue-state: 'open' - title-includes: "❌ build issues: ${{ inputs.projects }}" + title-includes: "❌ build issues: ${{ inputs.pkg }}" labels: 'build-failure' - name: Create Issue @@ -39,8 +33,8 @@ runs: with: actions: 'create-issue' token: ${{ inputs.token }} - title: "❌ build issues: ${{ inputs.projects }}" - body: "Running log of build failures for ${{ inputs.projects }}" + title: "❌ build issues: ${{ inputs.pkg }}" + body: "Running log of build failure for ${{ inputs.pkg }}" labels: 'build-failure' assignees: 'jhheider' @@ -52,18 +46,7 @@ runs: issue-number: ${{ steps.create.outputs.issue-number || fromJSON(steps.find.outputs.issues)[0].number }} body: | # Build failure - ## ${{ inputs.projects }} + ## ${{ inputs.pkg }} ### ${{ inputs.platform }} logs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - # - uses: martialonline/workflow-status@v3 - # id: status - - # - uses: rtCamp/action-slack-notify@v2 - # if: ${{ inputs.slack-webhook != '' }} - # env: - # SLACK_WEBHOOK: ${{ inputs.slack-webhook }} - # SLACK_CHANNEL: ${{ inputs.slack-channel }} - # SLACK_MESSAGE: new-version:${{ inputs.projects }} (${{ inputs.platform }}) ${{ steps.status.outputs.status }} - # SLACK_COLOR: ${{ steps.status.outputs.status }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..3b08e5a0 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,54 @@ +name: pkgx/brewkit/setup-codesign +description: Codesigns macOS binaries using Apple tools + +inputs: + p12-file-base64: + description: Base64 encoded p12 file + required: true + p12-password: + description: Password for p12 file + required: true + APPLE_IDENTITY: + required: false + +runs: + using: composite + steps: + # - name: purge tool PATH + # run: | + # if [ -d /usr/local/bin ]; then + # tmp=$(mktemp -d) + # sudo mv /usr/local/bin $tmp + # fi + # shell: bash + + - name: export APPLE_IDENTITY + run: echo 'APPLE_IDENTITY=${{inputs.identity || '-'}}' >> $GITHUB_ENV + shell: bash + + # the next three steps bless our code for Apple. It might be the case they should be + # encapulated separately. + # FIXME: using an explicit commit in a PR isn't great, but the last release was almost 3 years + # ago, and we need bugfixes. + # FIXME: replace this with a pkgx script based on https://localazy.com/blog/how-to-automatically-sign-macos-apps-using-github-actions + # github has a doc with similar content, but it's not returning to me atm. + + # apple-actions/import-codesign-certs will fail if the keychain already exists, so we prophylactically + # delete it if it does. + - name: Delete keychain + shell: sh + if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64 + run: security delete-keychain signing_temp.keychain || true + + - uses: apple-actions/import-codesign-certs@v2 + if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64 + with: + p12-file-base64: ${{ inputs.p12-file-base64 }} + p12-password: ${{ inputs.p12-password }} + + # Needed for self-hosted runner, since it doesn't destroy itself automatically. + - name: Delete keychain + uses: webiny/action-post-run@3.0.0 + if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64 + with: + run: security delete-keychain signing_temp.keychain diff --git a/.github/deno.jsonc b/.github/deno.jsonc index 2b6ddf6a..f732867b 100644 --- a/.github/deno.jsonc +++ b/.github/deno.jsonc @@ -10,6 +10,7 @@ }, "imports": { "pkgx": "https://deno.land/x/libpkgx@v0.15.1/mod.ts", - "pkgx/": "https://deno.land/x/libpkgx@v0.15.1/src/" + "pkgx/": "https://deno.land/x/libpkgx@v0.15.1/src/", + "is-what": "https://deno.land/x/is_what@v4.1.15/src/index.ts" } } diff --git a/.github/scripts/get-matrix.ts b/.github/scripts/get-matrix.ts new file mode 100755 index 00000000..5bcf3a8d --- /dev/null +++ b/.github/scripts/get-matrix.ts @@ -0,0 +1,86 @@ +#!/usr/bin/env -S pkgx deno run -A + +import { hooks, utils } from "pkgx" +import { isString, isArray } from "is-what" + +const pkg = utils.pkg.parse(Deno.args[0]) +const config = await get_config(pkg) + +const rv = {} as Record +for (const platform of config.platforms) { + const key = platform.replace('/', '+') + rv[key] = get_matrix(platform) +} + +const ghout = Deno.env.get("GITHUB_OUTPUT") +if (ghout) { + const json = JSON.stringify(Object.values(rv)) + Deno.writeTextFileSync(ghout, `matrix=${json}`, {append: true}) +} else { + const json = JSON.stringify(rv, null, 2) + console.log(json) +} + +/////////////////////////////////////////////////////////////////////// + +//TODO should be in libpkgx! +async function get_config(pkg: {project: string}) { + let { platforms, test } = await hooks.usePantry().project(pkg).yaml() + const get_platforms = (() => { + if (!platforms) return ["linux/x86-64", "linux/aarch64", "darwin/x86-64", "darwin/aarch64"] + if (isString(platforms)) platforms = [platforms] + if (!isArray(platforms)) throw new Error(`invalid platform node: ${platforms}`) + const rv = [] + for (const platform of platforms) { + if (platform.match(/^(linux|darwin)\/(aarch64|x86-64)$/)) rv.push(platform) + else if (platform.match(/^(linux|darwin)$/)) rv.push(`${platform}/x86-64`, `${platform}/aarch64`) + else throw new Error(`invalid platform: ${platform}`) + } + return rv + }) + + const qaRequired = test?.["qa-required"] === true + + return { + platforms: get_platforms(), + qaRequired + } +} + +function get_matrix(platform: string) { + const name = platform.replace('/', '+') + switch (platform) { + case 'darwin/aarch64': { + const os = ["self-hosted", "macOS", "ARM64"] + return { + os, name, + "test-os": [os], + "test-container": [null], + tinyname: "²" + }} + case 'darwin/x86-64': { + const os = ["self-hosted", "macOS", "X64"] + return { + os, name, + "test-os": ["macos-11", "macos-12"], + "test-container": [null], + tinyname: "x64" + }} + case 'linux/x86-64': { + const os = {group: "linux-x86-64"} + return { + os, name, + container: "debian:buster-slim", + "test-os": [os], + "test-container": ["debian:buster-slim", "ubuntu", "archlinux"], + tinyname: "Lnx·x64" + }} + case 'linux/aarch64': { + const os = ["self-hosted", "linux", "ARM64"] + return { + os, name, + "test-os": [os], + "test-container": [null], + tinyname: "Lnx·ARM64" + }}} +} diff --git a/.github/scripts/qa-required.ts b/.github/scripts/qa-required.ts new file mode 100755 index 00000000..cfc04ddb --- /dev/null +++ b/.github/scripts/qa-required.ts @@ -0,0 +1,9 @@ +#!/usr/bin/env -S pkgx deno run --allow-read + +import { hooks } from "pkgx" + +const project = Deno.args[0] + +const yml = await hooks.usePantry().project(project).yaml() +const qaRequired = yml?.["test"]?.["qa-required"] === true +Deno.exit(qaRequired ? 0 : 1) diff --git a/.github/scripts/utils/args.ts b/.github/scripts/utils/args.ts deleted file mode 100644 index 8bcb12e6..00000000 --- a/.github/scripts/utils/args.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Installation, Package, PackageRequirement, hooks, utils } from "pkgx" -const { useCellar } = hooks - -/// processes Deno.args unless STDIN is not a TTY and has input -export async function *args(): AsyncGenerator { - if (Deno.isatty(Deno.stdin.rid)) { - for (const arg of Deno.args) { - if (arg[0] != '-') yield arg - } - } else { - let yielded_something = false - const buf = new Uint8Array(10) - const decode = (() => { const d = new TextDecoder(); return d.decode.bind(d) })() - let n: number | null - let txt = '' - const rx = /\s*(.*?)\s+/ - while ((n = await Deno.stdin.read(buf)) !== null) { - txt += decode(buf.subarray(0, n)) - while (true) { - const match = txt.match(rx) - if (!match) break - yield match[1] - txt = txt.slice(match[0].length) - yielded_something = true - } - } - if (txt) { - yield txt - } else if (!yielded_something) { - for (const arg of Deno.args) { - yield arg - } - } - } -} - -export async function *pkgs(): AsyncGenerator { - for await (const arg of args()) { - const match = arg.match(/projects\/(.*)\/package.yml/) - const project = match ? match[1] : arg - yield utils.pkg.parse(project) - } -} - -export async function *installs(): AsyncGenerator { - const cellar = useCellar() - for await (const pkg of pkgs()) { - yield await cellar.resolve(pkg) - } -} - -export async function toArray(input: AsyncGenerator) { - const rv: T[] = [] - for await (const i of input) { - rv.push(i) - } - return rv -} diff --git a/.github/workflows/bottle.yml b/.github/workflows/bottle.yml deleted file mode 100644 index 8dedfa45..00000000 --- a/.github/workflows/bottle.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: bottle - -on: - workflow_call: - inputs: - new-version: - type: boolean - required: false - default: false - platform: - required: true - type: string - projects: - required: false - type: string - outputs: - pr: - description: "The PR number" - value: ${{ jobs.bottle.outputs.pr }} - qa-required: - description: "Whether QA is required" - value: ${{ jobs.upload.outputs.qa-required }} - -jobs: - get-platform: - runs-on: ubuntu-latest - outputs: - os: ${{ steps.platform.outputs.os }} - cache-set: ${{ steps.platform.outputs.cache-set }} - available: ${{ steps.platform.outputs.available }} - steps: - - uses: pkgxdev/brewkit/actions/get-platform@v0 - id: platform - with: - platform: ${{ inputs.platform }} - projects: ${{ inputs.projects }} - - bottle: - needs: [get-platform] - if: ${{ !inputs.new-version || needs.get-platform.outputs.available != '' }} - # runs-on: ubuntu-latest - runs-on: ${{ fromJson(needs.get-platform.outputs.os) }} - outputs: - srcs: ${{ env.srcs }} - built: ${{ env.built }} - pr: ${{ env.PR }} - steps: - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - id: pkgx - timeout-minutes: 10 - with: - prefix: ${{ github.workspace }}/.pkgx - pkgs: gnupg.org - - - uses: actions/download-artifact@v3 - if: ${{ inputs.new-version }} - with: - name: ${{ inputs.platform }} - - - uses: pkgxdev/brewkit/actions/fetch-pr-artifacts@v0 - if: ${{ !inputs.new-version }} - with: - platform: ${{ inputs.platform }} - token: ${{ github.token }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_CACHE }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: clean destination - # Note: needed when changing a directory to a symlink, for example in - # https://github.com/pkgxdev/pantry/pull/435 - run: | - tar tzf $GITHUB_WORKSPACE/artifacts.tgz | \ - awk '{ print length, $0 }' | \ - sort -n -s -r | \ - cut -d" " -f2- | \ - xargs rm -rf - working-directory: ${{ github.workspace }}/.pkgx - - - run: tar xzvf $GITHUB_WORKSPACE/artifacts.tgz - working-directory: ${{ github.workspace }}/.pkgx - - - run: | - for file in built srcs; do - echo "$file=$(cat $file)" >> $GITHUB_ENV - done - working-directory: ${{ github.workspace }}/.pkgx - - - run: echo $GPG_PRIVATE_KEY | - base64 -d | - pkgx gpg --import --batch --yes - env: - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - - - uses: pkgxdev/brewkit/actions/bottle@v0 - id: bottle-xz - with: - built: ${{ env.built }} - platform: ${{ inputs.platform }} - compression: xz - gpg-key-id: ${{ secrets.GPG_KEY_ID }} - env: - XDG_CACHE_HOME: ${{ github.workspace }}/.pkgx - - - uses: pkgxdev/brewkit/actions/bottle@v0 - id: bottle-gz - with: - built: ${{ env.built }} - platform: ${{ inputs.platform }} - compression: gz - gpg-key-id: ${{ secrets.GPG_KEY_ID }} - env: - XDG_CACHE_HOME: ${{ github.workspace }}/.pkgx - - - run: | - echo ${{ steps.bottle-gz.outputs.bottles }} ${{ steps.bottle-xz.outputs.bottles }} >bottles - echo ${{ steps.bottle-gz.outputs.checksums }} ${{ steps.bottle-xz.outputs.checksums }} >checksums - echo ${{ steps.bottle-gz.outputs.signatures }} ${{ steps.bottle-xz.outputs.signatures }} >signatures - - tar cf $GITHUB_WORKSPACE/artifacts.tar \ - ${{ steps.bottle-gz.outputs.bottles }} \ - ${{ steps.bottle-xz.outputs.bottles }} \ - bottles checksums signatures - working-directory: ${{ github.workspace }}/.pkgx - - - name: upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.platform }}-bottles - path: artifacts.tar - if-no-files-found: error - - upload: - needs: [get-platform, bottle] - if: ${{ !inputs.new-version || needs.get-platform.outputs.available != '' }} - runs-on: ubuntu-latest - outputs: - qa-required: ${{ steps.upload.outputs.qa-required }} - steps: - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - with: - prefix: ${{ github.workspace }}/.pkgx - timeout-minutes: 10 - - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.platform }}-bottles - - - run: | - tar xvf artifacts.tar - - for file in bottles checksums signatures; do - echo "$file=$(cat $file)" >>$GITHUB_ENV - done - - - uses: pkgxdev/brewkit/actions/upload@v0 - id: upload - with: - qa: ${{ inputs.new-version }} - pkgs: ${{ needs.bottle.outputs.built }} ${{ needs.bottle.outputs.built }} - srcs: "~" - bottles: ${{ env.bottles }} - checksums: ${{ env.checksums }} - signatures: ${{ env.signatures }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - AWS_S3_STAGING_BUCKET: ${{ secrets.AWS_S3_CACHE }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - uses: chetan/invalidate-cloudfront-action@v2 - if: ${{ steps.upload.outputs.cf-invalidation-paths != '' }} - env: - PATHS: ${{ steps.upload.outputs.cf-invalidation-paths }} - DISTRIBUTION: ${{ secrets.AWS_CF_DISTRIBUTION_ID }} - AWS_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 04098ae9..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,161 +0,0 @@ -name: build - -on: - workflow_call: - inputs: - projects: - required: true - type: string - platform: - required: true - type: string - -jobs: - get-platform: - runs-on: ubuntu-latest - outputs: - os: ${{ steps.platform.outputs.os }} - build-os: ${{ steps.platform.outputs.build-os }} - container: ${{ steps.platform.outputs.container }} - test-matrix: ${{ steps.platform.outputs.test-matrix }} - cache-set: ${{ steps.platform.outputs.cache-set }} - available: ${{ steps.platform.outputs.available }} - steps: - - uses: pkgxdev/brewkit/actions/get-platform@v0 - id: platform - with: - platform: ${{ inputs.platform }} - projects: ${{ inputs.projects }} - - build: - runs-on: ${{ fromJson(needs.get-platform.outputs.build-os) }} - container: ${{ fromJson(needs.get-platform.outputs.container) }} - needs: [get-platform] - if: ${{ needs.get-platform.outputs.available != '' }} - steps: - - uses: actions/checkout@v4 - - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - id: pkgx - with: - prefix: /opt - timeout-minutes: 10 - - - name: sanitize macOS runners - if: fromJson(needs.get-platform.outputs.build-os) == 'macos-11' - run: sudo mv /usr/local/bin/* /tmp/ - - # setup macOS codesigning - - uses: pkgxdev/brewkit/actions/setup-codesign@v0 - if: startsWith(inputs.platform, 'darwin+') && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name - with: - p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} - p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} - - # FIXME: this shouldn't be necessary, but it currently is for the - # gha+container build matrix entries. :/ - - name: set srcroot - run: echo "SRCROOT=$GITHUB_WORKSPACE" >>$GITHUB_ENV - - - run: pkg build ${{ needs.get-platform.outputs.available }} - id: build - env: - GITHUB_TOKEN: ${{ github.token }} - FORCE_UNSAFE_CONFIGURE: 1 # some configure scripts refuse to run as root - APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY || '-' }} - - - run: | - ABS_PATHS=$(echo $PATHS | tr ' ' '\n' | sed -e "s_^_/opt/_" | tr '\n' ' ') - echo "paths=$ABS_PATHS" >> $GITHUB_OUTPUT - if: startsWith(inputs.platform, 'darwin+') - id: absolute-paths - env: - PATHS: ${{ steps.build.outputs.relative-paths }} - - # cache data we'll need in the bottling job - - name: assemble artifact metadata - run: | - echo ${{ steps.build.outputs.pkgs }} >built - echo ${{ steps.build.outputs.srcs-relative-paths }} >srcs - working-directory: /opt - - # tarring ourselves ∵ GHA-artifacts (ludicrously) lose permissions - # /ref https://github.com/actions/upload-artifact/issues/38 - - name: create artifacts.tgz - run: tar czvf $GITHUB_WORKSPACE/artifacts.tgz - ${{ steps.build.outputs.relative-paths }} - ${{ steps.build.outputs.srcs-relative-paths }} - built srcs - working-directory: /opt - - - name: upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.platform }} - path: artifacts.tgz - if-no-files-found: error - - test: - needs: [get-platform, build] - if: ${{ needs.get-platform.outputs.available != '' }} - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: ${{ fromJson(needs.get-platform.outputs.test-matrix) }} - name: test ${{ matrix.platform.name-extra }} - outputs: - HAS_SECRETS: ${{ env.HAS_SECRETS }} - container: ${{ matrix.platform.container }} - steps: - - uses: actions/checkout@v4 - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - timeout-minutes: 10 - - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.platform }} - - - name: clean destination - # Note: needed when changing a directory to a symlink, for example in - # https://github.com/pkgxdev/pantry/pull/435 - run: | - cd $HOME/.pkgx - tar tzf $GITHUB_WORKSPACE/artifacts.tgz | \ - awk '{ print length, $0 }' | \ - sort -n -s -r | \ - cut -d" " -f2- | \ - xargs rm -rf - - - name: extract bottles - run: tar xzvf artifacts.tgz -C $HOME/.pkgx - - # FIXME: this shouldn't be necessary, but it currently is for the - # ubuntu+container test matrix entries. :/ - - name: set pantry path - run: echo "PKGX_PANTRY_PATH=$GITHUB_WORKSPACE" >>$GITHUB_ENV - - - run: pkg test ${{ needs.get-platform.outputs.available }} - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: "[post]" - run: echo "HAS_SECRETS=$HAS_SECRETS" >>$GITHUB_ENV - env: - HAS_SECRETS: ${{ secrets.AWS_S3_CACHE != null }} - - stage: - needs: [get-platform, test] - # this only works for PRs from our team to our repo (security! :( ) - if: startsWith(github.ref, 'refs/pull/') && github.repository_owner == 'pkgxdev' && needs.test.outputs.HAS_SECRETS == 'true' && needs.get-platform.outputs.available != '' - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.platform }} - - - uses: pkgxdev/brewkit/actions/stage-build-artifacts@v0 - with: - platform: ${{ inputs.platform }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_CACHE }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 42ac0f3a..116a01f3 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,79 +3,20 @@ run-name: 'cd: ${{ github.event.head_commit.message }}' on: push: - branches: main - paths: projects/**/* + branches: main + paths: projects/**/* + +concurrency: + group: cd/${{ github.ref }} + cancel-in-progress: true jobs: - cd: + get-projects: runs-on: ubuntu-latest outputs: - has-artifacts: ${{ steps.has-artifacts.outputs.has-artifacts }} - platforms: ${{ steps.has-artifacts.outputs.platforms }} + projects: ${{ steps.diff.outputs.diff }} steps: - uses: actions/checkout@v4 - # ^^ NOTE probably no longer required but I don’t dare try to remove it - - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - timeout-minutes: 10 - - - uses: pkgxdev/brewkit/actions/has-artifacts@v0 - id: has-artifacts - with: - repo: ${{ github.repository }} - sha: ${{ github.sha }} - token: ${{github.token}} - s3-bucket: ${{ secrets.AWS_S3_CACHE }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - bottle-pr: - strategy: - fail-fast: false - matrix: - platform: ${{ fromJson(needs.cd.outputs.platforms) }} - needs: [cd] - if: ${{ needs.cd.outputs.has-artifacts == 'true' }} - uses: ./.github/workflows/bottle.yml - with: - platform: ${{ matrix.platform }} - secrets: inherit - - cleanup: - needs: [bottle-pr] - runs-on: ubuntu-latest - if: ${{ needs.cd.outputs.has-artifacts == 'true' }} - env: - PR: ${{ needs.bottle.outputs.pr }} - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - run: | - REPO=$(echo ${{github.repository}} | sed -e 's_pkgxdev/__') - - if test -z "$PR"; then - echo "no PR to clean up" - exit 0 - fi - - aws s3 rm --recursive s3://$AWS_S3_CACHE/pull-request/$REPO/$PR - env: - AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} - PR: ${{ needs.bottle.outputs.pr }} - - bottle-standalone: - runs-on: ubuntu-latest - needs: [cd] - permissions: - contents: read - actions: write - if: ${{ needs.cd.outputs.has-artifacts == 'false' }} - steps: - - uses: actions/checkout@v3 - uses: technote-space/get-diff-action@v6 id: get-diff with: @@ -87,8 +28,12 @@ jobs: RESULT="$RESULT $y" done echo "diff=$RESULT" >> $GITHUB_OUTPUT - - run: gh workflow run new-version.yml -f "projects=$PROJECTS" - if: ${{ steps.diff.outputs.diff != '' }} - env: - GH_TOKEN: ${{ github.token }} - PROJECTS: ${{ steps.diff.outputs.diff }} + + pkgit: + strategy: + fail-fast: false + needs: get-projects + uses: ./.github/workflows/new-version.yml + with: + projects: ${{ needs.get-projects.outputs.diff }} + secrets: inherit diff --git a/.github/workflows/ci-squared.yml b/.github/workflows/ci-squared.yml new file mode 100644 index 00000000..2336e28f --- /dev/null +++ b/.github/workflows/ci-squared.yml @@ -0,0 +1,28 @@ +name: ci-squared +run-name: ci² + +on: + pull_request: + paths: + - .github/workflows/pkg.yml + - .github/workflows/pkg-for-platform.yml + - .github/workflows/ci-squared.yml + +concurrency: + group: pulls/${{ github.ref }} + cancel-in-progress: true + +jobs: + ci: + name: ci² + uses: ./.github/workflows/pkg.yml + strategy: + fail-fast: false + matrix: + pkg: + - r-wos.org/gti + - github.com/ggerganov/llama.cpp # has platform restrictions + with: + pkg: ${{ matrix.pkg }} + dry-run: true + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bbd18c8..70dec217 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,14 +8,14 @@ on: - .github/workflows/ci.yml concurrency: - group: ${{ github.event.pull_request.head.ref }} + group: ci/${{ github.event.pull_request.head.ref }} cancel-in-progress: true jobs: - get-projects: + diff: runs-on: ubuntu-latest outputs: - diff: ${{ steps.diff.outputs.diff }} + pkgs: ${{ steps.diff.outputs.pkgs }} steps: - uses: actions/checkout@v3 - uses: technote-space/get-diff-action@v6 @@ -25,23 +25,61 @@ jobs: - id: diff run: | for x in ${{ steps.get-diff.outputs.diff }}; do - y=$(echo $x | sed 's#projects/\(.*\)/package.yml#\1#') + y=$(echo $x | sed 's#projects/\(.*\)/[^/]*#\1#') RESULT="$RESULT $y" done - echo "diff=$RESULT" >> $GITHUB_OUTPUT + if [ -n "$RESULT" ]; then + RESULT="$(echo $RESULT | jq -R -s -c 'split(" ")')" + else + RESULT='["zlib.net"]' + fi + echo "pkgs=$RESULT" >> $GITHUB_OUTPUT + build: + needs: diff + name: ci ${{ matrix.platform.name }} ${{matrix.pkg}} strategy: fail-fast: false matrix: + pkg: ${{ fromJSON(needs.diff.outputs.pkgs) }} platform: - - darwin+x86-64 - - linux+x86-64 - - darwin+aarch64 - - linux+aarch64 - needs: [get-projects] - uses: ./.github/workflows/build.yml - name: ${{ matrix.platform }} - with: - projects: ${{ needs.get-projects.outputs.diff || 'zlib.net^1.2' }} - platform: ${{ matrix.platform }} - secrets: inherit + - os: ["self-hosted", "macOS", "X64"] + name: x64 + - os: ["self-hosted", "macOS", "ARM64"] + name: ² + - os: ["self-hosted", "linux", "ARM64"] + name: Lnx·ARM64 + - os: {group: "linux-x86-64"} + container: debian:buster-slim + name: Lnx·x64 + runs-on: ${{ matrix.platform.os }} + container: ${{ matrix.platform.container }} + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup + with: + p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} + p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} + APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }} + + - uses: pkgxdev/setup@v2 + with: + PKGX_DIR: /opt + + - uses: pkgxdev/brewkit/build@v1 + id: build + with: + pkg: ${{ matrix.pkg }} + env: + PKGX_PANTRY_PATH: ${{ github.workspace }} + + - uses: pkgxdev/brewkit/audit@v1 + if: steps.build.outputs.pkgspec + env: + PKGX_PANTRY_PATH: ${{ github.workspace }} + + - uses: pkgxdev/brewkit/test@v1 + if: steps.build.outputs.pkgspec + env: + PKGX_PANTRY_PATH: ${{ github.workspace }} \ No newline at end of file diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml deleted file mode 100644 index 817f0c1a..00000000 --- a/.github/workflows/cleanup.yml +++ /dev/null @@ -1,30 +0,0 @@ -# cleans up our S3 staging area if a PR is closed without merge - -name: pkgx/s3-cleanup -run-name: 'cleanup: ${{ github.event.pull_request.title }}' - -on: - pull_request: - types: [closed] - -jobs: - cleanup: - runs-on: ubuntu-latest - if: github.event.pull_request.merged == false - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: remove staged artifacts - run: | - REPO=$(echo ${{github.repository}} | sed -e 's_pkgxdev/__') - PR=$(echo ${{github.ref}} | sed -e 's_refs/pull/\(.*\)/merge_\1_') - - aws s3 rm --recursive s3://$AWS_S3_CACHE/pull-request/$REPO/$PR - if: startsWith(github.ref, 'refs/pull/') && github.repository_owner == 'pkgxdev' - env: - AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} \ No newline at end of file diff --git a/.github/workflows/issue-handler.yml b/.github/workflows/librarian.yml similarity index 98% rename from .github/workflows/issue-handler.yml rename to .github/workflows/librarian.yml index 7a57b150..f6bb34ae 100644 --- a/.github/workflows/issue-handler.yml +++ b/.github/workflows/librarian.yml @@ -1,6 +1,6 @@ # cleans up our issues based on tags applied -name: issue +name: librarian run-name: "handling #${{ github.event.issue.number }}: ${{ github.event.issue.title }}" on: diff --git a/.github/workflows/new-version.yml b/.github/workflows/new-version.yml index 30333a42..28741a65 100644 --- a/.github/workflows/new-version.yml +++ b/.github/workflows/new-version.yml @@ -2,6 +2,12 @@ name: new-version run-name: building ${{ inputs.projects }} on: + workflow_call: + inputs: + projects: + description: eg. `foo.com=1.2.3 bar.com^2.3.4` + required: true + type: string workflow_dispatch: inputs: projects: @@ -10,61 +16,35 @@ on: type: string jobs: - build: - strategy: - fail-fast: false - matrix: - platform: - - darwin+x86-64 - - linux+x86-64 - - darwin+aarch64 - - linux+aarch64 - uses: ./.github/workflows/build.yml - with: - projects: ${{ inputs.projects }} - platform: ${{ matrix.platform }} - secrets: inherit - - bottle: - strategy: - fail-fast: false - matrix: - platform: - - darwin+x86-64 - - linux+x86-64 - - darwin+aarch64 - - linux+aarch64 - needs: [build] - uses: ./.github/workflows/bottle.yml - with: - new-version: true - platform: ${{ matrix.platform }} - projects: ${{ inputs.projects }} - secrets: inherit - - request-qa: - needs: [bottle] - if: ${{ needs.bottle.outputs.qa-required != '[]' }} + divide: runs-on: ubuntu-latest + outputs: + pkgs: ${{ steps.divide.outputs.pkgs }} + steps: + - run: | + var="$(echo ${{ inputs.projects }} | jq -R -s -c 'split(" ")') + echo "pkgs=$var" >> $GITHUB_OUTPUT + id: divide + + pkgit: + needs: divide strategy: matrix: - project: ${{ fromJson(needs.bottle.outputs.qa-required) }} - steps: - - uses: pkgxdev/pantry/.github/actions/request-qa@main - with: - project: ${{ matrix.project }} - slack-webhook: ${{ secrets.SLACK_QA_WEBHOOK }} + pkg: ${{ fromJSON(needs.divide.outputs.pkgs) }} + uses: ./.github/workflows/pkg.yml + with: + pkg: ${{ matrix.pkg }} + secrets: inherit complain: - needs: [build, bottle] + needs: pkgit if: failure() - permissions: - issues: write runs-on: ubuntu-latest + strategy: + fail-fast: false steps: - - uses: pkgxdev/pantry/.github/actions/complain@main + - uses: actions/checkout@v4 + - uses: ./.github/actions/complain with: - projects: ${{ inputs.projects }} - platform: ${{ inputs.platform }} - # slack-webhook: ${{ secrets.SLACK_WEBHOOK }} - # slack-channel: ${{ secrets.SLACK_CHANNEL }} + pkg: ${{ input.pkg }} + platform: ${{ inputs.name }} diff --git a/.github/workflows/pkg-platform.yml b/.github/workflows/pkg-platform.yml new file mode 100644 index 00000000..1569c28f --- /dev/null +++ b/.github/workflows/pkg-platform.yml @@ -0,0 +1,210 @@ +name: pkg for platform +run-name: pkging ${{ inputs.project }} (${{ inputs.name }}) + +on: + workflow_call: + inputs: + name: + description: > + pretty name for the workflow to make GitHub Actions matrix output + more legible + required: false + type: string + os: + required: true + type: string + container: + required: false + type: string + pkg: + description: eg. `example.com@1.2.3` + required: true + type: string + dry-run: + description: dry runs do not modify bottle storage + type: boolean + default: false + test-os: + description: a JSON array of runner-names + required: true + type: string + test-container: + description: > + A JSON array of docker image names or `[null]`. + Indeed! You cannot leave this as `null` or undefined. + Sorry, GHA is not flexible enough to efficiently work around this. + required: true + type: string + secrets: + APPLE_CERTIFICATE_P12: { required: false } + APPLE_CERTIFICATE_P12_PASSWORD: { required: false } + APPLE_IDENTITY: { required: false } + GPG_KEY_ID: { required: true } + GPG_PRIVATE_KEY: { required: true } + AWS_ACCESS_KEY_ID: { required: false } + AWS_S3_BUCKET: { required: true } + AWS_SECRET_ACCESS_KEY: { required: true } + AWS_CF_DISTRIBUTION_ID: { required: true } + +jobs: + build: + name: build (${{inputs.name}}) + runs-on: ${{ fromJSON(inputs.os) }} + container: ${{ inputs.container }} + outputs: + pkg: ${{ steps.build.outputs.pkgspec }} + project: ${{ steps.build.outputs.project }} + version: ${{ steps.build.outputs.version }} + platform: ${{ steps.build.outputs.platform }} + arch: ${{ steps.build.outputs.arch }} + env: + PKGX_PANTRY_PATH: ${{ github.workspace }} + steps: + - uses: actions/checkout@v4 + + - uses: pkgxdev/setup@v2 + with: + PKGX_DIR: /opt + + - uses: ./.github/actions/setup + with: + p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} + p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} + APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }} + + - uses: pkgxdev/brewkit/build@v1 + id: build + with: + pkg: ${{ inputs.pkg }} + + - uses: styfle/cancel-workflow-action@0.12.0 + if: steps.build.outputs.noop + + - uses: pkgxdev/brewkit/audit@v1 + + - uses: pkgxdev/brewkit/upload-build-artifact@v1 + + test: + name: test (${{inputs.name}}) ${{ matrix.container || ''}} ${{ join(matrix.os) }} + needs: build + strategy: + matrix: + os: ${{ fromJSON(inputs.test-os) }} + container: ${{ fromJSON(inputs.test-container) }} + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + env: + PKGX_PANTRY_PATH: ${{ github.workspace }} + steps: + - uses: pkgxdev/setup@v2 + - uses: actions/checkout@v4 + - uses: pkgxdev/brewkit/download-build-artifact@v1 + - uses: pkgxdev/brewkit/test@v1 + with: + pkg: ${{ needs.build.outputs.pkg }} + + bottle: + name: bottle (${{inputs.name}}+${{matrix.compression}}) + needs: [build, test] + strategy: + matrix: + compression: [xz, gz] + runs-on: ubuntu-latest + outputs: + paths: ${{ steps.put.outputs.cf-paths }} + env: + PREFIX: ${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }}/v${{ needs.build.outputs.version }}.tar.${{ matrix.compression }} + steps: + - uses: pkgxdev/setup@v2 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: import GPG key + run: + echo $GPG_PRIVATE_KEY | + base64 -d | + pkgx gpg --import --batch --yes + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + + - uses: pkgxdev/brewkit/download-build-artifact@v1 + id: dl + with: + platform: ${{ inputs.name }} + extract: false + + - uses: pkgxdev/brewkit/bottle@v1 + id: bottle + with: + file: ${{ steps.dl.outputs.filename }} + compression: ${{ matrix.compression }} + + - name: gpg + run: pkgx gpg + --detach-sign + --armor + --output ${{ steps.bottle.outputs.filename }}.asc + --local-user ${{ secrets.GPG_KEY_ID }} + ${{ steps.bottle.outputs.filename }} + + - name: sha + run: pkgx + sha256sum + ${{ steps.bottle.outputs.filename }} > ${{ steps.bottle.outputs.filename }}.sha256sum + + - name: s3 put + run: | + aws s3 cp ${{ steps.bottle.outputs.filename }} $URL + aws s3 cp ${{ steps.bottle.outputs.filename }}.asc $URL.asc + aws s3 cp ${{ steps.bottle.outputs.filename }}.sha256sum $URL.sha256sum + + echo "cf-paths=$PREFIX $PREFIX.asc $PREFIX.sha256sum" >> $GITHUB_OUTPUT + env: + URL: s3://${{ secrets.AWS_S3_BUCKET }}/${{ env.PREFIX }} + id: put + if: ${{ ! inputs.dry-run }} + + versions: + runs-on: ubuntu-latest + needs: [bottle, build] + env: + DIRNAME: ${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }} + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - uses: pkgxdev/setup@v2 + + - name: generate versions.txt + run: | + aws s3 cp \ + s3://${{ secrets.AWS_S3_BUCKET }}/${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }}/versions.txt \ + ./remote-versions.txt + echo "$SCRIPT" > script.ts + pkgx deno run -A script.ts ./remote-versions.txt ${{ needs.build.outputs.version }} > versions.txt + env: + SCRIPT: | + import SemVer, { compare } from "https://raw.githubusercontent.com/pkgxdev/libpkgx/main/src/utils/semver.ts" + const versions = Deno.readTextFileSync(Deno.args[0]).trim().split("\n").filter(x => x) + versions.push(Deno.args[1]) + const out = versions.map(x => new SemVer(x)).sort(compare).join("\n") + await Deno.stdout.write(new TextEncoder().encode(out.trim())) + + - name: s3 put + run: aws s3 cp versions.txt s3://${{ secrets.AWS_S3_BUCKET }}/$DIRNAME/versions.txt + if: ${{ ! inputs.dry-run }} + + - name: invalidate cloudfront + run: aws cloudfront create-invalidation + --distribution-id ${{ secrets.AWS_CF_DISTRIBUTION_ID }} + --paths + /$DIRNAME/versions.txt + ${{ needs.bottle.outputs.paths }} + if: ${{ ! inputs.dry-run }} diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml new file mode 100644 index 00000000..b3bb4e6b --- /dev/null +++ b/.github/workflows/pkg.yml @@ -0,0 +1,41 @@ +name: pkg +run-name: pkging ${{inputs.pkg}} + +on: + workflow_call: + inputs: + pkg: + required: true + type: string + dry-run: + type: boolean + default: false + +jobs: + get-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: pkgxdev/setup@v2 + - uses: actions/checkout@v4 + - run: .github/scripts/get-matrix.ts ${{ inputs.pkg }} + id: matrix + + pkgit: + needs: get-matrix + name: ${{matrix.platform.tinyname}} + strategy: + fail-fast: false + matrix: + platform: ${{ fromJSON(needs.get-matrix.outputs.matrix) }} + uses: ./.github/workflows/pkg-platform.yml + with: + pkg: ${{ inputs.pkg }} + name: ${{ matrix.platform.name }} + os: ${{ toJSON(matrix.platform.os) }} + container: ${{ matrix.platform.container }} + test-os: ${{ toJSON(matrix.platform.test-os) }} + test-container: ${{ toJSON(matrix.platform.test-container) }} + dry-run: ${{ inputs.dry-run }} + secrets: inherit diff --git a/README.md b/README.md index 073368c2..05f92d93 100644 --- a/README.md +++ b/README.md @@ -12,28 +12,32 @@ $ git clone https://github.com/pkgxdev/pantry $ cd pantry $ dev # https://docs.pkgx.sh/dev -# ^^ IMPORTANT! Otherwise the `pkg` command cannot be found +# ^^ adds brewkit to your devenv +# ^^ IMPORTANT! Otherwise the `bk` command will not be found -$ pkg init +$ bk init # ^^ creates a “wip” package.yml # ^^ if you already know the name, you can pass it as an argument -$ pkg edit +$ bk edit # ^^ opens the new package.yml in your EDITOR -$ pkg build +$ bk build # builds to `./builds` # ^^ needs a zero permissions GITHUB_TOKEN to use the GitHub API # either set `GITHUB_TOKEN` or run `gh auth login` -$ pkgx yq .provides * `pkg build` and `pkg test` take a `-L` flag to run in a Linux Docker container -> * All commands take an optional pkg-spec eg. `pkg build node@19` +> [!TIP] +> * `bk build` and `bk test` can be invoked eg. `bk docker build` to run +> inside a Docker container for Linux builds and testing +> * All commands take an optional pkg-spec eg. `bk build node@19` +> [!TIP] > While inside the pantry `dev` environment you can run commands from any built > packages provided you specified their `provides:` key in the `package.yml`. +> [!NOTE] > We use a special package called [`brewkit`] to build packages both here and -> in CI/CD. `brewkit` provides the `pkg` command. +> in CI/CD. `brewkit` provides the `bk` command. + +> [!IMPORTANT] +> brewkit installs the built products to `${PKGX_DIR:-$HOME/.pkgx}` which +> means they are installed to your user’s pkgx cache. ## GitHub Codespaces @@ -101,7 +113,7 @@ $ gh pr checkout 123 $ gh pr checkout https://github.com/pkgxdev/pantry/pull/123 # then open for editing: -$ pkg edit +$ bk edit ``` diff --git a/pkgx.yaml b/pkgx.yaml index c9957361..810d1e0c 100644 --- a/pkgx.yaml +++ b/pkgx.yaml @@ -1,5 +1,4 @@ dependencies: - pkgx.sh/brewkit: ^0 + pkgx.sh/brewkit: ^0 || ^1 env: PKGX_PANTRY_PATH: ${{srcroot}} - SRCROOT: ${{srcroot}} diff --git a/projects/pkgx.sh/brewkit/package.yml b/projects/pkgx.sh/brewkit/package.yml index 50ed3921..f4a37798 100644 --- a/projects/pkgx.sh/brewkit/package.yml +++ b/projects/pkgx.sh/brewkit/package.yml @@ -8,24 +8,25 @@ versions: github: pkgxdev/brewkit dependencies: - deno.land: '>=1.30<1.36.1' - gnu.org/bash: '*' + deno.land: ^1.37 + gnu.org/bash: ^5 pkgx.sh: ^1 +provides: + - bin/bk + build: | mkdir -p {{prefix}} - rm -rf {{prefix}}/* - # ^^ because our build infra uses this pkg to build itself - for x in bin libexec lib share deno.*; do test -e $x && mv $x "{{prefix}}" done test: - dependencies: - zlib.net: '*' - script: - pkg test zlib.net - # we would like to test builds but we need a `GITHUB_TOKEN` and - # (currently) we can’t figure out how to set that up in CI/CD + # we can’t test builds since brewkit is not designed to be invoked + # recursively and fails in the toolchain setup on Linux and fails due to + # Ruby 2 not liking unicode paths on macOS + #- bk build pkgx.sh/brewkit + #- bk test pkgx.sh/brewkit + - bk --help + - bk build --help