From a2da16183971fe17cf9d0a701c68f428e82eeaff Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Thu, 2 Feb 2023 17:13:38 -0500 Subject: [PATCH] make each platform build independently (#148) * simplify index-packages * Fix import map location * sqs modules needs permission :x * move matrices up a level * move `complain:` to bottle/build * more cleanup * add build-os key to get-platform.ts for self-hosted x86-64 (x64) runners * needs arrays not strings * implement testMatrix * review changes --------- Co-authored-by: Max Howell --- .github/workflows/bottle.yml | 99 +++++++++++++++++-------------- .github/workflows/build.yml | 95 ++++++++++++++--------------- .github/workflows/cd.yml | 9 +++ .github/workflows/ci.yml | 8 +++ .github/workflows/new-version.yml | 16 +++++ scripts/check-core-sizes.ts | 49 --------------- scripts/get-platform.ts | 59 ++++++++++++++++++ 7 files changed, 191 insertions(+), 144 deletions(-) delete mode 100755 scripts/check-core-sizes.ts create mode 100755 scripts/get-platform.ts diff --git a/.github/workflows/bottle.yml b/.github/workflows/bottle.yml index 5f5b7644..c26dff9b 100644 --- a/.github/workflows/bottle.yml +++ b/.github/workflows/bottle.yml @@ -7,24 +7,31 @@ on: type: boolean required: false default: false + platform: + required: true + type: string jobs: + get-platform: + runs-on: ubuntu-latest + outputs: + os: ${{ steps.platform.outputs.OS }} + steps: + - uses: actions/checkout@v3 + with: + repository: teaxyz/pantry.core + - uses: teaxyz/setup@v0 + - run: scripts/get-platform.ts ${{ inputs.projects }} + id: platform + env: + PLATFORM: ${{ inputs.platform }} + bottle: - runs-on: ${{ matrix.platform.os }} + needs: [get-platform] + runs-on: ${{ fromJson(needs.get-platform.outputs.os) }} defaults: run: working-directory: tea.xyz/var/pantry - strategy: - matrix: - platform: - - os: macos-11 - name: darwin+x86-64 - - os: ubuntu-latest - name: linux+x86-64 - - os: [self-hosted, macOS, ARM64] - name: darwin+aarch64 - - os: [self-hosted, linux, ARM64] - name: linux+aarch64 outputs: srcs: ${{ env.srcs }} built: ${{ env.built }} @@ -45,20 +52,16 @@ jobs: srcroot: tea.xyz/var/pantry prefix: ${{ github.workspace }} - - run: | - # in case this PR contains updates to the scripts - #TODO only do for PRs - if test "$GITHUB_REPOSITORY" = "teaxyz/pantry.core"; then - cp -rv $GITHUB_WORKSPACE/pantry/scripts/* $(tea --prefix)/tea.xyz/var/pantry/scripts - fi + - name: configure scripts PATH + run: echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH - uses: actions/download-artifact@v3 if: ${{ inputs.new-version }} with: - name: ${{ matrix.platform.name }} + name: ${{ inputs.platform }} path: tea.xyz/var/pantry - - run: scripts/fetch-pr-artifacts.ts ${{ github.repository }} ${{ github.sha }} ${{ matrix.platform.name }} >>$GITHUB_ENV + - run: scripts/fetch-pr-artifacts.ts ${{ github.repository }} ${{ github.sha }} ${{ inputs.platform }} >>$GITHUB_ENV if: ${{ !inputs.new-version }} env: GITHUB_TOKEN: ${{github.token}} @@ -81,14 +84,14 @@ jobs: # FIXME: replace this with a tea script based on https://localazy.com/blog/how-to-automatically-sign-macos-apps-using-github-actions # github has a doc with similar content, but it's not returning to me atm. - uses: apple-actions/import-codesign-certs@d54750db52a4d3eaed0fc107a8bab3958f3f7494 - if: matrix.platform.name == 'darwin+aarch64' || matrix.platform.name == 'darwin+x86-64' + if: startsWith(inputs.platform, 'darwin+') with: p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} # Codesign libs and bins - name: Codesign package - if: matrix.platform.name == 'darwin+aarch64' || matrix.platform.name == 'darwin+x86-64' + if: startsWith(inputs.platform, 'darwin+') run: | for PKG in ${{ env.relative-paths }}; do find /opt/$PKG -name '*.so' -or -name '*.dylib' -print0 | \ @@ -99,7 +102,7 @@ jobs: # This isn't very informative, but even a no-op is safer than none - name: Check codesigning - if: matrix.platform.name == 'darwin+aarch64' || matrix.platform.name == 'darwin+x86-64' + if: startsWith(inputs.platform, 'darwin+') run: | for PKG in ${{ env.relative-paths }}; do for SIG in `find /opt/$PKG -name '*.so' -or -name '*.dylib'` `find /opt/$PKG/bin -type f`; do @@ -110,7 +113,7 @@ jobs: # Needed for self-hosted runner, since it doesn't destroy itself automatically. - name: Delete keychain - if: always() && matrix.platform.name == 'darwin+aarch64' + if: always() && inputs.platform == 'darwin+aarch64' run: security delete-keychain signing_temp.keychain - run: | @@ -153,7 +156,7 @@ jobs: - name: upload artifacts uses: actions/upload-artifact@v3 with: - name: ${{ matrix.platform.name }}-bottles + name: ${{ inputs.platform }}-bottles path: artifacts.tar if-no-files-found: error @@ -163,17 +166,6 @@ jobs: defaults: run: working-directory: tea.xyz/var/pantry - strategy: - matrix: - platform: - - os: macos-11 - name: darwin+x86-64 - - os: ubuntu-latest - name: linux+x86-64 - - os: [self-hosted, macOS, ARM64] - name: darwin+aarch64 - - os: [self-hosted, linux, ARM64] - name: linux+aarch64 steps: - uses: actions/checkout@v3 with: @@ -192,7 +184,7 @@ jobs: - uses: actions/download-artifact@v3 with: - name: ${{ matrix.platform.name }}-bottles + name: ${{ inputs.platform }}-bottles - run: | tar xvf $GITHUB_WORKSPACE/artifacts.tar @@ -202,12 +194,8 @@ jobs: done working-directory: ${{ steps.tea.outputs.prefix }} - - run: | - # in case this PR contains updates to the scripts - #TODO only do for PRs - if test "$GITHUB_REPOSITORY" = "teaxyz/pantry.core"; then - cp -rv $GITHUB_WORKSPACE/pantry/scripts/* $(tea --prefix)/tea.xyz/var/pantry/scripts - fi + - name: configure scripts PATH + run: echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH - name: upload bottles id: upload @@ -222,8 +210,6 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - #NOTE ideally we’d invalidate all at once so this is atomic - # however GHA can’t consolidate outputs from a matrix :/ - uses: chetan/invalidate-cloudfront-action@v2 env: PATHS: ${{ steps.upload.outputs.cf-invalidation-paths }} @@ -255,3 +241,26 @@ jobs: env: AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} PR: ${{ needs.bottle.outputs.pr }} + + complain: + if: failure() && inputs.new-version == 'true' + needs: [upload] + runs-on: ubuntu-latest + steps: + - uses: martialonline/workflow-status@v3 + id: status + - uses: rtCamp/action-slack-notify@v2 + if: ${{ env.SLACK_WEBHOOK != '' }} + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_MESSAGE: new-version:${{ inputs.projects }} (${{ inputs.platform }}) ${{ steps.status.outputs.status }} + SLACK_COLOR: ${{ steps.status.outputs.status }} + - uses: actions/checkout@v3 + if: ${{ steps.status.outputs.status == 'failure' }} + - uses: JasonEtco/create-an-issue@v2 + if: ${{ steps.status.outputs.status == 'failure' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PACKAGE: ${{ inputs.projects }} (${{ inputs.platform }}) + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 29b0b399..2de96fa5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,38 +6,35 @@ on: projects: required: true type: string + platform: + required: true + type: string + new-version: + type: boolean + required: false + default: false jobs: - check-core-sizes: + get-platform: runs-on: ubuntu-latest outputs: - gha-linux-build-size: ${{ env.GHA_LINUX_BUILD_SIZE }} + os: ${{ steps.platform.outputs.os }} + build-os: ${{ steps.platform.outputs.build-os }} + test-matrix: ${{ steps.platform.outputs.test-matrix }} steps: - uses: actions/checkout@v3 with: repository: teaxyz/pantry.core - uses: teaxyz/setup@v0 - - run: scripts/check-core-sizes.ts ${{ inputs.projects }} + - run: scripts/get-platform.ts ${{ inputs.projects }} + id: platform env: + PLATFORM: ${{ inputs.platform }} TEA_PANTRY_PATH: ${{ github.workspace }} build: - runs-on: ${{ matrix.platform.os }} - needs: [check-core-sizes] - strategy: - matrix: - platform: - - os: macos-11 - name: darwin+x86-64 - - os: ${{ needs.check-core-sizes.outputs.gha-linux-build-size }} - name: linux+x86-64 - container: - image: debian:buster-slim - - os: [self-hosted, macOS, ARM64] - name: darwin+aarch64 - - os: [self-hosted, linux, ARM64] - name: linux+aarch64 - container: ${{ matrix.platform.container }} + runs-on: ${{ fromJson(needs.get-platform.outputs.build-os) }} + needs: [get-platform] steps: - uses: actions/checkout@v3 @@ -89,30 +86,16 @@ jobs: - name: upload artifacts uses: actions/upload-artifact@v3 with: - name: ${{ matrix.platform.name }} + name: ${{ inputs.platform }} path: artifacts.tgz if-no-files-found: error test: - needs: [build] + needs: [get-platform, build] runs-on: ${{ matrix.platform.os }} strategy: matrix: - platform: - - os: macos-11 - name: darwin+x86-64 - - os: ubuntu-latest - name: linux+x86-64 - - os: ubuntu-latest - name: linux+x86-64 - container: ghcr.io/teaxyz/infuser:latest - - os: ubuntu-latest - name: linux+x86-64 - container: debian:buster-slim - - os: [self-hosted, macOS, ARM64] - name: darwin+aarch64 - - os: [self-hosted, linux, ARM64] - name: linux+aarch64 + platform: ${{ fromJson(needs.get-platform.outputs.test-matrix) }} outputs: HAS_SECRETS: ${{ env.HAS_SECRETS }} container: ${{ matrix.platform.container }} @@ -125,7 +108,7 @@ jobs: - uses: actions/download-artifact@v3 with: - name: ${{ matrix.platform.name }} + name: ${{ inputs.platform }} - name: extract bottles run: tar xzf artifacts.tgz -C $TEA_PREFIX @@ -145,17 +128,6 @@ jobs: # this only works for PRs from our team to our repo (security! :( ) if: startsWith(github.ref, 'refs/pull/') && startsWith(github.repository, 'teaxyz/pantry.') && needs.test.outputs.HAS_SECRETS == 'true' runs-on: ubuntu-latest - strategy: - matrix: - platform: - - os: macos-11 - name: darwin+x86-64 - - os: ubuntu-latest - name: linux+x86-64 - - os: [self-hosted, macOS, ARM64] - name: darwin+aarch64 - - os: [self-hosted, linux, ARM64] - name: linux+aarch64 steps: - uses: actions/checkout@v3 - uses: teaxyz/setup@v0 @@ -165,14 +137,37 @@ jobs: - uses: actions/download-artifact@v3 with: - name: ${{ matrix.platform.name }} + name: ${{ inputs.platform }} - run: cache-artifacts.ts ${{github.repository}} ${{github.ref}} - ${{matrix.platform.name}} + ${{inputs.platform}} artifacts.tgz env: AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + complain: + if: failure() && inputs.new-version == 'true' + needs: [test] + runs-on: ubuntu-latest + steps: + - uses: martialonline/workflow-status@v3 + id: status + - uses: rtCamp/action-slack-notify@v2 + if: ${{ env.SLACK_WEBHOOK != '' }} + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_MESSAGE: new-version:${{ inputs.projects }} (${{ inputs.platform }}) ${{ steps.status.outputs.status }} + SLACK_COLOR: ${{ steps.status.outputs.status }} + - uses: actions/checkout@v3 + if: ${{ steps.status.outputs.status == 'failure' }} + - uses: JasonEtco/create-an-issue@v2 + if: ${{ steps.status.outputs.status == 'failure' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PACKAGE: ${{ inputs.projects }} (${{ inputs.platform }}) + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index ffc9ae61..1eba3584 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -24,9 +24,18 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} bottle-pr: + strategy: + matrix: + platform: + - darwin+x86-64 + - linux+x86-64 + - darwin+aarch64 + - linux+aarch64 needs: [cd] if: ${{ needs.cd.outputs.HAS_ARTIFACTS == 'true' }} uses: ./.github/workflows/bottle.yml + with: + platform: ${{ matrix.platform }} secrets: inherit bottle-standalone: runs-on: ubuntu-latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73f55f1e..bb4a13a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,16 @@ jobs: with: PATTERNS: projects/**/package.yml build: + strategy: + matrix: + platform: + - darwin+x86-64 + - linux+x86-64 + - darwin+aarch64 + - linux+aarch64 needs: [get-diff] uses: ./.github/workflows/build.yml with: projects: ${{ needs.get-diff.outputs.diff || 'zlib.net' }} + platform: ${{ matrix.platform }} secrets: inherit diff --git a/.github/workflows/new-version.yml b/.github/workflows/new-version.yml index 0933bffb..db222660 100644 --- a/.github/workflows/new-version.yml +++ b/.github/workflows/new-version.yml @@ -14,15 +14,31 @@ permissions: jobs: build: + strategy: + matrix: + platform: + - darwin+x86-64 + - linux+x86-64 + - darwin+aarch64 + - linux+aarch64 uses: ./.github/workflows/build.yml with: projects: ${{ inputs.projects }} + platform: ${{ matrix.platform }} secrets: inherit bottle: + strategy: + matrix: + platform: + - darwin+x86-64 + - linux+x86-64 + - darwin+aarch64 + - linux+aarch64 needs: [build] uses: ./.github/workflows/bottle.yml with: new-version: true + platform: ${{ matrix.platform }} secrets: inherit index_data: needs: [bottle] diff --git a/scripts/check-core-sizes.ts b/scripts/check-core-sizes.ts deleted file mode 100755 index 9ccfef81..00000000 --- a/scripts/check-core-sizes.ts +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env -S tea -E - -/*--- -args: - - deno - - run - - --allow-read - - --allow-env - - --allow-write ----*/ - -import * as ARGV from "./utils/args.ts" - -const exceptions: { [pkg: string]: number } = { - "deno.land": 4, - "ziglang.org": 8, -} - -const pkgs = await ARGV.toArray(ARGV.pkgs()) - -let coreSize = 2 - -for (const pkg of pkgs) { - coreSize = Math.max(exceptions[pkg.project] || 2, coreSize) -} - -const output = `GHA_LINUX_BUILD_SIZE=${imageName(coreSize)}\n` - -Deno.stdout.write(new TextEncoder().encode(output)) - -if (Deno.env.get("GITHUB_ENV")) { - const envFile = Deno.env.get("GITHUB_ENV")! - await Deno.writeTextFile(envFile, output, { append: true}) -} - -function imageName(size: number) { - switch (size) { - case 0: - case 1: - case 2: - return "ubuntu-latest" - case 4: - case 8: - case 16: - return `ubuntu-latest-${size}-cores` - default: - throw new Error("Invalid core size") - } -} \ No newline at end of file diff --git a/scripts/get-platform.ts b/scripts/get-platform.ts new file mode 100755 index 00000000..8f422281 --- /dev/null +++ b/scripts/get-platform.ts @@ -0,0 +1,59 @@ +#!/usr/bin/env tea + +/*--- +args: + - deno + - run + - --allow-read + - --allow-env + - --allow-write +---*/ + +import { panic } from "utils"; + +const platform = Deno.env.get("PLATFORM") ?? panic("$PLATFORM not set") + +let os: string | string[] +let buildOs: string | string[] +let testMatrix: { os: string | string[], container: string | undefined }[] + +switch(platform) { + case "darwin+x86-64": + os = "macos-11" + buildOs = ["self-hosted", "macOS", "X64"] + testMatrix = [{ os, container: undefined }] + break + case "darwin+aarch64": + os = ["self-hosted", "macOS", "ARM64"] + buildOs = os + testMatrix = [{ os, container: undefined }] + break + case "linux+aarch64": + os = ["self-hosted", "linux", "ARM64"] + buildOs = os + testMatrix = [{ os, container: undefined }] + break + case "linux+x86-64": + os = "ubuntu-latest" + buildOs = ["self-hosted", "linux", "X64"] + testMatrix = [ + { os, container: undefined }, + { os: buildOs, container: undefined }, + { os, container: "ghcr.io/teaxyz/infuser:latest" }, + { os, container: "debian:buster-slim" }, + ] + break + default: + panic(`Invalid platform description: ${platform}`) +} + +const output = `os=${JSON.stringify(os)}\n` + + `build-os=${JSON.stringify(buildOs)}\n` + + `test-matrix=${JSON.stringify(testMatrix)}\n` + +Deno.stdout.write(new TextEncoder().encode(output)) + +if (Deno.env.get("GITHUB_OUTPUT")) { + const envFile = Deno.env.get("GITHUB_OUTPUT")! + await Deno.writeTextFile(envFile, output, { append: true}) +}