make each platform build independently (#148)

* simplify index-packages

* Fix import map location

* sqs modules needs permission :x

* move matrices up a level

* move `complain:` to bottle/build

* more cleanup

* add build-os key to get-platform.ts for self-hosted x86-64 (x64) runners

* needs arrays not strings

* implement testMatrix

* review changes

---------

Co-authored-by: Max Howell <mxcl@me.com>
This commit is contained in:
Jacob Heider 2023-02-02 17:13:38 -05:00 committed by GitHub
parent 72b9fa85bd
commit a2da161839
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 191 additions and 144 deletions

View file

@ -7,24 +7,31 @@ on:
type: boolean type: boolean
required: false required: false
default: false default: false
platform:
required: true
type: string
jobs: jobs:
get-platform:
runs-on: ubuntu-latest
outputs:
os: ${{ steps.platform.outputs.OS }}
steps:
- uses: actions/checkout@v3
with:
repository: teaxyz/pantry.core
- uses: teaxyz/setup@v0
- run: scripts/get-platform.ts ${{ inputs.projects }}
id: platform
env:
PLATFORM: ${{ inputs.platform }}
bottle: bottle:
runs-on: ${{ matrix.platform.os }} needs: [get-platform]
runs-on: ${{ fromJson(needs.get-platform.outputs.os) }}
defaults: defaults:
run: run:
working-directory: tea.xyz/var/pantry working-directory: tea.xyz/var/pantry
strategy:
matrix:
platform:
- os: macos-11
name: darwin+x86-64
- os: ubuntu-latest
name: linux+x86-64
- os: [self-hosted, macOS, ARM64]
name: darwin+aarch64
- os: [self-hosted, linux, ARM64]
name: linux+aarch64
outputs: outputs:
srcs: ${{ env.srcs }} srcs: ${{ env.srcs }}
built: ${{ env.built }} built: ${{ env.built }}
@ -45,20 +52,16 @@ jobs:
srcroot: tea.xyz/var/pantry srcroot: tea.xyz/var/pantry
prefix: ${{ github.workspace }} prefix: ${{ github.workspace }}
- run: | - name: configure scripts PATH
# in case this PR contains updates to the scripts run: echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH
#TODO only do for PRs
if test "$GITHUB_REPOSITORY" = "teaxyz/pantry.core"; then
cp -rv $GITHUB_WORKSPACE/pantry/scripts/* $(tea --prefix)/tea.xyz/var/pantry/scripts
fi
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
if: ${{ inputs.new-version }} if: ${{ inputs.new-version }}
with: with:
name: ${{ matrix.platform.name }} name: ${{ inputs.platform }}
path: tea.xyz/var/pantry path: tea.xyz/var/pantry
- run: scripts/fetch-pr-artifacts.ts ${{ github.repository }} ${{ github.sha }} ${{ matrix.platform.name }} >>$GITHUB_ENV - run: scripts/fetch-pr-artifacts.ts ${{ github.repository }} ${{ github.sha }} ${{ inputs.platform }} >>$GITHUB_ENV
if: ${{ !inputs.new-version }} if: ${{ !inputs.new-version }}
env: env:
GITHUB_TOKEN: ${{github.token}} GITHUB_TOKEN: ${{github.token}}
@ -81,14 +84,14 @@ jobs:
# FIXME: replace this with a tea script based on https://localazy.com/blog/how-to-automatically-sign-macos-apps-using-github-actions # FIXME: replace this with a tea script based on https://localazy.com/blog/how-to-automatically-sign-macos-apps-using-github-actions
# github has a doc with similar content, but it's not returning to me atm. # github has a doc with similar content, but it's not returning to me atm.
- uses: apple-actions/import-codesign-certs@d54750db52a4d3eaed0fc107a8bab3958f3f7494 - uses: apple-actions/import-codesign-certs@d54750db52a4d3eaed0fc107a8bab3958f3f7494
if: matrix.platform.name == 'darwin+aarch64' || matrix.platform.name == 'darwin+x86-64' if: startsWith(inputs.platform, 'darwin+')
with: with:
p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }}
p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }}
# Codesign libs and bins # Codesign libs and bins
- name: Codesign package - name: Codesign package
if: matrix.platform.name == 'darwin+aarch64' || matrix.platform.name == 'darwin+x86-64' if: startsWith(inputs.platform, 'darwin+')
run: | run: |
for PKG in ${{ env.relative-paths }}; do for PKG in ${{ env.relative-paths }}; do
find /opt/$PKG -name '*.so' -or -name '*.dylib' -print0 | \ find /opt/$PKG -name '*.so' -or -name '*.dylib' -print0 | \
@ -99,7 +102,7 @@ jobs:
# This isn't very informative, but even a no-op is safer than none # This isn't very informative, but even a no-op is safer than none
- name: Check codesigning - name: Check codesigning
if: matrix.platform.name == 'darwin+aarch64' || matrix.platform.name == 'darwin+x86-64' if: startsWith(inputs.platform, 'darwin+')
run: | run: |
for PKG in ${{ env.relative-paths }}; do for PKG in ${{ env.relative-paths }}; do
for SIG in `find /opt/$PKG -name '*.so' -or -name '*.dylib'` `find /opt/$PKG/bin -type f`; do for SIG in `find /opt/$PKG -name '*.so' -or -name '*.dylib'` `find /opt/$PKG/bin -type f`; do
@ -110,7 +113,7 @@ jobs:
# Needed for self-hosted runner, since it doesn't destroy itself automatically. # Needed for self-hosted runner, since it doesn't destroy itself automatically.
- name: Delete keychain - name: Delete keychain
if: always() && matrix.platform.name == 'darwin+aarch64' if: always() && inputs.platform == 'darwin+aarch64'
run: security delete-keychain signing_temp.keychain run: security delete-keychain signing_temp.keychain
- run: | - run: |
@ -153,7 +156,7 @@ jobs:
- name: upload artifacts - name: upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: ${{ matrix.platform.name }}-bottles name: ${{ inputs.platform }}-bottles
path: artifacts.tar path: artifacts.tar
if-no-files-found: error if-no-files-found: error
@ -163,17 +166,6 @@ jobs:
defaults: defaults:
run: run:
working-directory: tea.xyz/var/pantry working-directory: tea.xyz/var/pantry
strategy:
matrix:
platform:
- os: macos-11
name: darwin+x86-64
- os: ubuntu-latest
name: linux+x86-64
- os: [self-hosted, macOS, ARM64]
name: darwin+aarch64
- os: [self-hosted, linux, ARM64]
name: linux+aarch64
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
@ -192,7 +184,7 @@ jobs:
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.platform.name }}-bottles name: ${{ inputs.platform }}-bottles
- run: | - run: |
tar xvf $GITHUB_WORKSPACE/artifacts.tar tar xvf $GITHUB_WORKSPACE/artifacts.tar
@ -202,12 +194,8 @@ jobs:
done done
working-directory: ${{ steps.tea.outputs.prefix }} working-directory: ${{ steps.tea.outputs.prefix }}
- run: | - name: configure scripts PATH
# in case this PR contains updates to the scripts run: echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH
#TODO only do for PRs
if test "$GITHUB_REPOSITORY" = "teaxyz/pantry.core"; then
cp -rv $GITHUB_WORKSPACE/pantry/scripts/* $(tea --prefix)/tea.xyz/var/pantry/scripts
fi
- name: upload bottles - name: upload bottles
id: upload id: upload
@ -222,8 +210,6 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
#NOTE ideally wed invalidate all at once so this is atomic
# however GHA cant consolidate outputs from a matrix :/
- uses: chetan/invalidate-cloudfront-action@v2 - uses: chetan/invalidate-cloudfront-action@v2
env: env:
PATHS: ${{ steps.upload.outputs.cf-invalidation-paths }} PATHS: ${{ steps.upload.outputs.cf-invalidation-paths }}
@ -255,3 +241,26 @@ jobs:
env: env:
AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }}
PR: ${{ needs.bottle.outputs.pr }} PR: ${{ needs.bottle.outputs.pr }}
complain:
if: failure() && inputs.new-version == 'true'
needs: [upload]
runs-on: ubuntu-latest
steps:
- uses: martialonline/workflow-status@v3
id: status
- uses: rtCamp/action-slack-notify@v2
if: ${{ env.SLACK_WEBHOOK != '' }}
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_MESSAGE: new-version:${{ inputs.projects }} (${{ inputs.platform }}) ${{ steps.status.outputs.status }}
SLACK_COLOR: ${{ steps.status.outputs.status }}
- uses: actions/checkout@v3
if: ${{ steps.status.outputs.status == 'failure' }}
- uses: JasonEtco/create-an-issue@v2
if: ${{ steps.status.outputs.status == 'failure' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PACKAGE: ${{ inputs.projects }} (${{ inputs.platform }})
URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

View file

@ -6,38 +6,35 @@ on:
projects: projects:
required: true required: true
type: string type: string
platform:
required: true
type: string
new-version:
type: boolean
required: false
default: false
jobs: jobs:
check-core-sizes: get-platform:
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
gha-linux-build-size: ${{ env.GHA_LINUX_BUILD_SIZE }} os: ${{ steps.platform.outputs.os }}
build-os: ${{ steps.platform.outputs.build-os }}
test-matrix: ${{ steps.platform.outputs.test-matrix }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
with: with:
repository: teaxyz/pantry.core repository: teaxyz/pantry.core
- uses: teaxyz/setup@v0 - uses: teaxyz/setup@v0
- run: scripts/check-core-sizes.ts ${{ inputs.projects }} - run: scripts/get-platform.ts ${{ inputs.projects }}
id: platform
env: env:
PLATFORM: ${{ inputs.platform }}
TEA_PANTRY_PATH: ${{ github.workspace }} TEA_PANTRY_PATH: ${{ github.workspace }}
build: build:
runs-on: ${{ matrix.platform.os }} runs-on: ${{ fromJson(needs.get-platform.outputs.build-os) }}
needs: [check-core-sizes] needs: [get-platform]
strategy:
matrix:
platform:
- os: macos-11
name: darwin+x86-64
- os: ${{ needs.check-core-sizes.outputs.gha-linux-build-size }}
name: linux+x86-64
container:
image: debian:buster-slim
- os: [self-hosted, macOS, ARM64]
name: darwin+aarch64
- os: [self-hosted, linux, ARM64]
name: linux+aarch64
container: ${{ matrix.platform.container }}
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
@ -89,30 +86,16 @@ jobs:
- name: upload artifacts - name: upload artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: ${{ matrix.platform.name }} name: ${{ inputs.platform }}
path: artifacts.tgz path: artifacts.tgz
if-no-files-found: error if-no-files-found: error
test: test:
needs: [build] needs: [get-platform, build]
runs-on: ${{ matrix.platform.os }} runs-on: ${{ matrix.platform.os }}
strategy: strategy:
matrix: matrix:
platform: platform: ${{ fromJson(needs.get-platform.outputs.test-matrix) }}
- os: macos-11
name: darwin+x86-64
- os: ubuntu-latest
name: linux+x86-64
- os: ubuntu-latest
name: linux+x86-64
container: ghcr.io/teaxyz/infuser:latest
- os: ubuntu-latest
name: linux+x86-64
container: debian:buster-slim
- os: [self-hosted, macOS, ARM64]
name: darwin+aarch64
- os: [self-hosted, linux, ARM64]
name: linux+aarch64
outputs: outputs:
HAS_SECRETS: ${{ env.HAS_SECRETS }} HAS_SECRETS: ${{ env.HAS_SECRETS }}
container: ${{ matrix.platform.container }} container: ${{ matrix.platform.container }}
@ -125,7 +108,7 @@ jobs:
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.platform.name }} name: ${{ inputs.platform }}
- name: extract bottles - name: extract bottles
run: tar xzf artifacts.tgz -C $TEA_PREFIX run: tar xzf artifacts.tgz -C $TEA_PREFIX
@ -145,17 +128,6 @@ jobs:
# this only works for PRs from our team to our repo (security! :( ) # this only works for PRs from our team to our repo (security! :( )
if: startsWith(github.ref, 'refs/pull/') && startsWith(github.repository, 'teaxyz/pantry.') && needs.test.outputs.HAS_SECRETS == 'true' if: startsWith(github.ref, 'refs/pull/') && startsWith(github.repository, 'teaxyz/pantry.') && needs.test.outputs.HAS_SECRETS == 'true'
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy:
matrix:
platform:
- os: macos-11
name: darwin+x86-64
- os: ubuntu-latest
name: linux+x86-64
- os: [self-hosted, macOS, ARM64]
name: darwin+aarch64
- os: [self-hosted, linux, ARM64]
name: linux+aarch64
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: teaxyz/setup@v0 - uses: teaxyz/setup@v0
@ -165,14 +137,37 @@ jobs:
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: ${{ matrix.platform.name }} name: ${{ inputs.platform }}
- run: cache-artifacts.ts - run: cache-artifacts.ts
${{github.repository}} ${{github.repository}}
${{github.ref}} ${{github.ref}}
${{matrix.platform.name}} ${{inputs.platform}}
artifacts.tgz artifacts.tgz
env: env:
AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
complain:
if: failure() && inputs.new-version == 'true'
needs: [test]
runs-on: ubuntu-latest
steps:
- uses: martialonline/workflow-status@v3
id: status
- uses: rtCamp/action-slack-notify@v2
if: ${{ env.SLACK_WEBHOOK != '' }}
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_MESSAGE: new-version:${{ inputs.projects }} (${{ inputs.platform }}) ${{ steps.status.outputs.status }}
SLACK_COLOR: ${{ steps.status.outputs.status }}
- uses: actions/checkout@v3
if: ${{ steps.status.outputs.status == 'failure' }}
- uses: JasonEtco/create-an-issue@v2
if: ${{ steps.status.outputs.status == 'failure' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PACKAGE: ${{ inputs.projects }} (${{ inputs.platform }})
URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

View file

@ -24,9 +24,18 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
bottle-pr: bottle-pr:
strategy:
matrix:
platform:
- darwin+x86-64
- linux+x86-64
- darwin+aarch64
- linux+aarch64
needs: [cd] needs: [cd]
if: ${{ needs.cd.outputs.HAS_ARTIFACTS == 'true' }} if: ${{ needs.cd.outputs.HAS_ARTIFACTS == 'true' }}
uses: ./.github/workflows/bottle.yml uses: ./.github/workflows/bottle.yml
with:
platform: ${{ matrix.platform }}
secrets: inherit secrets: inherit
bottle-standalone: bottle-standalone:
runs-on: ubuntu-latest runs-on: ubuntu-latest

View file

@ -13,8 +13,16 @@ jobs:
with: with:
PATTERNS: projects/**/package.yml PATTERNS: projects/**/package.yml
build: build:
strategy:
matrix:
platform:
- darwin+x86-64
- linux+x86-64
- darwin+aarch64
- linux+aarch64
needs: [get-diff] needs: [get-diff]
uses: ./.github/workflows/build.yml uses: ./.github/workflows/build.yml
with: with:
projects: ${{ needs.get-diff.outputs.diff || 'zlib.net' }} projects: ${{ needs.get-diff.outputs.diff || 'zlib.net' }}
platform: ${{ matrix.platform }}
secrets: inherit secrets: inherit

View file

@ -14,15 +14,31 @@ permissions:
jobs: jobs:
build: build:
strategy:
matrix:
platform:
- darwin+x86-64
- linux+x86-64
- darwin+aarch64
- linux+aarch64
uses: ./.github/workflows/build.yml uses: ./.github/workflows/build.yml
with: with:
projects: ${{ inputs.projects }} projects: ${{ inputs.projects }}
platform: ${{ matrix.platform }}
secrets: inherit secrets: inherit
bottle: bottle:
strategy:
matrix:
platform:
- darwin+x86-64
- linux+x86-64
- darwin+aarch64
- linux+aarch64
needs: [build] needs: [build]
uses: ./.github/workflows/bottle.yml uses: ./.github/workflows/bottle.yml
with: with:
new-version: true new-version: true
platform: ${{ matrix.platform }}
secrets: inherit secrets: inherit
index_data: index_data:
needs: [bottle] needs: [bottle]

View file

@ -1,49 +0,0 @@
#!/usr/bin/env -S tea -E
/*---
args:
- deno
- run
- --allow-read
- --allow-env
- --allow-write
---*/
import * as ARGV from "./utils/args.ts"
const exceptions: { [pkg: string]: number } = {
"deno.land": 4,
"ziglang.org": 8,
}
const pkgs = await ARGV.toArray(ARGV.pkgs())
let coreSize = 2
for (const pkg of pkgs) {
coreSize = Math.max(exceptions[pkg.project] || 2, coreSize)
}
const output = `GHA_LINUX_BUILD_SIZE=${imageName(coreSize)}\n`
Deno.stdout.write(new TextEncoder().encode(output))
if (Deno.env.get("GITHUB_ENV")) {
const envFile = Deno.env.get("GITHUB_ENV")!
await Deno.writeTextFile(envFile, output, { append: true})
}
function imageName(size: number) {
switch (size) {
case 0:
case 1:
case 2:
return "ubuntu-latest"
case 4:
case 8:
case 16:
return `ubuntu-latest-${size}-cores`
default:
throw new Error("Invalid core size")
}
}

59
scripts/get-platform.ts Executable file
View file

@ -0,0 +1,59 @@
#!/usr/bin/env tea
/*---
args:
- deno
- run
- --allow-read
- --allow-env
- --allow-write
---*/
import { panic } from "utils";
const platform = Deno.env.get("PLATFORM") ?? panic("$PLATFORM not set")
let os: string | string[]
let buildOs: string | string[]
let testMatrix: { os: string | string[], container: string | undefined }[]
switch(platform) {
case "darwin+x86-64":
os = "macos-11"
buildOs = ["self-hosted", "macOS", "X64"]
testMatrix = [{ os, container: undefined }]
break
case "darwin+aarch64":
os = ["self-hosted", "macOS", "ARM64"]
buildOs = os
testMatrix = [{ os, container: undefined }]
break
case "linux+aarch64":
os = ["self-hosted", "linux", "ARM64"]
buildOs = os
testMatrix = [{ os, container: undefined }]
break
case "linux+x86-64":
os = "ubuntu-latest"
buildOs = ["self-hosted", "linux", "X64"]
testMatrix = [
{ os, container: undefined },
{ os: buildOs, container: undefined },
{ os, container: "ghcr.io/teaxyz/infuser:latest" },
{ os, container: "debian:buster-slim" },
]
break
default:
panic(`Invalid platform description: ${platform}`)
}
const output = `os=${JSON.stringify(os)}\n` +
`build-os=${JSON.stringify(buildOs)}\n` +
`test-matrix=${JSON.stringify(testMatrix)}\n`
Deno.stdout.write(new TextEncoder().encode(output))
if (Deno.env.get("GITHUB_OUTPUT")) {
const envFile = Deno.env.get("GITHUB_OUTPUT")!
await Deno.writeTextFile(envFile, output, { append: true})
}