use brewkit@v1 (#4314)

This commit is contained in:
Max Howell 2023-12-13 05:42:25 -05:00 committed by GitHub
parent 34ba236245
commit 7193b3c3d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 572 additions and 611 deletions

View file

@ -2,8 +2,8 @@ name: pkgx/pantry/complain
description: creates an issue for failure conditions
inputs:
projects:
description: projects to complain about
pkg:
description: project to complain about
required: true
platform:
description: platform key
@ -12,12 +12,6 @@ inputs:
description: github token
required: true
default: ${{ github.token }}
# slack-webhook:
# description: slack webhook
# required: false
# slack-channel:
# description: slack channel
# required: false
runs:
using: composite
@ -29,7 +23,7 @@ runs:
actions: 'find-issues'
token: ${{ inputs.token }}
issue-state: 'open'
title-includes: "❌ build issues: ${{ inputs.projects }}"
title-includes: "❌ build issues: ${{ inputs.pkg }}"
labels: 'build-failure'
- name: Create Issue
@ -39,8 +33,8 @@ runs:
with:
actions: 'create-issue'
token: ${{ inputs.token }}
title: "❌ build issues: ${{ inputs.projects }}"
body: "Running log of build failures for ${{ inputs.projects }}"
title: "❌ build issues: ${{ inputs.pkg }}"
body: "Running log of build failure for ${{ inputs.pkg }}"
labels: 'build-failure'
assignees: 'jhheider'
@ -52,18 +46,7 @@ runs:
issue-number: ${{ steps.create.outputs.issue-number || fromJSON(steps.find.outputs.issues)[0].number }}
body: |
# Build failure
## ${{ inputs.projects }}
## ${{ inputs.pkg }}
### ${{ inputs.platform }}
logs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
# - uses: martialonline/workflow-status@v3
# id: status
# - uses: rtCamp/action-slack-notify@v2
# if: ${{ inputs.slack-webhook != '' }}
# env:
# SLACK_WEBHOOK: ${{ inputs.slack-webhook }}
# SLACK_CHANNEL: ${{ inputs.slack-channel }}
# SLACK_MESSAGE: new-version:${{ inputs.projects }} (${{ inputs.platform }}) ${{ steps.status.outputs.status }}
# SLACK_COLOR: ${{ steps.status.outputs.status }}

54
.github/actions/setup/action.yml vendored Normal file
View file

@ -0,0 +1,54 @@
name: pkgx/brewkit/setup-codesign
description: Codesigns macOS binaries using Apple tools
inputs:
p12-file-base64:
description: Base64 encoded p12 file
required: true
p12-password:
description: Password for p12 file
required: true
APPLE_IDENTITY:
required: false
runs:
using: composite
steps:
# - name: purge tool PATH
# run: |
# if [ -d /usr/local/bin ]; then
# tmp=$(mktemp -d)
# sudo mv /usr/local/bin $tmp
# fi
# shell: bash
- name: export APPLE_IDENTITY
run: echo 'APPLE_IDENTITY=${{inputs.identity || '-'}}' >> $GITHUB_ENV
shell: bash
# the next three steps bless our code for Apple. It might be the case they should be
# encapulated separately.
# FIXME: using an explicit commit in a PR isn't great, but the last release was almost 3 years
# ago, and we need bugfixes.
# FIXME: replace this with a pkgx script based on https://localazy.com/blog/how-to-automatically-sign-macos-apps-using-github-actions
# github has a doc with similar content, but it's not returning to me atm.
# apple-actions/import-codesign-certs will fail if the keychain already exists, so we prophylactically
# delete it if it does.
- name: Delete keychain
shell: sh
if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64
run: security delete-keychain signing_temp.keychain || true
- uses: apple-actions/import-codesign-certs@v2
if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64
with:
p12-file-base64: ${{ inputs.p12-file-base64 }}
p12-password: ${{ inputs.p12-password }}
# Needed for self-hosted runner, since it doesn't destroy itself automatically.
- name: Delete keychain
uses: webiny/action-post-run@3.0.0
if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64
with:
run: security delete-keychain signing_temp.keychain

3
.github/deno.jsonc vendored
View file

@ -10,6 +10,7 @@
},
"imports": {
"pkgx": "https://deno.land/x/libpkgx@v0.15.1/mod.ts",
"pkgx/": "https://deno.land/x/libpkgx@v0.15.1/src/"
"pkgx/": "https://deno.land/x/libpkgx@v0.15.1/src/",
"is-what": "https://deno.land/x/is_what@v4.1.15/src/index.ts"
}
}

86
.github/scripts/get-matrix.ts vendored Executable file
View file

@ -0,0 +1,86 @@
#!/usr/bin/env -S pkgx deno run -A
import { hooks, utils } from "pkgx"
import { isString, isArray } from "is-what"
const pkg = utils.pkg.parse(Deno.args[0])
const config = await get_config(pkg)
const rv = {} as Record<string, any>
for (const platform of config.platforms) {
const key = platform.replace('/', '+')
rv[key] = get_matrix(platform)
}
const ghout = Deno.env.get("GITHUB_OUTPUT")
if (ghout) {
const json = JSON.stringify(Object.values(rv))
Deno.writeTextFileSync(ghout, `matrix=${json}`, {append: true})
} else {
const json = JSON.stringify(rv, null, 2)
console.log(json)
}
///////////////////////////////////////////////////////////////////////
//TODO should be in libpkgx!
async function get_config(pkg: {project: string}) {
let { platforms, test } = await hooks.usePantry().project(pkg).yaml()
const get_platforms = (() => {
if (!platforms) return ["linux/x86-64", "linux/aarch64", "darwin/x86-64", "darwin/aarch64"]
if (isString(platforms)) platforms = [platforms]
if (!isArray(platforms)) throw new Error(`invalid platform node: ${platforms}`)
const rv = []
for (const platform of platforms) {
if (platform.match(/^(linux|darwin)\/(aarch64|x86-64)$/)) rv.push(platform)
else if (platform.match(/^(linux|darwin)$/)) rv.push(`${platform}/x86-64`, `${platform}/aarch64`)
else throw new Error(`invalid platform: ${platform}`)
}
return rv
})
const qaRequired = test?.["qa-required"] === true
return {
platforms: get_platforms(),
qaRequired
}
}
function get_matrix(platform: string) {
const name = platform.replace('/', '+')
switch (platform) {
case 'darwin/aarch64': {
const os = ["self-hosted", "macOS", "ARM64"]
return {
os, name,
"test-os": [os],
"test-container": [null],
tinyname: "²"
}}
case 'darwin/x86-64': {
const os = ["self-hosted", "macOS", "X64"]
return {
os, name,
"test-os": ["macos-11", "macos-12"],
"test-container": [null],
tinyname: "x64"
}}
case 'linux/x86-64': {
const os = {group: "linux-x86-64"}
return {
os, name,
container: "debian:buster-slim",
"test-os": [os],
"test-container": ["debian:buster-slim", "ubuntu", "archlinux"],
tinyname: "Lnx·x64"
}}
case 'linux/aarch64': {
const os = ["self-hosted", "linux", "ARM64"]
return {
os, name,
"test-os": [os],
"test-container": [null],
tinyname: "Lnx·ARM64"
}}}
}

9
.github/scripts/qa-required.ts vendored Executable file
View file

@ -0,0 +1,9 @@
#!/usr/bin/env -S pkgx deno run --allow-read
import { hooks } from "pkgx"
const project = Deno.args[0]
const yml = await hooks.usePantry().project(project).yaml()
const qaRequired = yml?.["test"]?.["qa-required"] === true
Deno.exit(qaRequired ? 0 : 1)

View file

@ -1,58 +0,0 @@
import { Installation, Package, PackageRequirement, hooks, utils } from "pkgx"
const { useCellar } = hooks
/// processes Deno.args unless STDIN is not a TTY and has input
export async function *args(): AsyncGenerator<string> {
if (Deno.isatty(Deno.stdin.rid)) {
for (const arg of Deno.args) {
if (arg[0] != '-') yield arg
}
} else {
let yielded_something = false
const buf = new Uint8Array(10)
const decode = (() => { const d = new TextDecoder(); return d.decode.bind(d) })()
let n: number | null
let txt = ''
const rx = /\s*(.*?)\s+/
while ((n = await Deno.stdin.read(buf)) !== null) {
txt += decode(buf.subarray(0, n))
while (true) {
const match = txt.match(rx)
if (!match) break
yield match[1]
txt = txt.slice(match[0].length)
yielded_something = true
}
}
if (txt) {
yield txt
} else if (!yielded_something) {
for (const arg of Deno.args) {
yield arg
}
}
}
}
export async function *pkgs(): AsyncGenerator<Package | PackageRequirement> {
for await (const arg of args()) {
const match = arg.match(/projects\/(.*)\/package.yml/)
const project = match ? match[1] : arg
yield utils.pkg.parse(project)
}
}
export async function *installs(): AsyncGenerator<Installation> {
const cellar = useCellar()
for await (const pkg of pkgs()) {
yield await cellar.resolve(pkg)
}
}
export async function toArray<T>(input: AsyncGenerator<T>) {
const rv: T[] = []
for await (const i of input) {
rv.push(i)
}
return rv
}

View file

@ -1,177 +0,0 @@
name: bottle
on:
workflow_call:
inputs:
new-version:
type: boolean
required: false
default: false
platform:
required: true
type: string
projects:
required: false
type: string
outputs:
pr:
description: "The PR number"
value: ${{ jobs.bottle.outputs.pr }}
qa-required:
description: "Whether QA is required"
value: ${{ jobs.upload.outputs.qa-required }}
jobs:
get-platform:
runs-on: ubuntu-latest
outputs:
os: ${{ steps.platform.outputs.os }}
cache-set: ${{ steps.platform.outputs.cache-set }}
available: ${{ steps.platform.outputs.available }}
steps:
- uses: pkgxdev/brewkit/actions/get-platform@v0
id: platform
with:
platform: ${{ inputs.platform }}
projects: ${{ inputs.projects }}
bottle:
needs: [get-platform]
if: ${{ !inputs.new-version || needs.get-platform.outputs.available != '' }}
# runs-on: ubuntu-latest
runs-on: ${{ fromJson(needs.get-platform.outputs.os) }}
outputs:
srcs: ${{ env.srcs }}
built: ${{ env.built }}
pr: ${{ env.PR }}
steps:
- uses: pkgxdev/brewkit/actions/setup-brewkit@v0
id: pkgx
timeout-minutes: 10
with:
prefix: ${{ github.workspace }}/.pkgx
pkgs: gnupg.org
- uses: actions/download-artifact@v3
if: ${{ inputs.new-version }}
with:
name: ${{ inputs.platform }}
- uses: pkgxdev/brewkit/actions/fetch-pr-artifacts@v0
if: ${{ !inputs.new-version }}
with:
platform: ${{ inputs.platform }}
token: ${{ github.token }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_CACHE }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: clean destination
# Note: needed when changing a directory to a symlink, for example in
# https://github.com/pkgxdev/pantry/pull/435
run: |
tar tzf $GITHUB_WORKSPACE/artifacts.tgz | \
awk '{ print length, $0 }' | \
sort -n -s -r | \
cut -d" " -f2- | \
xargs rm -rf
working-directory: ${{ github.workspace }}/.pkgx
- run: tar xzvf $GITHUB_WORKSPACE/artifacts.tgz
working-directory: ${{ github.workspace }}/.pkgx
- run: |
for file in built srcs; do
echo "$file=$(cat $file)" >> $GITHUB_ENV
done
working-directory: ${{ github.workspace }}/.pkgx
- run: echo $GPG_PRIVATE_KEY |
base64 -d |
pkgx gpg --import --batch --yes
env:
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
- uses: pkgxdev/brewkit/actions/bottle@v0
id: bottle-xz
with:
built: ${{ env.built }}
platform: ${{ inputs.platform }}
compression: xz
gpg-key-id: ${{ secrets.GPG_KEY_ID }}
env:
XDG_CACHE_HOME: ${{ github.workspace }}/.pkgx
- uses: pkgxdev/brewkit/actions/bottle@v0
id: bottle-gz
with:
built: ${{ env.built }}
platform: ${{ inputs.platform }}
compression: gz
gpg-key-id: ${{ secrets.GPG_KEY_ID }}
env:
XDG_CACHE_HOME: ${{ github.workspace }}/.pkgx
- run: |
echo ${{ steps.bottle-gz.outputs.bottles }} ${{ steps.bottle-xz.outputs.bottles }} >bottles
echo ${{ steps.bottle-gz.outputs.checksums }} ${{ steps.bottle-xz.outputs.checksums }} >checksums
echo ${{ steps.bottle-gz.outputs.signatures }} ${{ steps.bottle-xz.outputs.signatures }} >signatures
tar cf $GITHUB_WORKSPACE/artifacts.tar \
${{ steps.bottle-gz.outputs.bottles }} \
${{ steps.bottle-xz.outputs.bottles }} \
bottles checksums signatures
working-directory: ${{ github.workspace }}/.pkgx
- name: upload artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.platform }}-bottles
path: artifacts.tar
if-no-files-found: error
upload:
needs: [get-platform, bottle]
if: ${{ !inputs.new-version || needs.get-platform.outputs.available != '' }}
runs-on: ubuntu-latest
outputs:
qa-required: ${{ steps.upload.outputs.qa-required }}
steps:
- uses: pkgxdev/brewkit/actions/setup-brewkit@v0
with:
prefix: ${{ github.workspace }}/.pkgx
timeout-minutes: 10
- uses: actions/download-artifact@v3
with:
name: ${{ inputs.platform }}-bottles
- run: |
tar xvf artifacts.tar
for file in bottles checksums signatures; do
echo "$file=$(cat $file)" >>$GITHUB_ENV
done
- uses: pkgxdev/brewkit/actions/upload@v0
id: upload
with:
qa: ${{ inputs.new-version }}
pkgs: ${{ needs.bottle.outputs.built }} ${{ needs.bottle.outputs.built }}
srcs: "~"
bottles: ${{ env.bottles }}
checksums: ${{ env.checksums }}
signatures: ${{ env.signatures }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
AWS_S3_STAGING_BUCKET: ${{ secrets.AWS_S3_CACHE }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- uses: chetan/invalidate-cloudfront-action@v2
if: ${{ steps.upload.outputs.cf-invalidation-paths != '' }}
env:
PATHS: ${{ steps.upload.outputs.cf-invalidation-paths }}
DISTRIBUTION: ${{ secrets.AWS_CF_DISTRIBUTION_ID }}
AWS_REGION: us-east-1
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

View file

@ -1,161 +0,0 @@
name: build
on:
workflow_call:
inputs:
projects:
required: true
type: string
platform:
required: true
type: string
jobs:
get-platform:
runs-on: ubuntu-latest
outputs:
os: ${{ steps.platform.outputs.os }}
build-os: ${{ steps.platform.outputs.build-os }}
container: ${{ steps.platform.outputs.container }}
test-matrix: ${{ steps.platform.outputs.test-matrix }}
cache-set: ${{ steps.platform.outputs.cache-set }}
available: ${{ steps.platform.outputs.available }}
steps:
- uses: pkgxdev/brewkit/actions/get-platform@v0
id: platform
with:
platform: ${{ inputs.platform }}
projects: ${{ inputs.projects }}
build:
runs-on: ${{ fromJson(needs.get-platform.outputs.build-os) }}
container: ${{ fromJson(needs.get-platform.outputs.container) }}
needs: [get-platform]
if: ${{ needs.get-platform.outputs.available != '' }}
steps:
- uses: actions/checkout@v4
- uses: pkgxdev/brewkit/actions/setup-brewkit@v0
id: pkgx
with:
prefix: /opt
timeout-minutes: 10
- name: sanitize macOS runners
if: fromJson(needs.get-platform.outputs.build-os) == 'macos-11'
run: sudo mv /usr/local/bin/* /tmp/
# setup macOS codesigning
- uses: pkgxdev/brewkit/actions/setup-codesign@v0
if: startsWith(inputs.platform, 'darwin+') && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name
with:
p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }}
p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }}
# FIXME: this shouldn't be necessary, but it currently is for the
# gha+container build matrix entries. :/
- name: set srcroot
run: echo "SRCROOT=$GITHUB_WORKSPACE" >>$GITHUB_ENV
- run: pkg build ${{ needs.get-platform.outputs.available }}
id: build
env:
GITHUB_TOKEN: ${{ github.token }}
FORCE_UNSAFE_CONFIGURE: 1 # some configure scripts refuse to run as root
APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY || '-' }}
- run: |
ABS_PATHS=$(echo $PATHS | tr ' ' '\n' | sed -e "s_^_/opt/_" | tr '\n' ' ')
echo "paths=$ABS_PATHS" >> $GITHUB_OUTPUT
if: startsWith(inputs.platform, 'darwin+')
id: absolute-paths
env:
PATHS: ${{ steps.build.outputs.relative-paths }}
# cache data we'll need in the bottling job
- name: assemble artifact metadata
run: |
echo ${{ steps.build.outputs.pkgs }} >built
echo ${{ steps.build.outputs.srcs-relative-paths }} >srcs
working-directory: /opt
# tarring ourselves ∵ GHA-artifacts (ludicrously) lose permissions
# /ref https://github.com/actions/upload-artifact/issues/38
- name: create artifacts.tgz
run: tar czvf $GITHUB_WORKSPACE/artifacts.tgz
${{ steps.build.outputs.relative-paths }}
${{ steps.build.outputs.srcs-relative-paths }}
built srcs
working-directory: /opt
- name: upload artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.platform }}
path: artifacts.tgz
if-no-files-found: error
test:
needs: [get-platform, build]
if: ${{ needs.get-platform.outputs.available != '' }}
runs-on: ${{ matrix.platform.os }}
strategy:
matrix:
platform: ${{ fromJson(needs.get-platform.outputs.test-matrix) }}
name: test ${{ matrix.platform.name-extra }}
outputs:
HAS_SECRETS: ${{ env.HAS_SECRETS }}
container: ${{ matrix.platform.container }}
steps:
- uses: actions/checkout@v4
- uses: pkgxdev/brewkit/actions/setup-brewkit@v0
timeout-minutes: 10
- uses: actions/download-artifact@v3
with:
name: ${{ inputs.platform }}
- name: clean destination
# Note: needed when changing a directory to a symlink, for example in
# https://github.com/pkgxdev/pantry/pull/435
run: |
cd $HOME/.pkgx
tar tzf $GITHUB_WORKSPACE/artifacts.tgz | \
awk '{ print length, $0 }' | \
sort -n -s -r | \
cut -d" " -f2- | \
xargs rm -rf
- name: extract bottles
run: tar xzvf artifacts.tgz -C $HOME/.pkgx
# FIXME: this shouldn't be necessary, but it currently is for the
# ubuntu+container test matrix entries. :/
- name: set pantry path
run: echo "PKGX_PANTRY_PATH=$GITHUB_WORKSPACE" >>$GITHUB_ENV
- run: pkg test ${{ needs.get-platform.outputs.available }}
env:
GITHUB_TOKEN: ${{ github.token }}
- name: "[post]"
run: echo "HAS_SECRETS=$HAS_SECRETS" >>$GITHUB_ENV
env:
HAS_SECRETS: ${{ secrets.AWS_S3_CACHE != null }}
stage:
needs: [get-platform, test]
# this only works for PRs from our team to our repo (security! :( )
if: startsWith(github.ref, 'refs/pull/') && github.repository_owner == 'pkgxdev' && needs.test.outputs.HAS_SECRETS == 'true' && needs.get-platform.outputs.available != ''
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v3
with:
name: ${{ inputs.platform }}
- uses: pkgxdev/brewkit/actions/stage-build-artifacts@v0
with:
platform: ${{ inputs.platform }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_CACHE }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

View file

@ -6,76 +6,17 @@ on:
branches: main
paths: projects/**/*
concurrency:
group: cd/${{ github.ref }}
cancel-in-progress: true
jobs:
cd:
get-projects:
runs-on: ubuntu-latest
outputs:
has-artifacts: ${{ steps.has-artifacts.outputs.has-artifacts }}
platforms: ${{ steps.has-artifacts.outputs.platforms }}
projects: ${{ steps.diff.outputs.diff }}
steps:
- uses: actions/checkout@v4
# ^^ NOTE probably no longer required but I dont dare try to remove it
- uses: pkgxdev/brewkit/actions/setup-brewkit@v0
timeout-minutes: 10
- uses: pkgxdev/brewkit/actions/has-artifacts@v0
id: has-artifacts
with:
repo: ${{ github.repository }}
sha: ${{ github.sha }}
token: ${{github.token}}
s3-bucket: ${{ secrets.AWS_S3_CACHE }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
bottle-pr:
strategy:
fail-fast: false
matrix:
platform: ${{ fromJson(needs.cd.outputs.platforms) }}
needs: [cd]
if: ${{ needs.cd.outputs.has-artifacts == 'true' }}
uses: ./.github/workflows/bottle.yml
with:
platform: ${{ matrix.platform }}
secrets: inherit
cleanup:
needs: [bottle-pr]
runs-on: ubuntu-latest
if: ${{ needs.cd.outputs.has-artifacts == 'true' }}
env:
PR: ${{ needs.bottle.outputs.pr }}
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- run: |
REPO=$(echo ${{github.repository}} | sed -e 's_pkgxdev/__')
if test -z "$PR"; then
echo "no PR to clean up"
exit 0
fi
aws s3 rm --recursive s3://$AWS_S3_CACHE/pull-request/$REPO/$PR
env:
AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }}
PR: ${{ needs.bottle.outputs.pr }}
bottle-standalone:
runs-on: ubuntu-latest
needs: [cd]
permissions:
contents: read
actions: write
if: ${{ needs.cd.outputs.has-artifacts == 'false' }}
steps:
- uses: actions/checkout@v3
- uses: technote-space/get-diff-action@v6
id: get-diff
with:
@ -87,8 +28,12 @@ jobs:
RESULT="$RESULT $y"
done
echo "diff=$RESULT" >> $GITHUB_OUTPUT
- run: gh workflow run new-version.yml -f "projects=$PROJECTS"
if: ${{ steps.diff.outputs.diff != '' }}
env:
GH_TOKEN: ${{ github.token }}
PROJECTS: ${{ steps.diff.outputs.diff }}
pkgit:
strategy:
fail-fast: false
needs: get-projects
uses: ./.github/workflows/new-version.yml
with:
projects: ${{ needs.get-projects.outputs.diff }}
secrets: inherit

28
.github/workflows/ci-squared.yml vendored Normal file
View file

@ -0,0 +1,28 @@
name: ci-squared
run-name: ci²
on:
pull_request:
paths:
- .github/workflows/pkg.yml
- .github/workflows/pkg-for-platform.yml
- .github/workflows/ci-squared.yml
concurrency:
group: pulls/${{ github.ref }}
cancel-in-progress: true
jobs:
ci:
name: ci²
uses: ./.github/workflows/pkg.yml
strategy:
fail-fast: false
matrix:
pkg:
- r-wos.org/gti
- github.com/ggerganov/llama.cpp # has platform restrictions
with:
pkg: ${{ matrix.pkg }}
dry-run: true
secrets: inherit

View file

@ -8,14 +8,14 @@ on:
- .github/workflows/ci.yml
concurrency:
group: ${{ github.event.pull_request.head.ref }}
group: ci/${{ github.event.pull_request.head.ref }}
cancel-in-progress: true
jobs:
get-projects:
diff:
runs-on: ubuntu-latest
outputs:
diff: ${{ steps.diff.outputs.diff }}
pkgs: ${{ steps.diff.outputs.pkgs }}
steps:
- uses: actions/checkout@v3
- uses: technote-space/get-diff-action@v6
@ -25,23 +25,61 @@ jobs:
- id: diff
run: |
for x in ${{ steps.get-diff.outputs.diff }}; do
y=$(echo $x | sed 's#projects/\(.*\)/package.yml#\1#')
y=$(echo $x | sed 's#projects/\(.*\)/[^/]*#\1#')
RESULT="$RESULT $y"
done
echo "diff=$RESULT" >> $GITHUB_OUTPUT
if [ -n "$RESULT" ]; then
RESULT="$(echo $RESULT | jq -R -s -c 'split(" ")')"
else
RESULT='["zlib.net"]'
fi
echo "pkgs=$RESULT" >> $GITHUB_OUTPUT
build:
needs: diff
name: ci ${{ matrix.platform.name }} ${{matrix.pkg}}
strategy:
fail-fast: false
matrix:
pkg: ${{ fromJSON(needs.diff.outputs.pkgs) }}
platform:
- darwin+x86-64
- linux+x86-64
- darwin+aarch64
- linux+aarch64
needs: [get-projects]
uses: ./.github/workflows/build.yml
name: ${{ matrix.platform }}
- os: ["self-hosted", "macOS", "X64"]
name: x64
- os: ["self-hosted", "macOS", "ARM64"]
name: ²
- os: ["self-hosted", "linux", "ARM64"]
name: Lnx·ARM64
- os: {group: "linux-x86-64"}
container: debian:buster-slim
name: Lnx·x64
runs-on: ${{ matrix.platform.os }}
container: ${{ matrix.platform.container }}
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup
with:
projects: ${{ needs.get-projects.outputs.diff || 'zlib.net^1.2' }}
platform: ${{ matrix.platform }}
secrets: inherit
p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }}
p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }}
APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }}
- uses: pkgxdev/setup@v2
with:
PKGX_DIR: /opt
- uses: pkgxdev/brewkit/build@v1
id: build
with:
pkg: ${{ matrix.pkg }}
env:
PKGX_PANTRY_PATH: ${{ github.workspace }}
- uses: pkgxdev/brewkit/audit@v1
if: steps.build.outputs.pkgspec
env:
PKGX_PANTRY_PATH: ${{ github.workspace }}
- uses: pkgxdev/brewkit/test@v1
if: steps.build.outputs.pkgspec
env:
PKGX_PANTRY_PATH: ${{ github.workspace }}

View file

@ -1,30 +0,0 @@
# cleans up our S3 staging area if a PR is closed without merge
name: pkgx/s3-cleanup
run-name: 'cleanup: ${{ github.event.pull_request.title }}'
on:
pull_request:
types: [closed]
jobs:
cleanup:
runs-on: ubuntu-latest
if: github.event.pull_request.merged == false
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: remove staged artifacts
run: |
REPO=$(echo ${{github.repository}} | sed -e 's_pkgxdev/__')
PR=$(echo ${{github.ref}} | sed -e 's_refs/pull/\(.*\)/merge_\1_')
aws s3 rm --recursive s3://$AWS_S3_CACHE/pull-request/$REPO/$PR
if: startsWith(github.ref, 'refs/pull/') && github.repository_owner == 'pkgxdev'
env:
AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }}

View file

@ -1,6 +1,6 @@
# cleans up our issues based on tags applied
name: issue
name: librarian
run-name: "handling #${{ github.event.issue.number }}: ${{ github.event.issue.title }}"
on:

View file

@ -2,6 +2,12 @@ name: new-version
run-name: building ${{ inputs.projects }}
on:
workflow_call:
inputs:
projects:
description: eg. `foo.com=1.2.3 bar.com^2.3.4`
required: true
type: string
workflow_dispatch:
inputs:
projects:
@ -10,61 +16,35 @@ on:
type: string
jobs:
build:
strategy:
fail-fast: false
matrix:
platform:
- darwin+x86-64
- linux+x86-64
- darwin+aarch64
- linux+aarch64
uses: ./.github/workflows/build.yml
with:
projects: ${{ inputs.projects }}
platform: ${{ matrix.platform }}
secrets: inherit
bottle:
strategy:
fail-fast: false
matrix:
platform:
- darwin+x86-64
- linux+x86-64
- darwin+aarch64
- linux+aarch64
needs: [build]
uses: ./.github/workflows/bottle.yml
with:
new-version: true
platform: ${{ matrix.platform }}
projects: ${{ inputs.projects }}
secrets: inherit
request-qa:
needs: [bottle]
if: ${{ needs.bottle.outputs.qa-required != '[]' }}
divide:
runs-on: ubuntu-latest
outputs:
pkgs: ${{ steps.divide.outputs.pkgs }}
steps:
- run: |
var="$(echo ${{ inputs.projects }} | jq -R -s -c 'split(" ")')
echo "pkgs=$var" >> $GITHUB_OUTPUT
id: divide
pkgit:
needs: divide
strategy:
matrix:
project: ${{ fromJson(needs.bottle.outputs.qa-required) }}
steps:
- uses: pkgxdev/pantry/.github/actions/request-qa@main
pkg: ${{ fromJSON(needs.divide.outputs.pkgs) }}
uses: ./.github/workflows/pkg.yml
with:
project: ${{ matrix.project }}
slack-webhook: ${{ secrets.SLACK_QA_WEBHOOK }}
pkg: ${{ matrix.pkg }}
secrets: inherit
complain:
needs: [build, bottle]
needs: pkgit
if: failure()
permissions:
issues: write
runs-on: ubuntu-latest
strategy:
fail-fast: false
steps:
- uses: pkgxdev/pantry/.github/actions/complain@main
- uses: actions/checkout@v4
- uses: ./.github/actions/complain
with:
projects: ${{ inputs.projects }}
platform: ${{ inputs.platform }}
# slack-webhook: ${{ secrets.SLACK_WEBHOOK }}
# slack-channel: ${{ secrets.SLACK_CHANNEL }}
pkg: ${{ input.pkg }}
platform: ${{ inputs.name }}

210
.github/workflows/pkg-platform.yml vendored Normal file
View file

@ -0,0 +1,210 @@
name: pkg for platform
run-name: pkging ${{ inputs.project }} (${{ inputs.name }})
on:
workflow_call:
inputs:
name:
description: >
pretty name for the workflow to make GitHub Actions matrix output
more legible
required: false
type: string
os:
required: true
type: string
container:
required: false
type: string
pkg:
description: eg. `example.com@1.2.3`
required: true
type: string
dry-run:
description: dry runs do not modify bottle storage
type: boolean
default: false
test-os:
description: a JSON array of runner-names
required: true
type: string
test-container:
description: >
A JSON array of docker image names or `[null]`.
Indeed! You cannot leave this as `null` or undefined.
Sorry, GHA is not flexible enough to efficiently work around this.
required: true
type: string
secrets:
APPLE_CERTIFICATE_P12: { required: false }
APPLE_CERTIFICATE_P12_PASSWORD: { required: false }
APPLE_IDENTITY: { required: false }
GPG_KEY_ID: { required: true }
GPG_PRIVATE_KEY: { required: true }
AWS_ACCESS_KEY_ID: { required: false }
AWS_S3_BUCKET: { required: true }
AWS_SECRET_ACCESS_KEY: { required: true }
AWS_CF_DISTRIBUTION_ID: { required: true }
jobs:
build:
name: build (${{inputs.name}})
runs-on: ${{ fromJSON(inputs.os) }}
container: ${{ inputs.container }}
outputs:
pkg: ${{ steps.build.outputs.pkgspec }}
project: ${{ steps.build.outputs.project }}
version: ${{ steps.build.outputs.version }}
platform: ${{ steps.build.outputs.platform }}
arch: ${{ steps.build.outputs.arch }}
env:
PKGX_PANTRY_PATH: ${{ github.workspace }}
steps:
- uses: actions/checkout@v4
- uses: pkgxdev/setup@v2
with:
PKGX_DIR: /opt
- uses: ./.github/actions/setup
with:
p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }}
p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }}
APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }}
- uses: pkgxdev/brewkit/build@v1
id: build
with:
pkg: ${{ inputs.pkg }}
- uses: styfle/cancel-workflow-action@0.12.0
if: steps.build.outputs.noop
- uses: pkgxdev/brewkit/audit@v1
- uses: pkgxdev/brewkit/upload-build-artifact@v1
test:
name: test (${{inputs.name}}) ${{ matrix.container || ''}} ${{ join(matrix.os) }}
needs: build
strategy:
matrix:
os: ${{ fromJSON(inputs.test-os) }}
container: ${{ fromJSON(inputs.test-container) }}
runs-on: ${{ matrix.os }}
container: ${{ matrix.container }}
env:
PKGX_PANTRY_PATH: ${{ github.workspace }}
steps:
- uses: pkgxdev/setup@v2
- uses: actions/checkout@v4
- uses: pkgxdev/brewkit/download-build-artifact@v1
- uses: pkgxdev/brewkit/test@v1
with:
pkg: ${{ needs.build.outputs.pkg }}
bottle:
name: bottle (${{inputs.name}}+${{matrix.compression}})
needs: [build, test]
strategy:
matrix:
compression: [xz, gz]
runs-on: ubuntu-latest
outputs:
paths: ${{ steps.put.outputs.cf-paths }}
env:
PREFIX: ${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }}/v${{ needs.build.outputs.version }}.tar.${{ matrix.compression }}
steps:
- uses: pkgxdev/setup@v2
- uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: import GPG key
run:
echo $GPG_PRIVATE_KEY |
base64 -d |
pkgx gpg --import --batch --yes
env:
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
- uses: pkgxdev/brewkit/download-build-artifact@v1
id: dl
with:
platform: ${{ inputs.name }}
extract: false
- uses: pkgxdev/brewkit/bottle@v1
id: bottle
with:
file: ${{ steps.dl.outputs.filename }}
compression: ${{ matrix.compression }}
- name: gpg
run: pkgx gpg
--detach-sign
--armor
--output ${{ steps.bottle.outputs.filename }}.asc
--local-user ${{ secrets.GPG_KEY_ID }}
${{ steps.bottle.outputs.filename }}
- name: sha
run: pkgx
sha256sum
${{ steps.bottle.outputs.filename }} > ${{ steps.bottle.outputs.filename }}.sha256sum
- name: s3 put
run: |
aws s3 cp ${{ steps.bottle.outputs.filename }} $URL
aws s3 cp ${{ steps.bottle.outputs.filename }}.asc $URL.asc
aws s3 cp ${{ steps.bottle.outputs.filename }}.sha256sum $URL.sha256sum
echo "cf-paths=$PREFIX $PREFIX.asc $PREFIX.sha256sum" >> $GITHUB_OUTPUT
env:
URL: s3://${{ secrets.AWS_S3_BUCKET }}/${{ env.PREFIX }}
id: put
if: ${{ ! inputs.dry-run }}
versions:
runs-on: ubuntu-latest
needs: [bottle, build]
env:
DIRNAME: ${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }}
steps:
- uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- uses: pkgxdev/setup@v2
- name: generate versions.txt
run: |
aws s3 cp \
s3://${{ secrets.AWS_S3_BUCKET }}/${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }}/versions.txt \
./remote-versions.txt
echo "$SCRIPT" > script.ts
pkgx deno run -A script.ts ./remote-versions.txt ${{ needs.build.outputs.version }} > versions.txt
env:
SCRIPT: |
import SemVer, { compare } from "https://raw.githubusercontent.com/pkgxdev/libpkgx/main/src/utils/semver.ts"
const versions = Deno.readTextFileSync(Deno.args[0]).trim().split("\n").filter(x => x)
versions.push(Deno.args[1])
const out = versions.map(x => new SemVer(x)).sort(compare).join("\n")
await Deno.stdout.write(new TextEncoder().encode(out.trim()))
- name: s3 put
run: aws s3 cp versions.txt s3://${{ secrets.AWS_S3_BUCKET }}/$DIRNAME/versions.txt
if: ${{ ! inputs.dry-run }}
- name: invalidate cloudfront
run: aws cloudfront create-invalidation
--distribution-id ${{ secrets.AWS_CF_DISTRIBUTION_ID }}
--paths
/$DIRNAME/versions.txt
${{ needs.bottle.outputs.paths }}
if: ${{ ! inputs.dry-run }}

41
.github/workflows/pkg.yml vendored Normal file
View file

@ -0,0 +1,41 @@
name: pkg
run-name: pkging ${{inputs.pkg}}
on:
workflow_call:
inputs:
pkg:
required: true
type: string
dry-run:
type: boolean
default: false
jobs:
get-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
- uses: pkgxdev/setup@v2
- uses: actions/checkout@v4
- run: .github/scripts/get-matrix.ts ${{ inputs.pkg }}
id: matrix
pkgit:
needs: get-matrix
name: ${{matrix.platform.tinyname}}
strategy:
fail-fast: false
matrix:
platform: ${{ fromJSON(needs.get-matrix.outputs.matrix) }}
uses: ./.github/workflows/pkg-platform.yml
with:
pkg: ${{ inputs.pkg }}
name: ${{ matrix.platform.name }}
os: ${{ toJSON(matrix.platform.os) }}
container: ${{ matrix.platform.container }}
test-os: ${{ toJSON(matrix.platform.test-os) }}
test-container: ${{ toJSON(matrix.platform.test-container) }}
dry-run: ${{ inputs.dry-run }}
secrets: inherit

View file

@ -12,28 +12,32 @@ $ git clone https://github.com/pkgxdev/pantry
$ cd pantry
$ dev # https://docs.pkgx.sh/dev
# ^^ IMPORTANT! Otherwise the `pkg` command cannot be found
# ^^ adds brewkit to your devenv
# ^^ IMPORTANT! Otherwise the `bk` command will not be found
$ pkg init
$ bk init
# ^^ creates a “wip” package.yml
# ^^ if you already know the name, you can pass it as an argument
$ pkg edit
$ bk edit
# ^^ opens the new package.yml in your EDITOR
$ pkg build
$ bk build
# builds to `./builds`
# ^^ needs a zero permissions GITHUB_TOKEN to use the GitHub API
# either set `GITHUB_TOKEN` or run `gh auth login`
$ pkgx yq .provides <projects/$(pkg status | tr -d '[:space:]')/package.yml
$ pkgx yq .provides <projects/$(bk status | tr -d '[:space:]')/package.yml
- bin/foo
# ^^ purely demonstrative for the next step
$ pkgx foo
# ^^ anything in the `provides:` key will now run
$ pkg test
$ bk audit
# ^^ worth doing an audit to check for common pkging issues
$ bk test
# ^^ you need to write a test that verifies the package works
$ gh repo fork
@ -42,14 +46,22 @@ $ git push origin my-new-package
$ gh pr create
```
> * `pkg build` and `pkg test` take a `-L` flag to run in a Linux Docker container
> * All commands take an optional pkg-spec eg. `pkg build node@19`
> [!TIP]
> * `bk build` and `bk test` can be invoked eg. `bk docker build` to run
> inside a Docker container for Linux builds and testing
> * All commands take an optional pkg-spec eg. `bk build node@19`
> [!TIP]
> While inside the pantry `dev` environment you can run commands from any built
> packages provided you specified their `provides:` key in the `package.yml`.
> [!NOTE]
> We use a special package called [`brewkit`] to build packages both here and
> in CI/CD. `brewkit` provides the `pkg` command.
> in CI/CD. `brewkit` provides the `bk` command.
> [!IMPORTANT]
> brewkit installs the built products to `${PKGX_DIR:-$HOME/.pkgx}` which
> means they are installed to your users pkgx cache.
## GitHub Codespaces
@ -101,7 +113,7 @@ $ gh pr checkout 123
$ gh pr checkout https://github.com/pkgxdev/pantry/pull/123
# then open for editing:
$ pkg edit
$ bk edit
```

View file

@ -1,5 +1,4 @@
dependencies:
pkgx.sh/brewkit: ^0
pkgx.sh/brewkit: ^0 || ^1
env:
PKGX_PANTRY_PATH: ${{srcroot}}
SRCROOT: ${{srcroot}}

View file

@ -8,24 +8,25 @@ versions:
github: pkgxdev/brewkit
dependencies:
deno.land: '>=1.30<1.36.1'
gnu.org/bash: '*'
deno.land: ^1.37
gnu.org/bash: ^5
pkgx.sh: ^1
provides:
- bin/bk
build: |
mkdir -p {{prefix}}
rm -rf {{prefix}}/*
# ^^ because our build infra uses this pkg to build itself
for x in bin libexec lib share deno.*; do
test -e $x && mv $x "{{prefix}}"
done
test:
dependencies:
zlib.net: '*'
script:
pkg test zlib.net
# we would like to test builds but we need a `GITHUB_TOKEN` and
# (currently) we cant figure out how to set that up in CI/CD
# we cant test builds since brewkit is not designed to be invoked
# recursively and fails in the toolchain setup on Linux and fails due to
# Ruby 2 not liking unicode paths on macOS
#- bk build pkgx.sh/brewkit
#- bk test pkgx.sh/brewkit
- bk --help
- bk build --help