From 007c0d8772a380faf9beeae8e8720da1c2cc49a5 Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Wed, 7 Dec 2022 15:33:13 -0400 Subject: [PATCH] separate upload logic from bottling logic to deal with S3 503s ??? sync script changes amazingly close wip wip wip --- .github/workflows/build.yml | 98 +++++++++++++++++++++++++++++++++++-- scripts/bottle.ts | 2 +- scripts/upload.ts | 2 +- 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a14ff19e..9d4f73e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -143,6 +143,12 @@ jobs: - run: | apt-get update apt-get --yes install libc-dev libstdc++-8-dev libgcc-8-dev + + # in case this PR contains updates to the scripts + #TODO only do for PRs + if test "$GITHUB_REPOSITORY" = "teaxyz/pantry.core"; then + cp -rv pantry/scripts/* $(tea --prefix)/tea.xyz/var/pantry/scripts + fi if: ${{ matrix.container != '' }} - uses: actions/download-artifact@v3 @@ -154,7 +160,6 @@ jobs: - run: tea.xyz/var/pantry/scripts/test.ts ${{ inputs.projects }} bottle: - name: ${{ matrix.name }} needs: [test, build] if: ${{ inputs.upload }} runs-on: ${{ matrix.platform.os }} @@ -190,6 +195,13 @@ jobs: srcroot: tea.xyz/var/pantry prefix: ${{ github.workspace }} + - run: | + # in case this PR contains updates to the scripts + #TODO only do for PRs + if test "$GITHUB_REPOSITORY" = "teaxyz/pantry.core"; then + cp -rv $GITHUB_WORKSPACE/pantry/scripts/* $(tea --prefix)/tea.xyz/var/pantry/scripts + fi + - uses: actions/download-artifact@v3 with: name: ${{ matrix.platform.tag || matrix.platform.os }} @@ -260,14 +272,90 @@ jobs: GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + - run: | + echo ${{ steps.bottle-gz.outputs.bottles }} ${{ steps.bottle-xz.outputs.bottles }} >bottles + echo ${{ steps.bottle-gz.outputs.checksums }} ${{ steps.bottle-xz.outputs.checksums }} >checksums + echo ${{ steps.bottle-gz.outputs.signatures }} ${{ steps.bottle-xz.outputs.signatures }} >signatures + + tar cf $GITHUB_WORKSPACE/artifacts.tar \ + ${{ needs.build.outputs.srcs }} \ + ${{ steps.bottle-gz.outputs.bottles }} \ + ${{ steps.bottle-xz.outputs.bottles }} \ + bottles checksums signatures + + working-directory: ${{ steps.tea.outputs.prefix }} + + - name: upload artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.platform.name }}-bottles + path: artifacts.tar + if-no-files-found: error + + upload: + needs: [build, bottle] + if: ${{ inputs.upload }} + runs-on: ${{ matrix.platform.os }} + defaults: + run: + working-directory: tea.xyz/var/pantry + strategy: + matrix: + platform: + - os: macos-11 + name: darwin+x86-64 + - os: ubuntu-latest + name: linux+x86-64 + - os: [self-hosted, macOS, ARM64] + tag: darwin-aarch64 + name: darwin+aarch64 + - os: [self-hosted, linux, ARM64] + tag: linux-aarch64 + name: linux+aarch64 + steps: + - uses: actions/checkout@v3 + with: + path: tea.xyz/var/cli + repository: teaxyz/cli + + - uses: actions/checkout@v3 + with: + path: pantry + + - uses: teaxyz/setup@v0 + id: tea + with: + srcroot: tea.xyz/var/pantry + prefix: ${{ github.workspace }} + + - uses: actions/download-artifact@v3 + with: + name: ${{ matrix.platform.name }}-bottles + path: ${{ steps.tea.outputs.prefix }} + + - run: tar xvf artifacts.tar + working-directory: ${{ steps.tea.outputs.prefix }} + + - run: | + # in case this PR contains updates to the scripts + #TODO only do for PRs + if test "$GITHUB_REPOSITORY" = "teaxyz/pantry.core"; then + cp -rv $GITHUB_WORKSPACE/pantry/scripts/* $(tea --prefix)/tea.xyz/var/pantry/scripts + fi + + cd ${{ steps.tea.outputs.prefix }} + for file in bottles checksums signatures; do + echo "$file=$(cat $file)" >>$GITHUB_ENV + done + - name: upload bottles id: upload run: scripts/upload.ts --pkgs ${{ needs.build.outputs.built }} ${{ needs.build.outputs.built }} --srcs ${{ needs.build.outputs.srcs }} ${{ needs.build.outputs.srcs }} - --bottles ${{ steps.bottle-gz.outputs.bottles }} ${{ steps.bottle-xz.outputs.bottles }} - --checksums ${{ steps.bottle-gz.outputs.checksums }} ${{ steps.bottle-xz.outputs.checksums }} - --signatures ${{ steps.bottle-gz.outputs.signatures }} ${{ steps.bottle-xz.outputs.signatures }} + --bottles ${{ env.bottles }} + --checksums ${{ env.checksums }} + --signatures ${{ env.signatures }} env: AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -285,7 +373,7 @@ jobs: notify: if: always() - needs: [test, build, bottle] + needs: [test, build, upload] runs-on: ubuntu-latest steps: - uses: martialonline/workflow-status@v3 diff --git a/scripts/bottle.ts b/scripts/bottle.ts index 6e629c41..80c75241 100755 --- a/scripts/bottle.ts +++ b/scripts/bottle.ts @@ -57,7 +57,7 @@ if (import.meta.main) { signatures.push(signature) } - await set_output("bottles", bottles) + await set_output("bottles", bottles.map(b => b.relative({ to: usePrefix() }))) await set_output("checksums", checksums) await set_output("signatures", signatures) } diff --git a/scripts/upload.ts b/scripts/upload.ts index 1d8fd738..6549c873 100755 --- a/scripts/upload.ts +++ b/scripts/upload.ts @@ -107,7 +107,7 @@ const signatures = args_get("signatures") const rv: string[] = [] for (const [index, pkg] of pkgs.entries()) { - const bottle = new Path(bottles[index]) + const bottle = usePrefix().join(bottles[index]) const checksum = checksums[index] const signature = base64Decode(signatures[index]) const stowed = cache.decode(bottle)!