From 63632639a0fb25caaed83d57a6f0800f9e124469 Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Sat, 4 Feb 2023 18:49:21 -0500 Subject: [PATCH] use actions/cache to reduce deno.land 500s --- .github/workflows/bottle.yml | 40 +++++------ .github/workflows/build.yml | 18 ++++- .github/workflows/cd.yml | 4 ++ .github/workflows/index-data.yml | 4 ++ scripts/get-platform.ts | 120 +++++++++++++++++++------------ 5 files changed, 117 insertions(+), 69 deletions(-) diff --git a/.github/workflows/bottle.yml b/.github/workflows/bottle.yml index 6a8347a0..fac1782b 100644 --- a/.github/workflows/bottle.yml +++ b/.github/workflows/bottle.yml @@ -15,11 +15,16 @@ jobs: get-platform: runs-on: ubuntu-latest outputs: - os: ${{ steps.platform.outputs.OS }} + os: ${{ steps.platform.outputs.os }} + cache-set: ${{ steps.platform.outputs.cache-set }} steps: - uses: actions/checkout@v3 with: repository: teaxyz/pantry.core + - uses: actions/cache@v2 + with: + path: $HOME/.cache/deno + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 - run: scripts/get-platform.ts ${{ inputs.projects }} id: platform @@ -37,15 +42,10 @@ jobs: built: ${{ env.built }} pr: ${{ env.PR }} steps: - - uses: actions/checkout@v3 + - uses: actions/cache@v2 with: - path: tea.xyz/var/cli - repository: teaxyz/cli - - - uses: actions/checkout@v3 - with: - path: pantry - + path: ${{ needs.get-platform.outputs.cache-set }} + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 id: tea with: @@ -53,7 +53,7 @@ jobs: prefix: ${{ github.workspace }} - name: configure scripts PATH - run: echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH + run: echo "$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH - uses: actions/download-artifact@v3 if: ${{ inputs.new-version }} @@ -61,7 +61,7 @@ jobs: name: ${{ inputs.platform }} path: tea.xyz/var/pantry - - run: scripts/fetch-pr-artifacts.ts ${{ github.repository }} ${{ github.sha }} ${{ inputs.platform }} >>$GITHUB_ENV + - run: fetch-pr-artifacts.ts ${{ github.repository }} ${{ github.sha }} ${{ inputs.platform }} >>$GITHUB_ENV if: ${{ !inputs.new-version }} env: GITHUB_TOKEN: ${{github.token}} @@ -124,14 +124,14 @@ jobs: env: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - - run: scripts/bottle.ts $built + - run: bottle.ts $built id: bottle-xz env: COMPRESSION: xz GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - - run: scripts/bottle.ts $built + - run: bottle.ts $built id: bottle-gz env: COMPRESSION: gz @@ -167,14 +167,10 @@ jobs: run: working-directory: tea.xyz/var/pantry steps: - - uses: actions/checkout@v3 + - uses: actions/cache@v2 with: - path: tea.xyz/var/cli - repository: teaxyz/cli - - - uses: actions/checkout@v3 - with: - path: pantry + path: $HOME/.cache/deno + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 id: tea @@ -195,11 +191,11 @@ jobs: working-directory: ${{ steps.tea.outputs.prefix }} - name: configure scripts PATH - run: echo "$PWD/scripts:$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH + run: echo "$TEA_PREFIX/tea.xyz/var/pantry/scripts" >> $GITHUB_PATH - name: upload bottles id: upload - run: scripts/upload.ts + run: upload.ts --pkgs ${{ needs.bottle.outputs.built }} ${{ needs.bottle.outputs.built }} --srcs ${{ needs.bottle.outputs.srcs }} ${{ needs.bottle.outputs.srcs }} --bottles $bottles diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee2a4ed8..7251fc67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,10 +22,15 @@ jobs: build-os: ${{ steps.platform.outputs.build-os }} container: ${{ steps.platform.outputs.container }} test-matrix: ${{ steps.platform.outputs.test-matrix }} + cache-set: ${{ steps.platform.outputs.cache-set }} steps: - uses: actions/checkout@v3 with: repository: teaxyz/pantry.core + - uses: actions/cache@v2 + with: + path: $HOME/.cache/deno + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 - run: scripts/get-platform.ts ${{ inputs.projects }} id: platform @@ -39,7 +44,10 @@ jobs: needs: [get-platform] steps: - uses: actions/checkout@v3 - + - uses: actions/cache@v2 + with: + path: ${{ needs.get-platform.outputs.cache-set }} + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 id: tea with: @@ -104,6 +112,10 @@ jobs: container: ${{ matrix.platform.container }} steps: - uses: actions/checkout@v3 + - uses: actions/cache@v2 + with: + path: ${{ needs.get-platform.outputs.cache-set }} + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 - name: configure scripts PATH @@ -151,6 +163,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/cache@v2 + with: + path: $HOME/.cache/deno + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 - name: configure scripts PATH diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 6ab3c9b9..2852fbd8 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -11,6 +11,10 @@ jobs: HAS_ARTIFACTS: ${{ env.HAS_ARTIFACTS }} steps: - uses: actions/checkout@v3 + - uses: actions/cache@v2 + with: + path: $HOME/.cache/deno + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 - run: ./scripts/map-projects-to-githubs.ts env: diff --git a/.github/workflows/index-data.yml b/.github/workflows/index-data.yml index ad010b05..1242404d 100644 --- a/.github/workflows/index-data.yml +++ b/.github/workflows/index-data.yml @@ -12,6 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/cache@v2 + with: + path: $HOME/.cache/deno + key: ${{ runner.os }}-deno-deps - uses: teaxyz/setup@v0 - run: ./scripts/index-packages.ts ${{ inputs.projects }} env: diff --git a/scripts/get-platform.ts b/scripts/get-platform.ts index 35d449ab..e5f50ec0 100755 --- a/scripts/get-platform.ts +++ b/scripts/get-platform.ts @@ -11,56 +11,84 @@ args: import { panic } from "utils"; -const platform = Deno.env.get("PLATFORM") ?? panic("$PLATFORM not set") - -let os: string | string[] -let buildOs: string | string[] -let container: string | undefined -let testMatrix: { os: string | string[], container: string | undefined }[] - -switch(platform) { - case "darwin+x86-64": - os = "macos-11" - // Using GHA resources for now, until we resolve network issues with our runners - // buildOs = ["self-hosted", "macOS", "X64"] - buildOs = os - testMatrix = [{ os, container: undefined }] - break - case "darwin+aarch64": - os = ["self-hosted", "macOS", "ARM64"] - buildOs = os - testMatrix = [{ os, container: undefined }] - break - case "linux+aarch64": - os = ["self-hosted", "linux", "ARM64"] - buildOs = os - testMatrix = [{ os, container: undefined }] - break - case "linux+x86-64": - os = "ubuntu-latest" - // Using GHA resources for now, until we resolve network issues with our runners - // buildOs = ["self-hosted", "linux", "X64"] - buildOs = os - container = "ghcr.io/teaxyz/infuser:latest" - testMatrix = [ - { os, container: undefined }, - // { os: buildOs, container: undefined }, - { os, container }, - { os, container: "debian:buster-slim" }, - ] - break - default: - panic(`Invalid platform description: ${platform}`) +type Output = { + os: OS, + buildOs: OS, + container?: string, + testMatrix: { os: OS, container?: string }[] + cacheSet: string } -const output = `os=${JSON.stringify(os)}\n` + - `build-os=${JSON.stringify(buildOs)}\n` + - `container=${JSON.stringify(container)}\n` + - `test-matrix=${JSON.stringify(testMatrix)}\n` +type OS = string | string[] -Deno.stdout.write(new TextEncoder().encode(output)) +const platform = Deno.env.get("PLATFORM") ?? panic("$PLATFORM not set") + +const cacheSets = { + "darwin": "$HOME/Library/Caches/deno/deps/https/", + "linux": "$HOME/.cache/deno/deps/https/" +} + +const output: Output = (() => { + switch(platform) { + case "darwin+x86-64": { + // Using GHA resources for now, until we resolve network issues with our runners + // buildOs: ["self-hosted", "macOS", "X64"] + const os = "macos-11" + return { + os, + buildOs: os, + testMatrix: [{ os }], + cacheSet: cacheSets["darwin"] + } + } + case "darwin+aarch64": { + const os = ["self-hosted", "macOS", "ARM64"] + return { + os, + buildOs: os, + testMatrix: [{ os }], + cacheSet: cacheSets["darwin"] + } + } + case "linux+aarch64": { + const os = ["self-hosted", "linux", "ARM64"] + return { + os, + buildOs: os, + testMatrix: [{ os }], + cacheSet: cacheSets["linux"] + } + } + case "linux+x86-64": { + // Using GHA resources for now, until we resolve network issues with our runners + // buildOs: ["self-hosted", "linux", "X64"] + // testMatrix.push({ os: buildOs, container: undefined }) + const os = "ubuntu-latest" + const container = "ghcr.io/teaxyz/infuser:latest" + return { os, + buildOs: os, + container, + testMatrix: [ + { os }, + { os, container }, + { os, container: "debian:buster-slim" } + ], + cacheSet: cacheSets["linux"] + } + } + default: + panic(`Invalid platform description: ${platform}`) +}})() + +const rv = `os=${JSON.stringify(output.os)}\n` + + `build-os=${JSON.stringify(output.buildOs)}\n` + + `container=${JSON.stringify(output.container)}\n` + + `test-matrix=${JSON.stringify(output.testMatrix)}\n` + + `cache-set=${JSON.stringify(output.cacheSet)}\n` + +Deno.stdout.write(new TextEncoder().encode(rv)) if (Deno.env.get("GITHUB_OUTPUT")) { const envFile = Deno.env.get("GITHUB_OUTPUT")! - await Deno.writeTextFile(envFile, output, { append: true}) + await Deno.writeTextFile(envFile, rv, { append: true}) }