add more pkg tests (#102)

* more-tests pt.1

* useCellar() is overkill here.

Co-authored-by: Jacob Heider <jacob@tea.xyz>

.github/workflows/build.yml

@@ -42,7 +42,7 @@ jobs:
             mkdir .git  # no git in our image
 
             #FIXME needed for gdk-pixbuf
-            apt --yes install shared-mime-info file
+            apt --yes install shared-mime-info
             ;;
           macos-11)
             # screws up a lot of build scripts
@@ -116,3 +116,12 @@ jobs:
           SLACK_MESSAGE: build job for ${{ inputs.projects }} ${{ needs.build.result == 'success' && 'succeeded' || 'failed' }}
           #FIXME SO UGLY: if one of the matrix fails, it cancels the others and returns `cancelled`
           SLACK_COLOR: ${{ needs.build.result == 'cancelled' && 'failed' || needs.build.result }}
+
+      #FIXME incredibly inefficient
+      - uses: chetan/invalidate-cloudfront-action@v2
+        env:
+          DISTRIBUTION: ${{ secrets.AWS_CF_DISTRIBUTION_ID }}
+          PATHS: "/*"
+          AWS_REGION: "us-east-1"
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/ci.yml

@@ -1,10 +1,10 @@
 name: ci
 on: pull_request
 
-concurrency:
-  #TODO group: ${{ github.ref }}
-  group: only-one-due-to-versions.txt-generation-requiring-serial-ness
-  cancel-in-progress: true
+# concurrency:
+#   #TODO group: ${{ github.ref }}
+#   group: only-one-due-to-versions.txt-generation-requiring-serial-ness
+#   cancel-in-progress: true
 
 jobs:
   get-diff:

README.md

@@ -48,7 +48,7 @@ knowledge. Please keep it tidy.
 
 |   Project   | Version |
 |-------------|---------|
-| deno.land   | ^1.18   |
+| deno.land   | ^1.23   |
 | tea.xyz     | ^0      |
 
 ## Build All

projects/sourceware.org/libffi/package.yml

@@ -17,5 +17,5 @@ test:
   dependencies:
     tea.xyz/gx/cc: c99
   script: |
-    cc -o closure {{ pkg.pantry-prefix }}/test.c -lffi
+    cc -o closure test.c -lffi
     ./closure

projects/unicode.org/package.yml

@@ -22,5 +22,18 @@ build:
       - --disable-tests
       - --enable-static
       - --with-library-bits=64
+    darwin:
+      LDFLAGS:
+      # changing install names or rpaths can't be redone for
+      - -headerpad_max_install_names
 
-test: true  #FIXME
+test:
+  script: |
+    if test -f /usr/share/dict/words; then
+      gendict --uchars /usr/share/dict/words dict
+    else
+      gendict --uchars $FIXTURE dict
+    fi
+  fixture: |
+    hello
+    world

projects/zlib.net/package.yml

@@ -19,7 +19,7 @@ test:
   dependencies:
     tea.xyz/gx/cc: c99
   script: |
-    cc {{ pkg.pantry-prefix }}/test.c -lz
+    cc test.c -lz
     OUT=$(echo "$INPUT" | ./a.out | ./a.out -d)
     test "$OUT" = "$INPUT"
   env:

scripts/README.md

@@ -0,0 +1,5 @@
+# build all
+
+```sh
+scripts/ls.ts | xargs scripts/sort.ts | xargs scripts/build.ts
+```

scripts/_lib.ts

@@ -1,6 +1,9 @@
 import usePantry from "hooks/usePantry.ts"
 import { PackageRequirement } from "types"
 
+/// this function is poorly named and does too many things
+/// sorry. Refactor is desired.
+
 export function get_build_deps(dry: Set<string>) {
   const pantry = usePantry()
 
@@ -10,7 +13,19 @@ export function get_build_deps(dry: Set<string>) {
       // we hydrate the runtime deps of any build deps since if
       // any of `dry` is a runtime dep of any build dep (obv. from another)
       // pkg in the set we are building then it needs to be sorted first
-      return [...deps.runtime, ...deps.build]
+      const rv = [...deps.runtime, ...deps.build]
+
+      // if we are building a test-dep then we need it to be built before we
+      // build `pkg` or we will not be able to test it before building the whole
+      // graph
+      for (const test_dep of deps.test) {
+        if (dry.has(test_dep.project)) {
+          rv.push(test_dep)
+        }
+      }
+
+      return rv
+
     } else {
       return deps.runtime
     }

scripts/build-all.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -eo pipefail
+
+d="$(cd "$(dirname "$0")" && pwd)"
+all=$($d/ls.ts | xargs $d/sort.ts)
+
+for x in $all
+do
+  $d/build.ts $x
+  $d/test.ts $x --magic  #FIXME be precise
+done
+
+# $d/bottle.ts $all

scripts/test-all.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -eo pipefail
+
+for x in $(scripts/ls.ts); do
+  scripts/test.ts $x --magic
+done

scripts/test.ts

@@ -19,6 +19,7 @@ import { run, undent, isPlainObject } from "utils"
 import { validatePackageRequirement } from "utils/lvl2.ts"
 import useFlags from "hooks/useFlags.ts"
 import useCellar from "hooks/useCellar.ts"
+import hydrate from "prefab/hydrate.ts"
 
 const { debug } = useFlags()
 
@@ -39,7 +40,7 @@ const self = {
   constraint: new semver.Range(pkg.version.toString())
 }
 const [yml] = await pantry.getYAML(pkg)
-const deps: PackageRequirement[] = [self, ...get_deps()]
+const deps: PackageRequirement[] = [self, ...await get_deps()]
 
 const env = await useShellEnv(deps)
 
@@ -83,11 +84,11 @@ try {
   if (!debug) tmp.rm({ recursive: true })
 }
 
-function get_deps() {
+async function get_deps() {
   const rv: PackageRequirement[] = []
   attempt(yml.dependencies)
   attempt(yml.test.dependencies)
-  return rv
+  return await hydrate(rv)
 
   function attempt(obj: PlainObject) {
     if (isPlainObject(obj))

scripts/upload.ts

@@ -14,7 +14,6 @@ args:
 import { S3 } from "s3"
 import { PackageRequirement, Path } from "types"
 import useCache from "hooks/useCache.ts"
-import useCellar from "hooks/useCellar.ts"
 import { Package, parsePackageRequirement, SemVer, semver } from "types"
 import useFlags from "hooks/useFlags.ts"
 
@@ -29,7 +28,6 @@ const s3 = new S3({
 })
 
 const bucket = s3.getBucket(Deno.env.get("AWS_S3")!)
-const cellar = useCellar()
 
 const encode = (() => { const e = new TextEncoder(); return e.encode.bind(e) })()
 
@@ -59,7 +57,10 @@ if (bottles.size !== checksums.size || ![...bottles].every(b => checksums.has(`$
 }
 
 for (const rq of bottles) {
-  const {pkg} = await cellar.resolve(rq)
+  // Packages should be a fixed version, so this should be fine:
+  const version = semver.parse(rq.constraint.raw)
+  if (!version) { throw new Error(`Incomplete package version: ${rq.constraint.raw}`)}
+  const pkg = { project: rq.project, version }
   const key = useCache().s3Key(pkg)
   const bottle = useCache().bottle(pkg)
   const checksum = new Path(`${bottle.string}.sha256sum`)