various fixes (#80)

README.md

@@ -50,3 +50,9 @@ knowledge. Please keep it tidy.
 |-------------|---------|
 | deno.land   | ^1.18   |
 | tea.xyz     | ^0      |
+
+## Build All
+
+```sh
+scripts/ls.ts | xargs scripts/sort.ts | xargs scripts/build.ts
+```

projects/gnu.org/tar/package.yml

@@ -13,6 +13,9 @@ build:
   script: |
     ./configure --prefix={{ prefix }} --disable-debug
     make --jobs {{ hw.concurrency }} install
+  env:
+    #FIXME for our docker images, but we should make a `build` user for those
+    FORCE_UNSAFE_CONFIGURE: 1
 
 test:
   script:

projects/info-zip.org/unzip/package.yml

@@ -1,6 +1,7 @@
 distributable:
   url:
-    https://downloads.sourceforge.net/project/infozip/UnZip
+    #FIXME https://downloads.sourceforge.net/project/infozip/UnZip
+    https://cytranet.dl.sourceforge.net/project/infozip/UnZip
     {{ version.major }}.x
     (latest)/UnZip
     {{ version.raw }}/unzip{{ version.major }}{{version.minor }}.tar.gz

projects/llvm.org/package.yml

@@ -34,7 +34,7 @@ build:
     ln -sf clang cc
     ln -sf clang++ c++
     ln -sf lld ld
-    for x in ar as strip objcopy nm objdump ranlib readelf; do
+    for x in ar as strip objcopy nm objdump ranlib readelf strings; do
       ln -sf llvm-$x $x
     done
 

projects/mesonbuild.com/package.yml

@@ -21,7 +21,7 @@ build:
     # tidy up
     find \
       {{prefix}}/libexec/bin \
-      -name pip\* -o -name easy_install\* -o -name activate\* -o -name Activate.ps1 \
+      -name easy_install\* -o -name activate\* -o -name Activate.ps1 \
       | xargs rm
 
 test:

projects/sourceware.org/bzip2/package.yml

@@ -28,8 +28,22 @@ build:
       --environment-overrides \
       --jobs {{ hw.concurrency }} \
       install
+
+    if test {{ hw.platform }} = linux; then
+      #TODO we want this on all platforms, but the makefile is ELF specific
+      make \
+        --file Makefile-libbz2_so \
+        --environment-overrides \
+        --jobs {{ hw.concurrency }}
+
+      mv libbz2.*.1.* {{ prefix }}/lib
+      cd {{ prefix }}/lib
+      ln -s libbz2.so.{{ version }} libbz2.so
+      ln -s libbz2.so.{{ version }} libbz2.so.{{ version.major }}
+    fi
+
   env:
-    CC: clang
+    CFLAGS: -fPIC  # so deps link without relocation issues on linux/aarch64
     PREFIX: ${{ prefix }}
 
 test:

projects/tukaani.org/xz/package.yml

@@ -1,5 +1,6 @@
 distributable:
-  url: https://tukaani.org/xz/xz-{{ version }}.tar.gz
+  #FIXME url: https://tukaani.org/xz/xz-{{ version }}.tar.gz
+  url: https://cytranet.dl.sourceforge.net/project/lzmautils/xz-{{ version }}.tar.gz
   strip-components: 1
 
 versions:

scripts/bottle.ts

@@ -6,6 +6,7 @@ args:
   - run
   - --allow-net
   - --allow-run
+  - --allow-env
   - --allow-read=/opt/
   - --allow-write=/opt/
   - --import-map={{ srcroot }}/import-map.json
@@ -16,6 +17,9 @@ import { Path } from "types"
 import useCellar from "hooks/useCellar.ts"
 import { run } from "utils"
 import useCache from "hooks/useCache.ts"
+import useFlags from "hooks/useFlags.ts"
+
+useFlags()
 
 const cellar = useCellar()
 const filesListName = 'files.txt'

scripts/build-deps.ts

@@ -0,0 +1,46 @@
+#!/usr/bin/env -S tea -E
+
+/// deps to build the provided including building all its deps too
+
+/*---
+args:
+  - deno
+  - run
+  - --allow-net
+  - --allow-run
+  - --allow-read=/opt,/Library/Developer/CommandLineTools
+  - --allow-write=/opt
+  - --allow-env
+  - --import-map={{ srcroot }}/import-map.json
+---*/
+
+import { parsePackageRequirement } from "types"
+import hydrate from "prefab/hydrate.ts"
+import useFlags from "hooks/useFlags.ts"
+import { PackageRequirement } from "types"
+import usePantry from "hooks/usePantry.ts"
+
+const flags = useFlags()
+const pantry = usePantry()
+
+const dry = Deno.args.map(project => {
+  const match = project.match(/projects\/(.*)\/package.yml/)
+  return match ? match[1] : project
+}).map(parsePackageRequirement)
+
+const wet = await hydrate(dry, get)
+const gas = wet.map(x => x.project)
+
+if (Deno.env.get("GITHUB_ACTIONS")) {
+  console.log(`::set-output name=pkgs::${gas.join(" ")}`)
+} else if (flags.json) {
+  console.log(gas)
+} else {
+  console.log(gas.join("\n"))
+}
+
+
+async function get(pkg: PackageRequirement) {
+  const { build, runtime } = await pantry.getDeps(pkg)
+  return [...build, ...runtime]
+}

scripts/build.ts

@@ -24,11 +24,9 @@ import { parsePackageRequirement } from "types"
 import hydrate from "prefab/hydrate.ts"
 import resolve from "prefab/resolve.ts"
 import { get_build_deps } from "./_lib.ts"
+import useFlags from "hooks/useFlags.ts"
 
-//<FIXME>
-import { print } from "utils"
-print("this because otherwise console.verbose is not defined lol")
-//</FIXME>
+useFlags()
 
 const pantry = usePantry()
 const cellar = useCellar()

scripts/ls.ts

@@ -0,0 +1,64 @@
+#!/usr/bin/env -S tea -E
+
+// returns all pantry entries as `[{ name, path }]`
+
+/*---
+args:
+  - deno
+  - run
+  - --allow-env
+  - --allow-read=/opt/tea.xyz/var/pantry
+  - --import-map={{ srcroot }}/import-map.json
+---*/
+
+import { Path } from "types"
+import useFlags from "hooks/useFlags.ts"
+
+const flags = useFlags()
+
+
+const prefix = new Path('/opt/tea.xyz/var/pantry/projects')
+
+//FIXME unfortunately importing executes the script below
+export async function* ls(): AsyncGenerator<Entry> {
+  for await (const path of _ls_pantry(prefix)) {
+    yield {
+      name: path.parent().relative({ to: prefix }),
+      path: path.string
+    }
+  }
+}
+
+async function* _ls_pantry(dir: Path): AsyncGenerator<Path> {
+  if (!dir.isDirectory()) throw new Error()
+
+  for await (const [path, { name, isDirectory }] of dir.ls()) {
+    if (isDirectory) {
+      for await (const x of _ls_pantry(path)) {
+        yield x
+      }
+    } else if (name === "package.yml") {
+      yield path
+    }
+  }
+}
+
+interface Entry {
+  name: string
+  path: string
+}
+
+const rv: Entry[] = []
+for await (const item of ls()) {
+  rv.push(item)
+}
+
+if (Deno.env.get("GITHUB_ACTIONS")) {
+  const projects = rv.map(x => x.name).join(":")
+  console.log(`::set-output name=projects::${projects}`)
+} else if (flags.json) {
+  const output = JSON.stringify(rv, null, 2)
+  console.log(output)
+} else {
+  console.log(rv.map(x => x.name).join("\n"))
+}

scripts/sort.ts

@@ -15,11 +15,9 @@ args:
 import { parsePackageRequirement } from "types"
 import hydrate from "prefab/hydrate.ts"
 import { get_build_deps } from "./_lib.ts"
+import useFlags from "hooks/useFlags.ts"
 
-//<FIXME>
-import { print } from "utils"
-print("this because otherwise console.verbose is not defined lol")
-//</FIXME>
+const flags = useFlags()
 
 const dry = Deno.args.map(project => {
   const match = project.match(/projects\/(.*)\/package.yml/)
@@ -31,4 +29,10 @@ const wet = await hydrate(dry, get_build_deps(set))
 const gas = wet.map(x => x.project)
   .filter(x => set.has(x)) // we're only sorting `dry` so reject the rest
 
-console.log(`::set-output name=pkgs::${gas.join(" ")}`)
+if (Deno.env.get("GITHUB_ACTIONS")) {
+  console.log(`::set-output name=pkgs::${gas.join(" ")}`)
+} else if (flags.json) {
+  console.log(gas)
+} else {
+  console.log(gas.join("\n"))
+}

scripts/test.ts

@@ -17,6 +17,9 @@ import usePantry from "hooks/usePantry.ts"
 import useShellEnv, { expand } from "hooks/useShellEnv.ts"
 import { run, undent, isPlainObject } from "utils"
 import { validatePackageRequirement } from "utils/lvl2.ts"
+import useFlags from "hooks/useFlags.ts"
+
+useFlags()
 
 //TODO install any other deps
 
@@ -69,10 +72,11 @@ function get_deps() {
   return rv
 
   function attempt(obj: PlainObject) {
-    if (isPlainObject(obj))
-    for (const [project, constraint] of Object.entries(obj)) {
-      const pkg = validatePackageRequirement({ project, constraint })
-      if (pkg) rv.push(pkg)
+    if (isPlainObject(obj)) {
+      for (const [project, constraint] of Object.entries(obj)) {
+        const pkg = validatePackageRequirement({ project, constraint })
+        if (pkg) rv.push(pkg)
+      }
     }
   }
 }

scripts/upload.ts

@@ -6,7 +6,7 @@ args:
   - run
   - --allow-net
   - --allow-read=/opt
-  - --allow-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_S3
+  - --allow-env
   - --import-map={{ srcroot }}/import-map.json
 ---*/
 
@@ -16,6 +16,9 @@ import useCache from "hooks/useCache.ts"
 import useCellar from "hooks/useCellar.ts"
 import { encodeToString } from "encodeToString"
 import { Package, parsePackageRequirement, SemVer, semver } from "types"
+import useFlags from "hooks/useFlags.ts"
+
+useFlags()
 
 if (Deno.args.length === 0) throw new Error("no args supplied")