attempt to install as little as possible with apt (#173)

scripts/brewkit/fix-elf.ts

@@ -0,0 +1,172 @@
+#!/usr/bin/env -S tea -E
+
+/*---
+args:
+  - deno
+  - run
+  - --allow-run
+  - --allow-env
+  - --allow-read
+  - --allow-write={{tea.prefix}}
+  - --import-map={{ srcroot }}/import-map.json
+dependencies:
+  nixos.org/patchelf: '*'
+  darwinsys.com/file: 5
+---*/
+
+import { useCellar } from "hooks"
+import { PackageRequirement, Installation, Package } from "types"
+import { backticks, run, host, pkg as pkgutils } from "utils"
+import Path from "path"
+
+
+if (import.meta.main) {
+  const cellar = useCellar()
+  const [installation, ...pkgs] = Deno.args
+  await fix_rpaths(
+    await cellar.resolve(new Path(installation)),
+    pkgs.map(pkgutils.parse)
+  )
+}
+
+
+//TODO this is not resilient to upgrades (obv)
+//NOTE solution is to have the rpath reference major version (or more specific if poss)
+
+/// fix rpaths or install names for executables and dynamic libraries
+export default async function fix_rpaths(installation: Installation, pkgs: (Package | PackageRequirement)[]) {
+  const skip_rpaths = [
+    "go.dev", // skipping because for some reason patchelf breaks the go binary resulting in the only output being: `Segmentation Fault`
+    "tea.xyz", // this causes tea to pass -E/--version (and everything else?) directly to deno, making it _too_ much of a wrapper.
+  ]
+  if (skip_rpaths.includes(installation.pkg.project)) {
+    console.info(`skipping rpath fixes for ${installation.pkg.project}`)
+    return
+  }
+  console.info("doing SLOW rpath fixes…")
+  for await (const [exename] of exefiles(installation.path)) {
+    await set_rpaths(exename, pkgs, installation)
+  }
+}
+
+
+//TODO it's an error if any binary has bad rpaths before bottling
+//NOTE we should have a `safety-inspector` step before bottling to check for this sort of thing
+//  and then have virtual env manager be more specific via (DY)?LD_LIBRARY_PATH
+//FIXME somewhat inefficient for eg. git since git is mostly hardlinks to the same file
+async function set_rpaths(exename: Path, pkgs: (Package | PackageRequirement)[], installation: Installation) {
+  if (host().platform != 'linux') throw new Error()
+
+  const cellar = useCellar()
+  const our_rpaths = await Promise.all(pkgs.map(pkg => prefix(pkg)))
+
+  const cmd = await (async () => {
+    //FIXME we need this for perl
+    // however really we should just have an escape hatch *just* for stuff that sets its own rpaths
+    const their_rpaths = (await backticks({
+        cmd: ["patchelf", "--print-rpath", exename],
+      }))
+      .split(":")
+      .compact(x => x.chuzzle())
+      //^^ split has ridiculous empty string behavior
+
+    const rpaths = [...their_rpaths, ...our_rpaths]
+      .map(x => {
+        const transformed = transform(x, installation)
+        if (transformed.startsWith("$ORIGIN")) {
+          console.warn("has own special rpath", transformed)
+          return transformed
+        } else {
+          const rel_path = new Path(transformed).relative({ to: exename.parent() })
+          return `$ORIGIN/${rel_path}`
+        }
+      })
+      .uniq()
+      .join(':')
+      ?? []
+
+    //FIXME use runtime-path since then LD_LIBRARY_PATH takes precedence which our virtual env manager requires
+    return ["patchelf", "--force-rpath", "--set-rpath", rpaths, exename]
+  })()
+
+  if (cmd.length) {
+    try {
+      await run({ cmd })
+    } catch (err) {
+      console.warn(err)
+      //FIXME allowing this error because on Linux:
+      //    patchelf: cannot find section '.dynamic'. The input file is most likely statically linked
+      // happens with eg. gofmt
+      // and we don't yet have a good way to detect and skip such files
+    }
+  }
+
+  async function prefix(pkg: Package | PackageRequirement) {
+    return (await cellar.resolve(pkg)).path.join("lib").string
+  }
+}
+
+//FIXME pretty slow since we execute `file` for every file
+// eg. perl has hundreds of `.pm` files in its `lib`
+async function* exefiles(prefix: Path): AsyncGenerator<[Path, 'exe' | 'lib']> {
+  for (const basename of ["bin", "lib", "libexec"]) {
+    const d = prefix.join(basename).isDirectory()
+    if (!d) continue
+    for await (const [exename, { isFile, isSymlink }] of d.walk()) {
+      if (!isFile || isSymlink) continue
+      const type = await exetype(exename)
+      if (type) yield [exename, type]
+    }
+  }
+}
+
+//FIXME lol use https://github.com/sindresorhus/file-type when we can
+export async function exetype(path: Path): Promise<'exe' | 'lib' | false> {
+  // speed this up a bit
+  switch (path.extname()) {
+    case ".py":
+    case ".pyc":
+    case ".pl":
+      return false
+  }
+
+  const out = await backticks({
+    cmd: ["file", "--mime-type", path.string]
+  })
+  const lines = out.split("\n")
+  const line1 = lines[0]
+  if (!line1) throw new Error()
+  const match = line1.match(/: (.*)$/)
+  if (!match) throw new Error()
+  const mime = match[1]
+
+  console.debug(mime)
+
+  switch (mime) {
+  case 'application/x-pie-executable':
+  case 'application/x-mach-binary':
+  case 'application/x-executable':
+    return 'exe'
+
+  case 'application/x-sharedlib':
+    return 'lib'
+  default:
+    return false
+  }
+}
+
+// convert a full version path to a major’d version path
+// this so we are resilient to upgrades without requiring us to rewrite binaries on install
+// since rewriting binaries would invalidate our signatures
+function transform(input: string, installation: Installation) {
+  if (input.startsWith("$ORIGIN")) {
+    // we leave these alone, trusting the build tool knew what it was doing
+    return input
+  } else if (input.startsWith(installation.path.parent().string)) {
+    // don’t transform stuff that links to this actual package
+    return input
+  } else {
+    //FIXME not very robust lol
+    return input.replace(/v(\d+)\.\d+\.\d+/, 'v$1')
+  }
+}

scripts/brewkit/fix-machos.rb

@@ -0,0 +1,164 @@
+#!/usr/bin/env ruby
+# ---
+# dependencies:
+#   ruby-lang.org: 2
+#   bundler.io:
+#     version: 3
+#     with:
+#       gems: [ruby-macho: 3]
+# ---
+
+#TODO file.stat.ino where file is Pathname
+
+require 'fileutils'
+require 'pathname'
+require 'macho'
+require 'find'
+
+#TODO lazy & memoized
+$tea_prefix = ENV['TEA_PREFIX'] || `tea --prefix`.chomp
+abort "set TEA_PREFIX" if $tea_prefix.empty?
+
+$pkg_prefix = ARGV.shift
+abort "arg1 should be pkg-prefix" if $pkg_prefix.empty?
+$pkg_prefix = Pathname.new($pkg_prefix).realpath.to_s
+
+$inodes = Hash.new
+
+
+def arm?
+  def type
+    case RUBY_PLATFORM
+    when /arm/, /aarch64/ then true
+    else false
+    end
+  end
+end
+
+class Fixer
+  def initialize(file)
+    @file = MachO::MachOFile.new(file)
+    @changed = false
+  end
+
+  def fix
+    case @file.filetype
+    when :dylib
+      fix_id
+      fix_rpaths
+      fix_install_names
+    when :execute
+      fix_rpaths
+      fix_install_names
+    when :bundle
+      fix_rpaths
+      fix_install_names
+    when :object
+      # noop
+    else
+      throw Error("unknown filetype: #{file.filetype}: #{file.filename}")
+    end
+
+    # M1 binaries must be signed
+    # changing the macho stuff invalidates the signature
+    # this resigns with the default adhoc signing profile
+    MachO.codesign!(@file.filename) if @changed and arm?
+  end
+
+  def fix_id
+    if @file.dylib_id != @file.filename
+      # only do work if we must
+      @file.change_dylib_id @file.filename
+      write
+    end
+  end
+
+  def write
+    @file.write!
+    @changed = true
+  end
+
+  def links_to_other_tea_libs?
+    @file.linked_dylibs.each do |lib|
+      return true if lib.start_with? $tea_prefix
+    end
+    return false
+  end
+
+  def fix_rpaths
+    #TODO remove spurious rpaths
+
+    rel_path = Pathname.new($tea_prefix).relative_path_from(Pathname.new(@file.filename).parent)
+    rpath = "@loader_path/#{rel_path}"
+
+    return if @file.rpaths.include? rpath
+    return unless links_to_other_tea_libs?
+
+    @file.add_rpath rpath
+    write
+  end
+
+  def bad_install_names
+    @file.linked_dylibs.map do |lib|
+      if lib.start_with? '/'
+        if Pathname.new(lib).cleanpath.to_s.start_with? $tea_prefix
+          lib
+        end
+      elsif lib.start_with? '@'
+        puts "warn:#{@file.filename}:#{lib}"
+        # noop
+      else
+        lib
+      end
+    end.compact
+  end
+
+  def fix_install_names
+    bad_names = bad_install_names
+    return if bad_names.empty?
+
+    bad_names.each do |old_name|
+      if old_name.start_with? $pkg_prefix
+        new_name = Pathname.new(old_name).relative_path_from(Pathname.new(@file.filename).parent)
+        new_name = "@loader_path/#{new_name}"
+      elsif old_name.start_with? '/'
+        new_name = Pathname.new(old_name).relative_path_from(Pathname.new($tea_prefix))
+        new_name = new_name.sub(%r{/v(\d+)\.\d+\.\d+/}, '/v\1/')
+        new_name = "@rpath/#{new_name}"
+      else
+        # assume they are meant to be relative to lib dir
+        new_name = Pathname.new($pkg_prefix).join("lib").relative_path_from(Pathname.new(@file.filename).parent)
+        new_name = "@loader_path/#{new_name}/#{old_name}"
+      end
+
+      @file.change_install_name old_name, new_name
+    end
+
+    write
+  end
+end
+
+ARGV.each do |arg|
+  Find.find(arg) do |file|
+    next unless File.file? file and !File.symlink? file
+    abs = Pathname.getwd.join(file).to_s
+    inode = File.stat(abs).ino
+    if $inodes[inode]
+      if arm?
+        # we have to code-sign on arm AND codesigning breaks the hard link
+        # so now we have to re-hardlink
+        puts "re-hardlinking #{abs} to #{$inodes[inode]}"
+        FileUtils.ln($inodes[inode], abs, :force => true)
+      end
+      # stuff like git has hardlinks to the same files
+      # avoid the work if we already did this inode
+      next
+    end
+    Fixer.new(abs).fix
+    $inodes[inode] = abs
+  rescue MachO::MagicError
+    #noop: not a Mach-O file
+  rescue MachO::TruncatedFileError
+    #noop: file can’t be a Mach-O file
+  end
+end

scripts/brewkit/fix-shebangs.ts

@@ -4,7 +4,6 @@
 args:
   - deno
   - run
-  - --allow-net
   - --allow-run
   - --allow-env
   - --allow-read

scripts/build/build.ts

@@ -3,10 +3,9 @@ import { link, hydrate } from "prefab"
 import { Installation, Package } from "types"
 import useShellEnv, { expand } from "hooks/useShellEnv.ts"
 import { run, undent, host, tuplize } from "utils"
+import { str as pkgstr } from "utils/pkg.ts"
 import fix_pkg_config_files from "./fix-pkg-config-files.ts"
-import fix_linux_rpaths from "./fix-linux-rpaths.ts"
 import Path from "path"
-import * as semver from "semver"
 
 const cellar = useCellar()
 const pantry = usePantry()
@@ -109,14 +108,30 @@ async function __build(pkg: Package) {
   }
 
   async function fix_binaries(installation: Installation) {
+    const prefix = usePrefix().join("tea.xyz/var/pantry/scripts/brewkit")
+    const env = {
+      TEA_PREFIX: usePrefix().string,
+    }
     switch (host().platform) {
     case 'darwin':
-      await fix_macho(installation)
-      break
-    case 'linux': {
-      const self = {project: pkg.project, constraint: new semver.Range(pkg.version.toString())}
-      await fix_linux_rpaths(installation, [...deps.runtime, self])
-    }}
+      return await run({
+        cmd: [
+          prefix.join('fix-machos.rb'),
+          installation.path,
+          ...['bin', 'lib', 'libexec'].map(x => installation.path.join(x)).filter(x => x.isDirectory())
+        ],
+        env
+     })
+    case 'linux':
+      return await run({
+        cmd: [
+          prefix.join('fix-elf.ts'),
+          installation.path,
+          ...[...deps.runtime, pkg].map(pkgstr)
+        ],
+        env
+      })
+    }
   }
 }
 
@@ -127,14 +142,3 @@ async function fetch_src(pkg: Package): Promise<Path> {
   await useSourceUnarchiver().unarchive({ dstdir, zipfile, stripComponents })
   return dstdir
 }
-
-async function fix_macho(installation: Installation) {
-  const d = new Path(new URL(import.meta.url).pathname).parent()
-  const walk = ['bin', 'lib', 'libexec'].map(x => installation.path.join(x)).filter(x => x.isDirectory())
-  await run({
-    cmd: [d.join('fix-machos.rb'), installation.path, ...walk],
-    env: {
-      TEA_PREFIX: usePrefix().string,
-    }
-  })
-}

scripts/build/fix-pkg-config-files.ts

@@ -1,5 +1,6 @@
 import { Installation } from "types"
 import Path from "path"
+import "utils"
 
 export default async function fix_pkg_config_files(installation: Installation) {
   for await (const pcfile of find_pkg_config_files(installation)) {