ivabus/pantry
1
0
Fork 0
mirror of https://github.com/ivabus/pantry synced 2024-11-26 02:15:06 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix-macho tweaks, etc.

Browse source
This commit is contained in:
Max Howell 2022-09-16 14:39:00 -04:00
parent b17592c636
commit c1a571a736
7 changed files with 303 additions and 375 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
projects/bytereef.org/mpdecimal/package.yml
View file

@ -14,6 +14,10 @@ build:
make --jobs {{ hw.concurrency }} install
rm -rf {{prefix}}/share # docs are online
if test {{hw.platform}} = darwin; then
install_name_tool -change @rpath/libmpdec.3.dylib @loader_path/libmpdec.3.dylib {{prefix}}/lib/libmpdec++.dylib
fi
test:
dependencies:
tea.xyz/gx/cc: c99

3
projects/perl.org/package.yml
View file

@ -15,7 +15,6 @@ build:
./Configure $ARGS
make --jobs {{ hw.concurrency }} install
# TODO not complete some parts are still not relocatable
cd "{{prefix}}"/bin
for x in *; do
case $x in
@ -27,7 +26,7 @@ build:
esac
done
rm -f {{prefix}}/bin/*.bak
rm -f *.bak
# relocatable fixes from: https://github.com/skaji/relocatable-perl

13
scripts/bottle-all.ts
View file

@ -18,11 +18,12 @@ import useCellar from "hooks/useCellar.ts"
const cellar = useCellar()
for await (const {path} of ls()) {
const pkg = (await cellar.resolve(path)).pkg
try {
await bottle({ path, pkg })
} catch (error) {
console.verbose({ 'bottling-failure': pkg, error })
for await (const {project} of ls()) {
for (const { path, pkg } of await cellar.ls(project)) {
try {
await bottle({ path, pkg })
} catch (error) {
console.error({ 'bottling-failure': pkg, error })
}
}
}

4
scripts/build.prefab.ts
View file

@ -4,7 +4,7 @@ import useCellar from "hooks/useCellar.ts"
import useShellEnv, { expand } from "hooks/useShellEnv.ts"
import { run, undent } from "utils"
import fix_pkg_config_files from "./fix-pkg-config-files.ts"
import fix_rpaths from "./fix-rpaths.ts"
import fix_linux_rpaths from "./fix-linux-rpaths.ts"
import usePlatform from "hooks/usePlatform.ts"
interface Options {
@ -83,7 +83,7 @@ export default async function build({ pkg, deps, prebuild, env: add_env }: Optio
})
} break
case 'linux':
await fix_rpaths(installation, [...deps.runtime, self])
await fix_linux_rpaths(installation, [...deps.runtime, self])
break
default:
throw new Error("unsupported platform")

170
scripts/fix-linux-rpaths.ts Normal file
View file

@ -0,0 +1,170 @@
import useCellar from "hooks/useCellar.ts"
import usePlatform from "hooks/usePlatform.ts"
import { Path, PackageRequirement, Installation } from "types"
import { runAndGetOutput,run } from "utils"
if (import.meta.main) {
console.log(await get_rpaths(new Path(Deno.args[0])))
}
//TODO this is not resilient to upgrades (obv)
//NOTE solution is to have the rpath reference major version (or more specific if poss)
/// fix rpaths or install names for executables and dynamic libraries
export default async function fix_rpaths(installation: Installation, pkgs: PackageRequirement[]) {
if (installation.pkg.project == "go.dev") {
console.info("skipping rpath fixes for go.dev")
// skipping because for some reason patchelf breaks the go binary
// resulting in the only output being: `Segmentation Fault`
return
}
console.info("doing SLOW rpath fixes…")
for await (const [exename, type] of exefiles(installation.path)) {
await set_rpaths(exename, type, pkgs, installation)
}
}
//TODO it's an error if any binary has bad rpaths before bottling
//NOTE we should have a `safety-inspector` step before bottling to check for this sort of thing
// and then have virtual env manager be more specific via (DY)?LD_LIBRARY_PATH
//FIXME somewhat inefficient for eg. git since git is mostly hardlinks to the same file
async function set_rpaths(exename: Path, type: 'exe' | 'lib', pkgs: PackageRequirement[], installation: Installation) {
if (usePlatform().platform != 'linux') throw new Error()
const cellar = useCellar()
const our_rpaths = await Promise.all(pkgs.map(pkg => prefix(pkg)))
const cmd = await (async () => {
//FIXME we need this for perl
// however really we should just have an escape hatch *just* for stuff that sets its own rpaths
const their_rpaths = (await runAndGetOutput({
cmd: ["patchelf", "--print-rpath", exename],
}))
.split(":")
.compactMap(x => x.chuzzle())
//^^ split has ridiculous empty string behavior
const rpaths = [...their_rpaths, ...our_rpaths]
.map(x => {
const transformed = transform(x, installation)
if (transformed.startsWith("$ORIGIN")) {
console.warn("has own special rpath", transformed)
return transformed
} else {
const rel_path = new Path(transformed).relative({ to: exename.parent() })
return `$ORIGIN/${rel_path}`
}
})
.uniq()
.join(':')
?? []
//FIXME use runtime-path since then LD_LIBRARY_PATH takes precedence which our virtual env manager requires
return ["patchelf", "--force-rpath", "--set-rpath", rpaths, exename]
})()
if (cmd.length) {
try {
await run({ cmd })
} catch (err) {
console.warn(err)
//FIXME allowing this error because on Linux:
// patchelf: cannot find section '.dynamic'. The input file is most likely statically linked
// happens with eg. gofmt
// and we don't yet have a good way to detect and skip such files
}
}
async function prefix(pkg: PackageRequirement) {
return (await cellar.resolve(pkg)).path.join("lib").string
}
}
async function get_rpaths(exename: Path): Promise<string[]> {
//GOOD_1ST_ISSUE better tokenizer for the output
const lines = (await runAndGetOutput({
cmd: ["otool", "-l", exename]
}))
.trim()
.split("\n")
const it = lines.values()
const rv: string[] = []
for (const line of it) {
if (line.trim().match(/^cmd\s+LC_RPATH$/)) {
it.next()
rv.push(it.next().value.trim().match(/^path\s+(.+)$/)[1])
console.debug(rv.slice(-1)[0])
}
}
return rv
}
//FIXME pretty slow since we execute `file` for every file
// eg. perl has hundreds of `.pm` files in its `lib`
async function* exefiles(prefix: Path): AsyncGenerator<[Path, 'exe' | 'lib']> {
for (const basename of ["bin", "lib", "libexec"]) {
const d = prefix.join(basename).isDirectory()
if (!d) continue
for await (const [exename, { isFile, isSymlink }] of d.walk()) {
if (!isFile || isSymlink) continue
const type = await exetype(exename)
if (type) yield [exename, type]
}
}
}
//FIXME lol use https://github.com/sindresorhus/file-type when we can
export async function exetype(path: Path): Promise<'exe' | 'lib' | false> {
// speed this up a bit
switch (path.extname()) {
case ".py":
case ".pyc":
case ".pl":
return false
}
const out = await runAndGetOutput({
cmd: ["file", "--mime-type", path.string]
})
const lines = out.split("\n")
const line1 = lines[0]
if (!line1) throw new Error()
const match = line1.match(/: (.*)$/)
if (!match) throw new Error()
const mime = match[1]
console.debug(mime)
switch (mime) {
case 'application/x-pie-executable':
case 'application/x-mach-binary':
case 'application/x-executable':
return 'exe'
case 'application/x-sharedlib':
return 'lib'
default:
return false
}
}
// convert a full version path to a major’d version path
// this so we are resilient to upgrades without requiring us to rewrite binaries on install
// since rewriting binaries would invalidate our signatures
function transform(input: string, installation: Installation) {
if (input.startsWith("$ORIGIN")) {
// we leave these alone, trusting the build tool knew what it was doing
return input
} else if (input.startsWith(installation.path.parent().string)) {
// don’t transform stuff that links to this actual package
return input
} else {
//FIXME not very robust lol
return input.replace(/v(\d+)\.\d+\.\d+/, 'v$1')
}
}

194
scripts/fix-machos.rb
View file

@ -10,6 +10,7 @@
#TODO file.stat.ino where file is Pathname
require 'fileutils'
require 'pathname'
require 'macho'
require 'find'
@ -22,99 +23,142 @@ $pkg_prefix = ARGV.shift
abort "arg1 should be pkg-prefix" if $pkg_prefix.empty?
$pkg_prefix = Pathname.new($pkg_prefix).realpath.to_s
def fix_id file
if file.dylib_id != file.filename
# only do work if we must
file.change_dylib_id file.filename
file.write!
$inodes = Hash.new
def arm?
def type
case RUBY_PLATFORM
when /arm/, /aarch64/ then true
else false
end
end
end
def links_to_other_tea_libs? file
file.linked_dylibs.each do |lib|
return true if lib.start_with? $tea_prefix
return true if lib.start_with? '@loader_path'
return true if lib.start_with? '@rpath'
return true if lib.start_with? '@executable_path'
class Fixer
def initialize(file)
@file = MachO::MachOFile.new(file)
@changed = false
end
end
def fix_rpaths file
#TODO remove spurious rpaths
rel_path = Pathname.new($tea_prefix).relative_path_from(Pathname.new(file.filename).parent)
rpath = "@loader_path/#{rel_path}"
return if file.rpaths.include? rpath
return unless links_to_other_tea_libs? file
file.add_rpath rpath
file.write!
end
def bad_install_names file
file.linked_dylibs.map do |lib|
if lib.start_with? '/'
if Pathname.new(lib).cleanpath.to_s.start_with? $tea_prefix
lib
end
elsif lib.start_with? '@'
puts "warn:#{file.filename}:#{lib}"
def fix
case @file.filetype
when :dylib
fix_id
fix_rpaths
fix_install_names
when :execute
fix_rpaths
fix_install_names
when :bundle
fix_rpaths
fix_install_names
when :object
# noop
else
lib
end
end.compact
end
def fix_install_names file
bad_names = bad_install_names(file)
return if bad_names.empty?
bad_names.each do |old_name|
if old_name.start_with? $pkg_prefix
new_name = Pathname.new(old_name).relative_path_from(Pathname.new(file.filename).parent)
new_name = "@loader_path/#{new_name}"
elsif old_name.start_with? '/'
new_name = Pathname.new(old_name).relative_path_from(Pathname.new($tea_prefix))
new_name = new_name.sub(%r{/v(\d+)\.\d+\.\d+/}, '/v\1/')
new_name = "@rpath/#{new_name}"
else
new_name = Pathname.new(file.filename).parent.join(old_name).cleanpath = "@loader_path/#{new_name}"
new_name = "@loader_path/#{new_name}"
throw Error("unknown filetype: #{file.filetype}: #{file.filename}")
end
file.change_install_name old_name, new_name
# M1 binaries must be signed
# changing the macho stuff invalidates the signature
# this resigns with the default adhoc signing profile
MachO.codesign!(@file.filename) if @changed and arm?
end
file.write!
end
def fix file
file = MachO::MachOFile.new(file)
case file.filetype
when :dylib
fix_id file
fix_rpaths file
fix_install_names file
when :execute
fix_rpaths file
fix_install_names file
when :bundle
fix_rpaths file
fix_install_names file
else
abort "unknown filetype: #{file.filetype}: #{file.filename}"
def fix_id
if @file.dylib_id != @file.filename
# only do work if we must
@file.change_dylib_id @file.filename
write
end
end
rescue MachO::MagicError
#noop: not a Mach-O file
def write
@file.write!
@changed = true
end
def links_to_other_tea_libs?
@file.linked_dylibs.each do |lib|
return true if lib.start_with? $tea_prefix
end
return false
end
def fix_rpaths
#TODO remove spurious rpaths
rel_path = Pathname.new($tea_prefix).relative_path_from(Pathname.new(@file.filename).parent)
rpath = "@loader_path/#{rel_path}"
return if @file.rpaths.include? rpath
return unless links_to_other_tea_libs?
@file.add_rpath rpath
write
end
def bad_install_names
@file.linked_dylibs.map do |lib|
if lib.start_with? '/'
if Pathname.new(lib).cleanpath.to_s.start_with? $tea_prefix
lib
end
elsif lib.start_with? '@'
puts "warn:#{@file.filename}:#{lib}"
# noop
else
lib
end
end.compact
end
def fix_install_names
bad_names = bad_install_names
return if bad_names.empty?
bad_names.each do |old_name|
if old_name.start_with? $pkg_prefix
new_name = Pathname.new(old_name).relative_path_from(Pathname.new(@file.filename).parent)
new_name = "@loader_path/#{new_name}"
elsif old_name.start_with? '/'
new_name = Pathname.new(old_name).relative_path_from(Pathname.new($tea_prefix))
new_name = new_name.sub(%r{/v(\d+)\.\d+\.\d+/}, '/v\1/')
new_name = "@rpath/#{new_name}"
else
# assume they are meant to be relative to lib dir
new_name = Pathname.new($pkg_prefix).join("lib").relative_path_from(Pathname.new(@file.filename).parent)
new_name = "@loader_path/#{new_name}/#{old_name}"
end
@file.change_install_name old_name, new_name
end
write
end
end
ARGV.each do |arg|
Find.find(arg) do |file|
next unless File.file? file and !File.symlink? file
abs = Pathname.getwd.join(file).to_s
fix abs
inode = File.stat(abs).ino
if $inodes[inode]
if arm?
# we have to code-sign on arm AND codesigning breaks the hard link
# so now we have to re-hardlink
puts "re-hardlinking #{abs} to #{$inodes[inode]}"
FileUtils.ln($inodes[inode], abs, :force => true)
end
# stuff like git has hardlinks to the same files
# avoid the work if we already did this inode
next
end
Fixer.new(abs).fix
$inodes[inode] = abs
rescue MachO::MagicError
#noop: not a Mach-O file
rescue MachO::TruncatedFileError
#noop: file can’t be a Mach-O file
end
end

290
scripts/fix-rpaths.ts
View file

@ -1,290 +0,0 @@
import useCellar from "hooks/useCellar.ts"
import usePlatform from "hooks/usePlatform.ts"
import { Path, PackageRequirement, Installation } from "types"
import { runAndGetOutput,run } from "utils"
if (import.meta.main) {
console.log(await get_rpaths(new Path(Deno.args[0])))
}
//TODO this is not resilient to upgrades (obv)
//NOTE solution is to have the rpath reference major version (or more specific if poss)
/// fix rpaths or install names for executables and dynamic libraries
export default async function fix_rpaths(installation: Installation, pkgs: PackageRequirement[]) {
if (installation.pkg.project == "go.dev") {
console.info("skipping rpath fixes for go.dev")
// skipping because for some reason patchelf breaks the go binary
// resulting in the only output being: `Segmentation Fault`
return
}
console.info("doing SLOW rpath fixes…")
for await (const [exename, type] of exefiles(installation.path)) {
await set_rpaths(exename, type, pkgs, installation)
}
}
const platform = usePlatform().platform
//TODO it's an error if any binary has bad rpaths before bottling
//NOTE we should have a `safety-inspector` step before bottling to check for this sort of thing
// and then have virtual env manager be more specific via (DY)?LD_LIBRARY_PATH
//FIXME somewhat inefficient for eg. git since git is mostly hardlinks to the same file
async function set_rpaths(exename: Path, type: 'exe' | 'lib', pkgs: PackageRequirement[], installation: Installation) {
const cellar = useCellar()
const our_rpaths = await Promise.all(pkgs.map(pkg => prefix(pkg)))
const cmd = await (async () => {
switch (platform) {
case 'linux': {
//FIXME we need this for perl
// however really we should just have an escape hatch *just* for stuff that sets its own rpaths
const their_rpaths = (await runAndGetOutput({
cmd: ["patchelf", "--print-rpath", exename],
}))
.split(":")
.compactMap(x => x.chuzzle())
//^^ split has ridiculous empty string behavior
const rpaths = [...their_rpaths, ...our_rpaths]
.map(x => {
const transformed = transform(x, installation)
if (transformed.startsWith("$ORIGIN")) {
console.warn("has own special rpath", transformed)
return transformed
} else {
const rel_path = new Path(transformed).relative({ to: exename.parent() })
return `$ORIGIN/${rel_path}`
}
})
.uniq()
.join(':')
?? []
//FIXME use runtime-path since then LD_LIBRARY_PATH takes precedence which our virtual env manager requires
return ["patchelf", "--force-rpath", "--set-rpath", rpaths, exename]
}
case 'darwin': {
const rpath = cellar.prefix.relative({ to: exename.parent() })
const args: (string | Path)[] = [
"install_name_tool"
]
// both types need rpath set or things linking to eg. independent libs
// will fail to find the transitive shit, especially in configure scripts
if (type == 'lib') {
// we can't trust the id the build system picked
// we need dependents to correctly link to this dylib
// and they often use the `id` to do so
// we tried setting it to @rpath/project/dir/lib but that was probematic since linked executables wouldn’t find the libs at *runtime*
//TODO possibly should transform to the major of this…
args.push(...[
"-id", exename!
])
}
for (const old_path of await get_bad_otool_listings(exename, type) ?? []) {
const dylib = await find_dylib(old_path, installation)
if (!dylib) {
console.error({old_path, installation, exename, type})
throw new Error()
}
//TODO ^^ probs should look through deps too
const new_path = (() => {
if (dylib.string.startsWith(installation.path.string)) {
const relname = dylib.relative({ to: exename.parent() })
return `@loader_path/${relname}`
} else {
const transformed = transform(dylib.string, installation)
const rel_path = new Path(transformed).relative({ to: cellar.prefix })
return `@rpath/${rel_path}`
}
})()
args.push("-change", old_path, new_path)
}
if (args.length == 1) return []
// install_name_tool barfs if the rpath already exists
if (!(await get_rpaths(exename)).includes(rpath)) {
args.push("-add_rpath", `@loader_path/${rpath}`)
}
if (args.length == 1) return []
args.push(exename.string)
return args
}
case 'windows':
throw new Error()
}
})()
if (cmd.length) {
try {
await run({ cmd })
} catch (err) {
console.warn(err)
//FIXME allowing this error because on Linux:
// patchelf: cannot find section '.dynamic'. The input file is most likely statically linked
// happens with eg. gofmt
// and we don't yet have a good way to detect and skip such files
}
}
async function prefix(pkg: PackageRequirement) {
return (await cellar.resolve(pkg)).path.join("lib").string
}
}
async function get_rpaths(exename: Path): Promise<string[]> {
//GOOD_1ST_ISSUE better tokenizer for the output
const lines = (await runAndGetOutput({
cmd: ["otool", "-l", exename]
}))
.trim()
.split("\n")
const it = lines.values()
const rv: string[] = []
for (const line of it) {
if (line.trim().match(/^cmd\s+LC_RPATH$/)) {
it.next()
rv.push(it.next().value.trim().match(/^path\s+(.+)$/)[1])
console.debug(rv.slice(-1)[0])
}
}
return rv
}
async function find_dylib(name: string, installation: Installation) {
if (name.startsWith("/")) {
return new Path(name)
} else {
for await (const [path, {name: basename}] of installation.path.join("lib").ls()) {
if (basename == name) return path
}
}
}
async function get_bad_otool_listings(exename: Path, type: 'exe' | 'lib'): Promise<string[]> {
const cellar = useCellar()
const lines = (await runAndGetOutput({cmd: ["otool", "-L", exename]}))
.trim()
.split("\n")
.slice(1)
// ^^ dylibs list themselves on 1st and 2nd lines
// NOTE that if the file is named .so then this is not true even though it is still a dylib
// NOBODY KNOW WHY
const rv: string[] = []
for (const line of lines) {
console.debug(line)
const match = line.match(/\t(.+) \(compatibility version/)
if (!match) throw new Error()
const dylib = match[1]
if (type == 'lib' && dylib.split("/").slice(-1)[0] == exename.basename()) {
// the dylib has an `id`, note that not all dylibs have ids (somehow)
// note the only place we found this true was python
//FIXME the above check is not really sufficient but ids may be expressed as relative or absolute paths
// and their base is not always clear
continue
}
if (dylib.startsWith(cellar.prefix.string)) {
rv.push(dylib)
}
if (dylib.startsWith("@")) {
console.warn("build created its own special dyld entry: " + dylib)
} else if (!dylib.startsWith("/")) {
rv.push(dylib)
}
}
return rv
}
//FIXME pretty slow since we execute `file` for every file
// eg. perl has hundreds of `.pm` files in its `lib`
async function* exefiles(prefix: Path): AsyncGenerator<[Path, 'exe' | 'lib']> {
for (const basename of ["bin", "lib", "libexec"]) {
const d = prefix.join(basename).isDirectory()
if (!d) continue
for await (const [exename, { isFile, isSymlink }] of d.walk()) {
if (!isFile || isSymlink) continue
const type = await exetype(exename)
if (type) yield [exename, type]
}
}
}
//FIXME lol use https://github.com/sindresorhus/file-type when we can
export async function exetype(path: Path): Promise<'exe' | 'lib' | false> {
// speed this up a bit
switch (path.extname()) {
case ".py":
case ".pyc":
case ".pl":
return false
}
const out = await runAndGetOutput({
cmd: ["file", "--mime-type", path.string]
})
const lines = out.split("\n")
const line1 = lines[0]
if (!line1) throw new Error()
const match = line1.match(/: (.*)$/)
if (!match) throw new Error()
const mime = match[1]
console.debug(mime)
switch (mime) {
case 'application/x-pie-executable':
case 'application/x-mach-binary':
case 'application/x-executable':
if (platform != 'darwin') return 'exe'
//FIXME on darwin the `file` utility returns x-mach-binary for both binary types
switch (path.extname()) {
case ".dylib":
case ".so": // lol python has .so files even on macOS
return 'lib'
case ".o":
return false
default:
if (path.parent().components().includes('lib')) return 'lib'
return 'exe'
}
return false
case 'application/x-sharedlib':
return 'lib'
default:
return false
}
}
// convert a full version path to a major’d version path
// this so we are resilient to upgrades without requiring us to rewrite binaries on install
// since rewriting binaries would invalidate our signatures
function transform(input: string, installation: Installation) {
if (input.startsWith("$ORIGIN")) {
// we leave these alone, trusting the build tool knew what it was doing
return input
} else if (input.startsWith(installation.path.parent().string)) {
// don’t transform stuff that links to this actual package
return input
} else {
//FIXME not very robust lol
return input.replace(/v(\d+)\.\d+\.\d+/, 'v$1')
}
}