mirror of
https://github.com/flatcar/scripts.git
synced 2025-09-21 21:51:57 +02:00
build_library: replace package checking script
The new python script check_root uses data that portage already maintains on what shared libraries packages need or provide instead of re-scanning whatever ELF files that can be found. This is much more comprehensive but there is a bit of a transition issue for folks with long-lived SDKs: packages built with portage older than 2.2.18 do not include this data. As such for now the check is non-fatal and provides a command you can use to refresh locally installed packages. The code checking for conflicts between top level directories and /usr has also been rewritten. Both tests now are considerably faster.
This commit is contained in:
Michael Marineau
parent
9d4d888429
commit
5985b1e3d6
@ -24,3 +24,31 @@ pkg_use_enabled() {
|
||||
equery-"${BOARD}" -q uses "${pkg}" | grep -q ${grep_args}
|
||||
return $?
|
||||
}
|
||||
|
||||
# get a package's SONAMEs in soname.provided format
|
||||
pkg_soname_provides() {
|
||||
local provides p
|
||||
# We could run this command but it ugly and silly slow:
|
||||
# portageq-"${BOARD}" metadata "${BOARD_ROOT}" installed "$1" PROVIDES
|
||||
provides=$(<"${BOARD_ROOT}/var/db/pkg/$1/PROVIDES")
|
||||
|
||||
if [[ -z "$provides" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# convert:
|
||||
# x86_32: libcom_err.so.2 libss.so.2 x86_64: libcom_err.so.2 libss.so.2
|
||||
# into:
|
||||
# x86_32 libcom_err.so.2 libss.so.2
|
||||
# x86_64 libcom_err.so.2 libss.so.2
|
||||
echo -n "# $1:"
|
||||
for p in ${provides}; do
|
||||
if [[ "$p" == *: ]]; then
|
||||
echo
|
||||
echo -n "${p%:}"
|
||||
else
|
||||
echo -n " $p"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
@ -105,6 +105,11 @@ emerge_to_image() {
|
||||
|
||||
# Make sure profile.env and ld.so.cache has been generated
|
||||
sudo -E ROOT="${root_fs_dir}" env-update
|
||||
|
||||
# TODO(marineam): just call ${BUILD_LIBRARY_DIR}/check_root directly once
|
||||
# all tests are fatal, for now let the old function skip soname errors.
|
||||
ROOT="${root_fs_dir}" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
|
||||
test_image_content "${root_fs_dir}"
|
||||
}
|
||||
|
||||
# Switch to the dev or prod sub-profile
|
||||
@ -189,8 +194,9 @@ extract_docs() {
|
||||
package_provided() {
|
||||
local p profile="${BUILD_DIR}/configroot/etc/portage/profile"
|
||||
for p in "$@"; do
|
||||
info "Writing $p to package.provided"
|
||||
info "Writing $p to package.provided and soname.provided"
|
||||
echo "$p" >> "${profile}/package.provided"
|
||||
pkg_soname_provides "$p" >> "${profile}/soname.provided"
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
@ -1,164 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import glob
|
||||
|
||||
_SHARED_RE = re.compile(r"Shared library: \[([^\]]+)\]")
|
||||
_RPATH_RE = re.compile(r"Library r(?:un)?path: \[([^\]]+)\]")
|
||||
|
||||
|
||||
class CheckDependencies(object):
|
||||
"""Check that dependencies for binaries can be found in the specified dir."""
|
||||
|
||||
def _ReadLdSoConf(self, path):
|
||||
"""Parse ld.so.conf files.
|
||||
|
||||
Starting with the file at PATH (searched relative to self._root), return
|
||||
all the valid libdirs found. Include directives are handled recursively.
|
||||
|
||||
Args:
|
||||
path: the path to the ld.so.conf file (inside of the root).
|
||||
|
||||
Returns:
|
||||
A list of valid libdirs.
|
||||
"""
|
||||
|
||||
libdirs = set()
|
||||
|
||||
ld_so_conf = self._root + path
|
||||
if os.path.exists(ld_so_conf):
|
||||
f = file(ld_so_conf)
|
||||
|
||||
for line in f:
|
||||
line = line.rstrip()
|
||||
|
||||
if line.startswith("/"):
|
||||
libpath = self._root + line
|
||||
if os.path.exists(libpath):
|
||||
libdirs.add(libpath)
|
||||
|
||||
elif line.startswith("include "):
|
||||
# Includes are absolute or relative to the file itself.
|
||||
line = os.path.join(os.path.dirname(path), line[8:])
|
||||
for p in glob.glob(self._root + line):
|
||||
rel_p = "/%s" % os.path.relpath(p, self._root)
|
||||
libdirs.update(self._ReadLdSoConf(rel_p))
|
||||
|
||||
f.close()
|
||||
|
||||
return libdirs
|
||||
|
||||
def __init__(self, root, verbose=False):
|
||||
"""Initializer.
|
||||
|
||||
Args:
|
||||
root: The sysroot (e.g. "/")
|
||||
verbose: Print helpful messages.
|
||||
"""
|
||||
|
||||
self._root = root
|
||||
self._libcache = set()
|
||||
self._verbose = verbose
|
||||
|
||||
libdirs = self._ReadLdSoConf("/etc/ld.so.conf")
|
||||
if self._verbose:
|
||||
print "Library search path: %s" % " ".join(sorted(libdirs))
|
||||
|
||||
self._ReadLibs(libdirs, self._libcache)
|
||||
|
||||
def _ReadLibs(self, paths, libcache):
|
||||
for path in paths:
|
||||
if os.path.exists(path):
|
||||
for lib in os.listdir(path):
|
||||
libcache.add(lib)
|
||||
|
||||
def _ReadDependencies(self, binary):
|
||||
"""Run readelf -d on BINARY, returning (deps, rpaths)."""
|
||||
|
||||
deps = set()
|
||||
rpaths = set()
|
||||
|
||||
# Read list of dynamic libraries, ignoring error messages that occur
|
||||
# when we look at files that aren't actually libraries
|
||||
f = os.popen("readelf -d '%s' 2>/dev/null" % binary)
|
||||
for line in f:
|
||||
|
||||
# Grab dependencies
|
||||
m = _SHARED_RE.search(line)
|
||||
if m:
|
||||
deps.add(m.group(1))
|
||||
|
||||
# Add RPATHs in our search path
|
||||
m = _RPATH_RE.search(line)
|
||||
if m:
|
||||
for path in m.group(1).split(":"):
|
||||
if path.startswith("$ORIGIN"):
|
||||
rpaths.add(path.replace("$ORIGIN", os.path.dirname(binary)))
|
||||
else:
|
||||
rpaths.add(os.path.join(self._root, path[1:]))
|
||||
f.close()
|
||||
|
||||
return (deps, rpaths)
|
||||
|
||||
def CheckDependencies(self, binary):
|
||||
"""Check whether the libs for BINARY can be found in our sysroot."""
|
||||
|
||||
good = True
|
||||
|
||||
deps, rpaths = self._ReadDependencies(binary)
|
||||
|
||||
if self._verbose:
|
||||
for lib in self._libcache & deps:
|
||||
print "Found %s" % lib
|
||||
|
||||
for lib in deps - self._libcache:
|
||||
if lib[0] != "/":
|
||||
for path in rpaths:
|
||||
if os.path.exists(os.path.join(path, lib)):
|
||||
if self._verbose:
|
||||
print "Found %s" % lib
|
||||
break
|
||||
else:
|
||||
print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib)
|
||||
good = False
|
||||
else:
|
||||
full_path = os.path.join(self._root, lib[1:])
|
||||
if os.path.exists(full_path):
|
||||
if self._verbose: print "Found %s" % lib
|
||||
else:
|
||||
print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib)
|
||||
good = False
|
||||
|
||||
return good
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
print "Usage: %s [-v] sysroot binary [ binary ... ]" % sys.argv[0]
|
||||
sys.exit(1)
|
||||
|
||||
verbose = False
|
||||
if sys.argv[1] == "-v":
|
||||
verbose = True
|
||||
sys.argv = sys.argv[0:1] + sys.argv[2:]
|
||||
|
||||
checker = CheckDependencies(sys.argv[1], verbose)
|
||||
errors = False
|
||||
for binary in sys.argv[2:]:
|
||||
if verbose: print "Checking %s" % binary
|
||||
if not checker.CheckDependencies(binary):
|
||||
errors = True
|
||||
|
||||
if errors:
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
156
build_library/check_root
Executable file
156
build_library/check_root
Executable file
@ -0,0 +1,156 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
# Copyright (c) 2015 The CoreOS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import sys
|
||||
|
||||
import portage
|
||||
from portage import dep
|
||||
from portage import output
|
||||
from portage.dep.soname.SonameAtom import SonameAtom
|
||||
from portage.dep.soname.parse import parse_soname_deps
|
||||
|
||||
VARDB = portage.db[portage.root]["vartree"].dbapi
|
||||
|
||||
# TODO(marneam): possibly accept globs for arch and sonames
|
||||
IGNORE_MISSING = {
|
||||
# /usr/lib/go/src/debug/elf/testdata/gcc-386-freebsd-exec
|
||||
"dev-lang/go": [SonameAtom("x86_32", "libc.so.6")],
|
||||
"dev-lang/go-bootstrap": [SonameAtom("x86_32", "libc.so.6")],
|
||||
|
||||
# https://bugs.gentoo.org/show_bug.cgi?id=554582
|
||||
"net-firewall/ebtables": [SonameAtom("x86_64", "libebt_802_3.so"),
|
||||
SonameAtom("x86_64", "libebt_among.so"),
|
||||
SonameAtom("x86_64", "libebt_arp.so"),
|
||||
SonameAtom("x86_64", "libebt_arpreply.so"),
|
||||
SonameAtom("x86_64", "libebt_ip.so"),
|
||||
SonameAtom("x86_64", "libebt_ip6.so"),
|
||||
SonameAtom("x86_64", "libebt_limit.so"),
|
||||
SonameAtom("x86_64", "libebt_log.so"),
|
||||
SonameAtom("x86_64", "libebt_mark.so"),
|
||||
SonameAtom("x86_64", "libebt_mark_m.so"),
|
||||
SonameAtom("x86_64", "libebt_nat.so"),
|
||||
SonameAtom("x86_64", "libebt_nflog.so"),
|
||||
SonameAtom("x86_64", "libebt_pkttype.so"),
|
||||
SonameAtom("x86_64", "libebt_redirect.so"),
|
||||
SonameAtom("x86_64", "libebt_standard.so"),
|
||||
SonameAtom("x86_64", "libebt_stp.so"),
|
||||
SonameAtom("x86_64", "libebt_ulog.so"),
|
||||
SonameAtom("x86_64", "libebt_vlan.so"),
|
||||
SonameAtom("x86_64", "libebtable_broute.so"),
|
||||
SonameAtom("x86_64", "libebtable_filter.so"),
|
||||
SonameAtom("x86_64", "libebtable_nat.so")],
|
||||
}
|
||||
|
||||
USR_LINKS = ("/bin/", "/sbin/", "/lib/", "/lib32/", "/lib64/")
|
||||
|
||||
def provided_sonames():
|
||||
for cpv in VARDB.cpv_all():
|
||||
raw = VARDB.aux_get(cpv, ["PROVIDES"])[0]
|
||||
for atom in parse_soname_deps(raw):
|
||||
yield atom
|
||||
|
||||
# soname.provided in PORTAGE_CONFIGROOT
|
||||
for atom in VARDB.settings.soname_provided:
|
||||
yield atom
|
||||
|
||||
def ignore_sonames(cpv):
|
||||
for key in dep.match_to_list(cpv, IGNORE_MISSING.iterkeys()):
|
||||
for atom in IGNORE_MISSING[key]:
|
||||
yield atom
|
||||
|
||||
def missing_sonames():
|
||||
provided = frozenset(provided_sonames())
|
||||
for cpv in VARDB.cpv_all():
|
||||
raw = VARDB.aux_get(cpv, ["REQUIRES"])[0]
|
||||
requires = frozenset(parse_soname_deps(raw))
|
||||
ignore = frozenset(ignore_sonames(cpv))
|
||||
missing = requires - provided - ignore
|
||||
if missing:
|
||||
yield (cpv, missing)
|
||||
|
||||
def usr_conflicts():
|
||||
for cpv in VARDB.cpv_all():
|
||||
raw = VARDB.aux_get(cpv, ["CONTENTS"])[0]
|
||||
usr = set()
|
||||
root = set()
|
||||
|
||||
# format is:
|
||||
# obj /path goo 123
|
||||
# dir /path/foo
|
||||
# sym /this -> that 123
|
||||
# and so on
|
||||
for line in raw.split("\n"):
|
||||
if line[:4] != "obj " and line[:4] != "sym ":
|
||||
continue
|
||||
|
||||
# yeah, hard to read, trying to make it fast...
|
||||
i = line.find("/", 5)
|
||||
topdir = line[4:i+1]
|
||||
if topdir == "/usr/":
|
||||
j = line.find("/", 9)
|
||||
nextdir = line[8:j+1]
|
||||
if nextdir in USR_LINKS:
|
||||
end = line.find(" ", 8)
|
||||
usr.add(line[8:end])
|
||||
elif topdir in USR_LINKS:
|
||||
end = line.find(" ", 4)
|
||||
root.add(line[4:end])
|
||||
|
||||
conflicts = frozenset(root).intersection(usr)
|
||||
if conflicts:
|
||||
yield (cpv, conflicts)
|
||||
|
||||
def check_libs():
|
||||
ok = True
|
||||
for cpv, sonames in missing_sonames():
|
||||
error("%s is missing libraries:", cpv)
|
||||
for soname in sonames:
|
||||
error("\t%s", soname)
|
||||
ok = False
|
||||
return ok
|
||||
|
||||
def check_usr():
|
||||
ok = True
|
||||
for cpv, conflicts in usr_conflicts():
|
||||
error("%s has paths that conflict with /usr", cpv)
|
||||
for path in conflicts:
|
||||
error("\t%s", path)
|
||||
ok = False
|
||||
return ok
|
||||
|
||||
def error(fmt, *args):
|
||||
sys.stderr.write(output.red(fmt % args))
|
||||
sys.stderr.write("\n")
|
||||
|
||||
def main():
|
||||
ok = True
|
||||
check_funcs = {
|
||||
"libs": check_libs,
|
||||
"usr": check_usr,
|
||||
}
|
||||
|
||||
if not sys.stderr.isatty():
|
||||
output.nocolor()
|
||||
|
||||
checks = sys.argv[1:]
|
||||
if not checks:
|
||||
checks = check_funcs.keys()
|
||||
|
||||
for check in checks:
|
||||
func = check_funcs.get(check)
|
||||
if func:
|
||||
ok = func() and ok
|
||||
else:
|
||||
error("Unknown test name '%s'", check)
|
||||
error("Valid tests: %s", " ".join(check_funcs))
|
||||
ok = False
|
||||
|
||||
return 0 if ok else 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@ -6,30 +6,14 @@ test_image_content() {
|
||||
local root="$1"
|
||||
local returncode=0
|
||||
|
||||
if [[ -z "$BOARD" ]]; then
|
||||
die '$BOARD is undefined!'
|
||||
fi
|
||||
local portageq="portageq-$BOARD"
|
||||
|
||||
local binaries=(
|
||||
"$root/usr/boot/vmlinuz"
|
||||
"$root/bin/sed"
|
||||
)
|
||||
|
||||
for test_file in "${binaries[@]}"; do
|
||||
if [ ! -f "$test_file" ]; then
|
||||
error "test_image_content: Cannot find '$test_file'"
|
||||
returncode=1
|
||||
fi
|
||||
done
|
||||
|
||||
local libs=( $(sudo find "$root" -type f -name '*.so*') )
|
||||
|
||||
# Check that all .so files, plus the binaries, have the appropriate
|
||||
# dependencies.
|
||||
local check_deps="${BUILD_LIBRARY_DIR}/check_deps"
|
||||
if ! "$check_deps" "$root" "${binaries[@]}" "${libs[@]}"; then
|
||||
error "test_image_content: Failed dependency check"
|
||||
info "Checking $1"
|
||||
local check_root="${BUILD_LIBRARY_DIR}/check_root"
|
||||
if ! ROOT="$root" "$check_root" libs; then
|
||||
warn "test_image_content: Failed dependency check"
|
||||
warn "This may be the result of having a long-lived SDK with binary"
|
||||
warn "packages that predate portage 2.2.18. If this is the case try:"
|
||||
echo " emerge-$BOARD -agkuDN --rebuilt-binaries=y -j9 @world"
|
||||
echo " emerge-$BOARD -a --depclean"
|
||||
returncode=1
|
||||
fi
|
||||
|
||||
@ -47,16 +31,10 @@ test_image_content() {
|
||||
done
|
||||
|
||||
# Check that there are no conflicts between /* and /usr/*
|
||||
local pkgdb=$(ROOT="${root}" $portageq vdb_path)
|
||||
local files=$(awk '$2 ~ /^\/(bin|sbin|lib|lib32|lib64)\// {print $2}' \
|
||||
"${pkgdb}"/*/*/CONTENTS)
|
||||
local check_file
|
||||
for check_file in $files; do
|
||||
if grep -q "^... /usr$check_file " "${pkgdb}"/*/*/CONTENTS; then
|
||||
error "test_image_content: $check_file conflicts with /usr$check_file"
|
||||
returncode=1
|
||||
fi
|
||||
done
|
||||
if ! ROOT="$root" "$check_root" usr; then
|
||||
error "test_image_content: Failed /usr conflict check"
|
||||
returncode=1
|
||||
fi
|
||||
|
||||
return $returncode
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user