From 5985b1e3d6c5bc0c34c8cb3a6909ef06cd9a606b Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 12 Jul 2015 15:50:58 -0700 Subject: [PATCH] build_library: replace package checking script The new python script check_root uses data that portage already maintains on what shared libraries packages need or provide instead of re-scanning whatever ELF files that can be found. This is much more comprehensive but there is a bit of a transition issue for folks with long-lived SDKs: packages built with portage older than 2.2.18 do not include this data. As such for now the check is non-fatal and provides a command you can use to refresh locally installed packages. The code checking for conflicts between top level directories and /usr has also been rewritten. Both tests now are considerably faster. --- build_library/board_options.sh | 28 +++++ build_library/build_image_util.sh | 8 +- build_library/check_deps | 164 ---------------------------- build_library/check_root | 156 ++++++++++++++++++++++++++ build_library/test_image_content.sh | 46 ++------ 5 files changed, 203 insertions(+), 199 deletions(-) delete mode 100755 build_library/check_deps create mode 100755 build_library/check_root diff --git a/build_library/board_options.sh b/build_library/board_options.sh index 4a706781d4..cd3fb6c4bd 100644 --- a/build_library/board_options.sh +++ b/build_library/board_options.sh @@ -24,3 +24,31 @@ pkg_use_enabled() { equery-"${BOARD}" -q uses "${pkg}" | grep -q ${grep_args} return $? } + +# get a package's SONAMEs in soname.provided format +pkg_soname_provides() { + local provides p + # We could run this command but it ugly and silly slow: + # portageq-"${BOARD}" metadata "${BOARD_ROOT}" installed "$1" PROVIDES + provides=$(<"${BOARD_ROOT}/var/db/pkg/$1/PROVIDES") + + if [[ -z "$provides" ]]; then + return + fi + + # convert: + # x86_32: libcom_err.so.2 libss.so.2 x86_64: libcom_err.so.2 libss.so.2 + # into: + # x86_32 libcom_err.so.2 libss.so.2 + # x86_64 libcom_err.so.2 libss.so.2 + echo -n "# $1:" + for p in ${provides}; do + if [[ "$p" == *: ]]; then + echo + echo -n "${p%:}" + else + echo -n " $p" + fi + done + echo +} diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index fb0a20bd6c..dc331f632f 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -105,6 +105,11 @@ emerge_to_image() { # Make sure profile.env and ld.so.cache has been generated sudo -E ROOT="${root_fs_dir}" env-update + + # TODO(marineam): just call ${BUILD_LIBRARY_DIR}/check_root directly once + # all tests are fatal, for now let the old function skip soname errors. + ROOT="${root_fs_dir}" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ + test_image_content "${root_fs_dir}" } # Switch to the dev or prod sub-profile @@ -189,8 +194,9 @@ extract_docs() { package_provided() { local p profile="${BUILD_DIR}/configroot/etc/portage/profile" for p in "$@"; do - info "Writing $p to package.provided" + info "Writing $p to package.provided and soname.provided" echo "$p" >> "${profile}/package.provided" + pkg_soname_provides "$p" >> "${profile}/soname.provided" done } diff --git a/build_library/check_deps b/build_library/check_deps deleted file mode 100755 index e72975c250..0000000000 --- a/build_library/check_deps +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import os -import re -import sys -import glob - -_SHARED_RE = re.compile(r"Shared library: \[([^\]]+)\]") -_RPATH_RE = re.compile(r"Library r(?:un)?path: \[([^\]]+)\]") - - -class CheckDependencies(object): - """Check that dependencies for binaries can be found in the specified dir.""" - - def _ReadLdSoConf(self, path): - """Parse ld.so.conf files. - - Starting with the file at PATH (searched relative to self._root), return - all the valid libdirs found. Include directives are handled recursively. - - Args: - path: the path to the ld.so.conf file (inside of the root). - - Returns: - A list of valid libdirs. - """ - - libdirs = set() - - ld_so_conf = self._root + path - if os.path.exists(ld_so_conf): - f = file(ld_so_conf) - - for line in f: - line = line.rstrip() - - if line.startswith("/"): - libpath = self._root + line - if os.path.exists(libpath): - libdirs.add(libpath) - - elif line.startswith("include "): - # Includes are absolute or relative to the file itself. - line = os.path.join(os.path.dirname(path), line[8:]) - for p in glob.glob(self._root + line): - rel_p = "/%s" % os.path.relpath(p, self._root) - libdirs.update(self._ReadLdSoConf(rel_p)) - - f.close() - - return libdirs - - def __init__(self, root, verbose=False): - """Initializer. - - Args: - root: The sysroot (e.g. "/") - verbose: Print helpful messages. - """ - - self._root = root - self._libcache = set() - self._verbose = verbose - - libdirs = self._ReadLdSoConf("/etc/ld.so.conf") - if self._verbose: - print "Library search path: %s" % " ".join(sorted(libdirs)) - - self._ReadLibs(libdirs, self._libcache) - - def _ReadLibs(self, paths, libcache): - for path in paths: - if os.path.exists(path): - for lib in os.listdir(path): - libcache.add(lib) - - def _ReadDependencies(self, binary): - """Run readelf -d on BINARY, returning (deps, rpaths).""" - - deps = set() - rpaths = set() - - # Read list of dynamic libraries, ignoring error messages that occur - # when we look at files that aren't actually libraries - f = os.popen("readelf -d '%s' 2>/dev/null" % binary) - for line in f: - - # Grab dependencies - m = _SHARED_RE.search(line) - if m: - deps.add(m.group(1)) - - # Add RPATHs in our search path - m = _RPATH_RE.search(line) - if m: - for path in m.group(1).split(":"): - if path.startswith("$ORIGIN"): - rpaths.add(path.replace("$ORIGIN", os.path.dirname(binary))) - else: - rpaths.add(os.path.join(self._root, path[1:])) - f.close() - - return (deps, rpaths) - - def CheckDependencies(self, binary): - """Check whether the libs for BINARY can be found in our sysroot.""" - - good = True - - deps, rpaths = self._ReadDependencies(binary) - - if self._verbose: - for lib in self._libcache & deps: - print "Found %s" % lib - - for lib in deps - self._libcache: - if lib[0] != "/": - for path in rpaths: - if os.path.exists(os.path.join(path, lib)): - if self._verbose: - print "Found %s" % lib - break - else: - print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib) - good = False - else: - full_path = os.path.join(self._root, lib[1:]) - if os.path.exists(full_path): - if self._verbose: print "Found %s" % lib - else: - print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib) - good = False - - return good - - -def main(): - if len(sys.argv) < 3: - print "Usage: %s [-v] sysroot binary [ binary ... ]" % sys.argv[0] - sys.exit(1) - - verbose = False - if sys.argv[1] == "-v": - verbose = True - sys.argv = sys.argv[0:1] + sys.argv[2:] - - checker = CheckDependencies(sys.argv[1], verbose) - errors = False - for binary in sys.argv[2:]: - if verbose: print "Checking %s" % binary - if not checker.CheckDependencies(binary): - errors = True - - if errors: - sys.exit(1) - else: - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/build_library/check_root b/build_library/check_root new file mode 100755 index 0000000000..b56f5dfd38 --- /dev/null +++ b/build_library/check_root @@ -0,0 +1,156 @@ +#!/usr/bin/python + +# Copyright (c) 2015 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import unicode_literals + +import sys + +import portage +from portage import dep +from portage import output +from portage.dep.soname.SonameAtom import SonameAtom +from portage.dep.soname.parse import parse_soname_deps + +VARDB = portage.db[portage.root]["vartree"].dbapi + +# TODO(marneam): possibly accept globs for arch and sonames +IGNORE_MISSING = { + # /usr/lib/go/src/debug/elf/testdata/gcc-386-freebsd-exec + "dev-lang/go": [SonameAtom("x86_32", "libc.so.6")], + "dev-lang/go-bootstrap": [SonameAtom("x86_32", "libc.so.6")], + + # https://bugs.gentoo.org/show_bug.cgi?id=554582 + "net-firewall/ebtables": [SonameAtom("x86_64", "libebt_802_3.so"), + SonameAtom("x86_64", "libebt_among.so"), + SonameAtom("x86_64", "libebt_arp.so"), + SonameAtom("x86_64", "libebt_arpreply.so"), + SonameAtom("x86_64", "libebt_ip.so"), + SonameAtom("x86_64", "libebt_ip6.so"), + SonameAtom("x86_64", "libebt_limit.so"), + SonameAtom("x86_64", "libebt_log.so"), + SonameAtom("x86_64", "libebt_mark.so"), + SonameAtom("x86_64", "libebt_mark_m.so"), + SonameAtom("x86_64", "libebt_nat.so"), + SonameAtom("x86_64", "libebt_nflog.so"), + SonameAtom("x86_64", "libebt_pkttype.so"), + SonameAtom("x86_64", "libebt_redirect.so"), + SonameAtom("x86_64", "libebt_standard.so"), + SonameAtom("x86_64", "libebt_stp.so"), + SonameAtom("x86_64", "libebt_ulog.so"), + SonameAtom("x86_64", "libebt_vlan.so"), + SonameAtom("x86_64", "libebtable_broute.so"), + SonameAtom("x86_64", "libebtable_filter.so"), + SonameAtom("x86_64", "libebtable_nat.so")], +} + +USR_LINKS = ("/bin/", "/sbin/", "/lib/", "/lib32/", "/lib64/") + +def provided_sonames(): + for cpv in VARDB.cpv_all(): + raw = VARDB.aux_get(cpv, ["PROVIDES"])[0] + for atom in parse_soname_deps(raw): + yield atom + + # soname.provided in PORTAGE_CONFIGROOT + for atom in VARDB.settings.soname_provided: + yield atom + +def ignore_sonames(cpv): + for key in dep.match_to_list(cpv, IGNORE_MISSING.iterkeys()): + for atom in IGNORE_MISSING[key]: + yield atom + +def missing_sonames(): + provided = frozenset(provided_sonames()) + for cpv in VARDB.cpv_all(): + raw = VARDB.aux_get(cpv, ["REQUIRES"])[0] + requires = frozenset(parse_soname_deps(raw)) + ignore = frozenset(ignore_sonames(cpv)) + missing = requires - provided - ignore + if missing: + yield (cpv, missing) + +def usr_conflicts(): + for cpv in VARDB.cpv_all(): + raw = VARDB.aux_get(cpv, ["CONTENTS"])[0] + usr = set() + root = set() + + # format is: + # obj /path goo 123 + # dir /path/foo + # sym /this -> that 123 + # and so on + for line in raw.split("\n"): + if line[:4] != "obj " and line[:4] != "sym ": + continue + + # yeah, hard to read, trying to make it fast... + i = line.find("/", 5) + topdir = line[4:i+1] + if topdir == "/usr/": + j = line.find("/", 9) + nextdir = line[8:j+1] + if nextdir in USR_LINKS: + end = line.find(" ", 8) + usr.add(line[8:end]) + elif topdir in USR_LINKS: + end = line.find(" ", 4) + root.add(line[4:end]) + + conflicts = frozenset(root).intersection(usr) + if conflicts: + yield (cpv, conflicts) + +def check_libs(): + ok = True + for cpv, sonames in missing_sonames(): + error("%s is missing libraries:", cpv) + for soname in sonames: + error("\t%s", soname) + ok = False + return ok + +def check_usr(): + ok = True + for cpv, conflicts in usr_conflicts(): + error("%s has paths that conflict with /usr", cpv) + for path in conflicts: + error("\t%s", path) + ok = False + return ok + +def error(fmt, *args): + sys.stderr.write(output.red(fmt % args)) + sys.stderr.write("\n") + +def main(): + ok = True + check_funcs = { + "libs": check_libs, + "usr": check_usr, + } + + if not sys.stderr.isatty(): + output.nocolor() + + checks = sys.argv[1:] + if not checks: + checks = check_funcs.keys() + + for check in checks: + func = check_funcs.get(check) + if func: + ok = func() and ok + else: + error("Unknown test name '%s'", check) + error("Valid tests: %s", " ".join(check_funcs)) + ok = False + + return 0 if ok else 1 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_library/test_image_content.sh b/build_library/test_image_content.sh index f67ba93fdf..1b769dbd38 100644 --- a/build_library/test_image_content.sh +++ b/build_library/test_image_content.sh @@ -6,30 +6,14 @@ test_image_content() { local root="$1" local returncode=0 - if [[ -z "$BOARD" ]]; then - die '$BOARD is undefined!' - fi - local portageq="portageq-$BOARD" - - local binaries=( - "$root/usr/boot/vmlinuz" - "$root/bin/sed" - ) - - for test_file in "${binaries[@]}"; do - if [ ! -f "$test_file" ]; then - error "test_image_content: Cannot find '$test_file'" - returncode=1 - fi - done - - local libs=( $(sudo find "$root" -type f -name '*.so*') ) - - # Check that all .so files, plus the binaries, have the appropriate - # dependencies. - local check_deps="${BUILD_LIBRARY_DIR}/check_deps" - if ! "$check_deps" "$root" "${binaries[@]}" "${libs[@]}"; then - error "test_image_content: Failed dependency check" + info "Checking $1" + local check_root="${BUILD_LIBRARY_DIR}/check_root" + if ! ROOT="$root" "$check_root" libs; then + warn "test_image_content: Failed dependency check" + warn "This may be the result of having a long-lived SDK with binary" + warn "packages that predate portage 2.2.18. If this is the case try:" + echo " emerge-$BOARD -agkuDN --rebuilt-binaries=y -j9 @world" + echo " emerge-$BOARD -a --depclean" returncode=1 fi @@ -47,16 +31,10 @@ test_image_content() { done # Check that there are no conflicts between /* and /usr/* - local pkgdb=$(ROOT="${root}" $portageq vdb_path) - local files=$(awk '$2 ~ /^\/(bin|sbin|lib|lib32|lib64)\// {print $2}' \ - "${pkgdb}"/*/*/CONTENTS) - local check_file - for check_file in $files; do - if grep -q "^... /usr$check_file " "${pkgdb}"/*/*/CONTENTS; then - error "test_image_content: $check_file conflicts with /usr$check_file" - returncode=1 - fi - done + if ! ROOT="$root" "$check_root" usr; then + error "test_image_content: Failed /usr conflict check" + returncode=1 + fi return $returncode }