From 9d4d88842966d815bbf4a559e46b2311cc720016 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 12 Jul 2015 15:47:34 -0700 Subject: [PATCH 1/2] update_chroot/build_packages: trigger @preserved-rebuild Rebuilds packages that are linked against old libraries that have been upgraded or removed from the system. Skipping this can lead to shared library checks looking ok in the build root but then built images have broken library dependencies. --- build_packages | 7 +++++++ update_chroot | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/build_packages b/build_packages index b4691509ac..ec45617de9 100755 --- a/build_packages +++ b/build_packages @@ -125,6 +125,7 @@ fi # Setup all the emerge command/flags. EMERGE_FLAGS=( -uDNv --backtrack=30 --select ) +REBUILD_FLAGS=() EMERGE_CMD=( "emerge-${FLAGS_board}" ) if [[ "${FLAGS_fetchonly}" -eq "${FLAGS_TRUE}" ]]; then EMERGE_CMD+=( --fetchonly ) @@ -144,10 +145,12 @@ if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" || # Only update toolchain when binpkgs are available. EMERGE_FLAGS+=( $(get_binonly_args) ) + REBUILD_FLAGS+=( $(get_binonly_args) ) fi if [[ "${FLAGS_jobs}" -ne -1 ]]; then EMERGE_FLAGS+=( --jobs=${FLAGS_jobs} ) + REBUILD_FLAGS+=( --jobs=${FLAGS_jobs} ) fi if [[ "${FLAGS_rebuild}" -eq "${FLAGS_TRUE}" ]]; then @@ -222,6 +225,10 @@ info "Merging board packages now" sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \ @system coreos-devel/board-packages +if "portageq-${BOARD}" list_preserved_libs "${BOARD_ROOT}" >/dev/null; then + sudo -E "${EMERGE_CMD[@]}" "${REBUILD_FLAGS[@]}" @preserved-rebuild +fi + info "Checking build root" test_image_content "${BOARD_ROOT}" diff --git a/update_chroot b/update_chroot index 594eef2e01..b496c1a6b7 100755 --- a/update_chroot +++ b/update_chroot @@ -169,6 +169,7 @@ done "${BUILD_LIBRARY_DIR}/set_lsb_release" --root / EMERGE_FLAGS="-uNv --with-bdeps=y --select" +REBUILD_FLAGS="" if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then EMERGE_FLAGS="${EMERGE_FLAGS} --usepkg" if [ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]; then @@ -177,10 +178,12 @@ if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then # Only update toolchain when binpkgs are available. EMERGE_FLAGS+=" $(get_binonly_args $(get_chost_list))" + REBUILD_FLAGS+=" $(get_binonly_args $(get_chost_list))" fi if [[ "${FLAGS_jobs}" -ne -1 ]]; then EMERGE_FLAGS+=" --jobs=${FLAGS_jobs}" + REBUILD_FLAGS+=" --jobs=${FLAGS_jobs}" fi # Perform an update of coreos-devel/sdk-depends and world in the chroot. @@ -223,6 +226,11 @@ info "Updating all SDK packages" sudo -E ${EMERGE_CMD} ${EMERGE_FLAGS} \ coreos-devel/sdk-depends world +if portageq list_preserved_libs / >/dev/null; then + info "Rebuilding packages linked against old libraries" + sudo -E ${EMERGE_CMD} ${REBUILD_FLAGS} @preserved-rebuild +fi + # Automatically discard all CONFIG_PROTECT'ed files. Those that are # protected should not be overwritten until the variable is changed. # Autodiscard is option "-9" followed by the "YES" confirmation. From 5985b1e3d6c5bc0c34c8cb3a6909ef06cd9a606b Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 12 Jul 2015 15:50:58 -0700 Subject: [PATCH 2/2] build_library: replace package checking script The new python script check_root uses data that portage already maintains on what shared libraries packages need or provide instead of re-scanning whatever ELF files that can be found. This is much more comprehensive but there is a bit of a transition issue for folks with long-lived SDKs: packages built with portage older than 2.2.18 do not include this data. As such for now the check is non-fatal and provides a command you can use to refresh locally installed packages. The code checking for conflicts between top level directories and /usr has also been rewritten. Both tests now are considerably faster. --- build_library/board_options.sh | 28 +++++ build_library/build_image_util.sh | 8 +- build_library/check_deps | 164 ---------------------------- build_library/check_root | 156 ++++++++++++++++++++++++++ build_library/test_image_content.sh | 46 ++------ 5 files changed, 203 insertions(+), 199 deletions(-) delete mode 100755 build_library/check_deps create mode 100755 build_library/check_root diff --git a/build_library/board_options.sh b/build_library/board_options.sh index 4a706781d4..cd3fb6c4bd 100644 --- a/build_library/board_options.sh +++ b/build_library/board_options.sh @@ -24,3 +24,31 @@ pkg_use_enabled() { equery-"${BOARD}" -q uses "${pkg}" | grep -q ${grep_args} return $? } + +# get a package's SONAMEs in soname.provided format +pkg_soname_provides() { + local provides p + # We could run this command but it ugly and silly slow: + # portageq-"${BOARD}" metadata "${BOARD_ROOT}" installed "$1" PROVIDES + provides=$(<"${BOARD_ROOT}/var/db/pkg/$1/PROVIDES") + + if [[ -z "$provides" ]]; then + return + fi + + # convert: + # x86_32: libcom_err.so.2 libss.so.2 x86_64: libcom_err.so.2 libss.so.2 + # into: + # x86_32 libcom_err.so.2 libss.so.2 + # x86_64 libcom_err.so.2 libss.so.2 + echo -n "# $1:" + for p in ${provides}; do + if [[ "$p" == *: ]]; then + echo + echo -n "${p%:}" + else + echo -n " $p" + fi + done + echo +} diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index fb0a20bd6c..dc331f632f 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -105,6 +105,11 @@ emerge_to_image() { # Make sure profile.env and ld.so.cache has been generated sudo -E ROOT="${root_fs_dir}" env-update + + # TODO(marineam): just call ${BUILD_LIBRARY_DIR}/check_root directly once + # all tests are fatal, for now let the old function skip soname errors. + ROOT="${root_fs_dir}" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ + test_image_content "${root_fs_dir}" } # Switch to the dev or prod sub-profile @@ -189,8 +194,9 @@ extract_docs() { package_provided() { local p profile="${BUILD_DIR}/configroot/etc/portage/profile" for p in "$@"; do - info "Writing $p to package.provided" + info "Writing $p to package.provided and soname.provided" echo "$p" >> "${profile}/package.provided" + pkg_soname_provides "$p" >> "${profile}/soname.provided" done } diff --git a/build_library/check_deps b/build_library/check_deps deleted file mode 100755 index e72975c250..0000000000 --- a/build_library/check_deps +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import os -import re -import sys -import glob - -_SHARED_RE = re.compile(r"Shared library: \[([^\]]+)\]") -_RPATH_RE = re.compile(r"Library r(?:un)?path: \[([^\]]+)\]") - - -class CheckDependencies(object): - """Check that dependencies for binaries can be found in the specified dir.""" - - def _ReadLdSoConf(self, path): - """Parse ld.so.conf files. - - Starting with the file at PATH (searched relative to self._root), return - all the valid libdirs found. Include directives are handled recursively. - - Args: - path: the path to the ld.so.conf file (inside of the root). - - Returns: - A list of valid libdirs. - """ - - libdirs = set() - - ld_so_conf = self._root + path - if os.path.exists(ld_so_conf): - f = file(ld_so_conf) - - for line in f: - line = line.rstrip() - - if line.startswith("/"): - libpath = self._root + line - if os.path.exists(libpath): - libdirs.add(libpath) - - elif line.startswith("include "): - # Includes are absolute or relative to the file itself. - line = os.path.join(os.path.dirname(path), line[8:]) - for p in glob.glob(self._root + line): - rel_p = "/%s" % os.path.relpath(p, self._root) - libdirs.update(self._ReadLdSoConf(rel_p)) - - f.close() - - return libdirs - - def __init__(self, root, verbose=False): - """Initializer. - - Args: - root: The sysroot (e.g. "/") - verbose: Print helpful messages. - """ - - self._root = root - self._libcache = set() - self._verbose = verbose - - libdirs = self._ReadLdSoConf("/etc/ld.so.conf") - if self._verbose: - print "Library search path: %s" % " ".join(sorted(libdirs)) - - self._ReadLibs(libdirs, self._libcache) - - def _ReadLibs(self, paths, libcache): - for path in paths: - if os.path.exists(path): - for lib in os.listdir(path): - libcache.add(lib) - - def _ReadDependencies(self, binary): - """Run readelf -d on BINARY, returning (deps, rpaths).""" - - deps = set() - rpaths = set() - - # Read list of dynamic libraries, ignoring error messages that occur - # when we look at files that aren't actually libraries - f = os.popen("readelf -d '%s' 2>/dev/null" % binary) - for line in f: - - # Grab dependencies - m = _SHARED_RE.search(line) - if m: - deps.add(m.group(1)) - - # Add RPATHs in our search path - m = _RPATH_RE.search(line) - if m: - for path in m.group(1).split(":"): - if path.startswith("$ORIGIN"): - rpaths.add(path.replace("$ORIGIN", os.path.dirname(binary))) - else: - rpaths.add(os.path.join(self._root, path[1:])) - f.close() - - return (deps, rpaths) - - def CheckDependencies(self, binary): - """Check whether the libs for BINARY can be found in our sysroot.""" - - good = True - - deps, rpaths = self._ReadDependencies(binary) - - if self._verbose: - for lib in self._libcache & deps: - print "Found %s" % lib - - for lib in deps - self._libcache: - if lib[0] != "/": - for path in rpaths: - if os.path.exists(os.path.join(path, lib)): - if self._verbose: - print "Found %s" % lib - break - else: - print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib) - good = False - else: - full_path = os.path.join(self._root, lib[1:]) - if os.path.exists(full_path): - if self._verbose: print "Found %s" % lib - else: - print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib) - good = False - - return good - - -def main(): - if len(sys.argv) < 3: - print "Usage: %s [-v] sysroot binary [ binary ... ]" % sys.argv[0] - sys.exit(1) - - verbose = False - if sys.argv[1] == "-v": - verbose = True - sys.argv = sys.argv[0:1] + sys.argv[2:] - - checker = CheckDependencies(sys.argv[1], verbose) - errors = False - for binary in sys.argv[2:]: - if verbose: print "Checking %s" % binary - if not checker.CheckDependencies(binary): - errors = True - - if errors: - sys.exit(1) - else: - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/build_library/check_root b/build_library/check_root new file mode 100755 index 0000000000..b56f5dfd38 --- /dev/null +++ b/build_library/check_root @@ -0,0 +1,156 @@ +#!/usr/bin/python + +# Copyright (c) 2015 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import unicode_literals + +import sys + +import portage +from portage import dep +from portage import output +from portage.dep.soname.SonameAtom import SonameAtom +from portage.dep.soname.parse import parse_soname_deps + +VARDB = portage.db[portage.root]["vartree"].dbapi + +# TODO(marneam): possibly accept globs for arch and sonames +IGNORE_MISSING = { + # /usr/lib/go/src/debug/elf/testdata/gcc-386-freebsd-exec + "dev-lang/go": [SonameAtom("x86_32", "libc.so.6")], + "dev-lang/go-bootstrap": [SonameAtom("x86_32", "libc.so.6")], + + # https://bugs.gentoo.org/show_bug.cgi?id=554582 + "net-firewall/ebtables": [SonameAtom("x86_64", "libebt_802_3.so"), + SonameAtom("x86_64", "libebt_among.so"), + SonameAtom("x86_64", "libebt_arp.so"), + SonameAtom("x86_64", "libebt_arpreply.so"), + SonameAtom("x86_64", "libebt_ip.so"), + SonameAtom("x86_64", "libebt_ip6.so"), + SonameAtom("x86_64", "libebt_limit.so"), + SonameAtom("x86_64", "libebt_log.so"), + SonameAtom("x86_64", "libebt_mark.so"), + SonameAtom("x86_64", "libebt_mark_m.so"), + SonameAtom("x86_64", "libebt_nat.so"), + SonameAtom("x86_64", "libebt_nflog.so"), + SonameAtom("x86_64", "libebt_pkttype.so"), + SonameAtom("x86_64", "libebt_redirect.so"), + SonameAtom("x86_64", "libebt_standard.so"), + SonameAtom("x86_64", "libebt_stp.so"), + SonameAtom("x86_64", "libebt_ulog.so"), + SonameAtom("x86_64", "libebt_vlan.so"), + SonameAtom("x86_64", "libebtable_broute.so"), + SonameAtom("x86_64", "libebtable_filter.so"), + SonameAtom("x86_64", "libebtable_nat.so")], +} + +USR_LINKS = ("/bin/", "/sbin/", "/lib/", "/lib32/", "/lib64/") + +def provided_sonames(): + for cpv in VARDB.cpv_all(): + raw = VARDB.aux_get(cpv, ["PROVIDES"])[0] + for atom in parse_soname_deps(raw): + yield atom + + # soname.provided in PORTAGE_CONFIGROOT + for atom in VARDB.settings.soname_provided: + yield atom + +def ignore_sonames(cpv): + for key in dep.match_to_list(cpv, IGNORE_MISSING.iterkeys()): + for atom in IGNORE_MISSING[key]: + yield atom + +def missing_sonames(): + provided = frozenset(provided_sonames()) + for cpv in VARDB.cpv_all(): + raw = VARDB.aux_get(cpv, ["REQUIRES"])[0] + requires = frozenset(parse_soname_deps(raw)) + ignore = frozenset(ignore_sonames(cpv)) + missing = requires - provided - ignore + if missing: + yield (cpv, missing) + +def usr_conflicts(): + for cpv in VARDB.cpv_all(): + raw = VARDB.aux_get(cpv, ["CONTENTS"])[0] + usr = set() + root = set() + + # format is: + # obj /path goo 123 + # dir /path/foo + # sym /this -> that 123 + # and so on + for line in raw.split("\n"): + if line[:4] != "obj " and line[:4] != "sym ": + continue + + # yeah, hard to read, trying to make it fast... + i = line.find("/", 5) + topdir = line[4:i+1] + if topdir == "/usr/": + j = line.find("/", 9) + nextdir = line[8:j+1] + if nextdir in USR_LINKS: + end = line.find(" ", 8) + usr.add(line[8:end]) + elif topdir in USR_LINKS: + end = line.find(" ", 4) + root.add(line[4:end]) + + conflicts = frozenset(root).intersection(usr) + if conflicts: + yield (cpv, conflicts) + +def check_libs(): + ok = True + for cpv, sonames in missing_sonames(): + error("%s is missing libraries:", cpv) + for soname in sonames: + error("\t%s", soname) + ok = False + return ok + +def check_usr(): + ok = True + for cpv, conflicts in usr_conflicts(): + error("%s has paths that conflict with /usr", cpv) + for path in conflicts: + error("\t%s", path) + ok = False + return ok + +def error(fmt, *args): + sys.stderr.write(output.red(fmt % args)) + sys.stderr.write("\n") + +def main(): + ok = True + check_funcs = { + "libs": check_libs, + "usr": check_usr, + } + + if not sys.stderr.isatty(): + output.nocolor() + + checks = sys.argv[1:] + if not checks: + checks = check_funcs.keys() + + for check in checks: + func = check_funcs.get(check) + if func: + ok = func() and ok + else: + error("Unknown test name '%s'", check) + error("Valid tests: %s", " ".join(check_funcs)) + ok = False + + return 0 if ok else 1 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_library/test_image_content.sh b/build_library/test_image_content.sh index f67ba93fdf..1b769dbd38 100644 --- a/build_library/test_image_content.sh +++ b/build_library/test_image_content.sh @@ -6,30 +6,14 @@ test_image_content() { local root="$1" local returncode=0 - if [[ -z "$BOARD" ]]; then - die '$BOARD is undefined!' - fi - local portageq="portageq-$BOARD" - - local binaries=( - "$root/usr/boot/vmlinuz" - "$root/bin/sed" - ) - - for test_file in "${binaries[@]}"; do - if [ ! -f "$test_file" ]; then - error "test_image_content: Cannot find '$test_file'" - returncode=1 - fi - done - - local libs=( $(sudo find "$root" -type f -name '*.so*') ) - - # Check that all .so files, plus the binaries, have the appropriate - # dependencies. - local check_deps="${BUILD_LIBRARY_DIR}/check_deps" - if ! "$check_deps" "$root" "${binaries[@]}" "${libs[@]}"; then - error "test_image_content: Failed dependency check" + info "Checking $1" + local check_root="${BUILD_LIBRARY_DIR}/check_root" + if ! ROOT="$root" "$check_root" libs; then + warn "test_image_content: Failed dependency check" + warn "This may be the result of having a long-lived SDK with binary" + warn "packages that predate portage 2.2.18. If this is the case try:" + echo " emerge-$BOARD -agkuDN --rebuilt-binaries=y -j9 @world" + echo " emerge-$BOARD -a --depclean" returncode=1 fi @@ -47,16 +31,10 @@ test_image_content() { done # Check that there are no conflicts between /* and /usr/* - local pkgdb=$(ROOT="${root}" $portageq vdb_path) - local files=$(awk '$2 ~ /^\/(bin|sbin|lib|lib32|lib64)\// {print $2}' \ - "${pkgdb}"/*/*/CONTENTS) - local check_file - for check_file in $files; do - if grep -q "^... /usr$check_file " "${pkgdb}"/*/*/CONTENTS; then - error "test_image_content: $check_file conflicts with /usr$check_file" - returncode=1 - fi - done + if ! ROOT="$root" "$check_root" usr; then + error "test_image_content: Failed /usr conflict check" + returncode=1 + fi return $returncode }