build_library: replace package checking script

The new python script check_root uses data that portage already
maintains on what shared libraries packages need or provide instead of
re-scanning whatever ELF files that can be found. This is much more
comprehensive but there is a bit of a transition issue for folks with
long-lived SDKs: packages built with portage older than 2.2.18 do not
include this data. As such for now the check is non-fatal and provides a
command you can use to refresh locally installed packages.

The code checking for conflicts between top level directories and /usr
has also been rewritten. Both tests now are considerably faster.

build_library/board_options.sh

@@ -24,3 +24,31 @@ pkg_use_enabled() {
   equery-"${BOARD}" -q uses "${pkg}" | grep -q ${grep_args}
   return $?
 }
+
+# get a package's SONAMEs in soname.provided format
+pkg_soname_provides() {
+  local provides p
+  # We could run this command but it ugly and silly slow:
+  # portageq-"${BOARD}" metadata "${BOARD_ROOT}" installed "$1" PROVIDES
+  provides=$(<"${BOARD_ROOT}/var/db/pkg/$1/PROVIDES")
+
+  if [[ -z "$provides" ]]; then
+    return
+  fi
+
+  # convert:
+  #  x86_32: libcom_err.so.2 libss.so.2 x86_64: libcom_err.so.2 libss.so.2
+  # into:
+  #  x86_32 libcom_err.so.2 libss.so.2
+  #  x86_64 libcom_err.so.2 libss.so.2
+  echo -n "# $1:"
+  for p in ${provides}; do
+    if [[ "$p" == *: ]]; then
+      echo
+      echo -n "${p%:}"
+    else
+      echo -n " $p"
+    fi
+  done
+  echo
+}

build_library/build_image_util.sh

@@ -105,6 +105,11 @@ emerge_to_image() {
 
   # Make sure profile.env and ld.so.cache has been generated
   sudo -E ROOT="${root_fs_dir}" env-update
+
+  # TODO(marineam): just call ${BUILD_LIBRARY_DIR}/check_root directly once
+  # all tests are fatal, for now let the old function skip soname errors.
+  ROOT="${root_fs_dir}" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
+      test_image_content "${root_fs_dir}"
 }
 
 # Switch to the dev or prod sub-profile
@@ -189,8 +194,9 @@ extract_docs() {
 package_provided() {
     local p profile="${BUILD_DIR}/configroot/etc/portage/profile"    
     for p in "$@"; do
-        info "Writing $p to package.provided"
+        info "Writing $p to package.provided and soname.provided"
         echo "$p" >> "${profile}/package.provided"
+	pkg_soname_provides "$p" >> "${profile}/soname.provided"
     done
 }
 

build_library/check_deps

@@ -1,164 +0,0 @@
-#!/usr/bin/python
-
-# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import os
-import re
-import sys
-import glob
-
-_SHARED_RE = re.compile(r"Shared library: \[([^\]]+)\]")
-_RPATH_RE = re.compile(r"Library r(?:un)?path: \[([^\]]+)\]")
-
-
-class CheckDependencies(object):
-  """Check that dependencies for binaries can be found in the specified dir."""
-
-  def _ReadLdSoConf(self, path):
-    """Parse ld.so.conf files.
-
-    Starting with the file at PATH (searched relative to self._root), return
-    all the valid libdirs found. Include directives are handled recursively.
-
-    Args:
-      path: the path to the ld.so.conf file (inside of the root).
-
-    Returns:
-      A list of valid libdirs.
-    """
-
-    libdirs = set()
-
-    ld_so_conf = self._root + path
-    if os.path.exists(ld_so_conf):
-      f = file(ld_so_conf)
-
-      for line in f:
-        line = line.rstrip()
-
-        if line.startswith("/"):
-          libpath = self._root + line
-          if os.path.exists(libpath):
-            libdirs.add(libpath)
-
-        elif line.startswith("include "):
-          # Includes are absolute or relative to the file itself.
-          line = os.path.join(os.path.dirname(path), line[8:])
-          for p in glob.glob(self._root + line):
-            rel_p = "/%s" % os.path.relpath(p, self._root)
-            libdirs.update(self._ReadLdSoConf(rel_p))
-
-      f.close()
-
-    return libdirs
-
-  def __init__(self, root, verbose=False):
-    """Initializer.
-
-    Args:
-      root: The sysroot (e.g. "/")
-      verbose: Print helpful messages.
-    """
-
-    self._root = root
-    self._libcache = set()
-    self._verbose = verbose
-
-    libdirs = self._ReadLdSoConf("/etc/ld.so.conf")
-    if self._verbose:
-        print "Library search path: %s" % " ".join(sorted(libdirs))
-
-    self._ReadLibs(libdirs, self._libcache)
-
-  def _ReadLibs(self, paths, libcache):
-    for path in paths:
-      if os.path.exists(path):
-        for lib in os.listdir(path):
-          libcache.add(lib)
-
-  def _ReadDependencies(self, binary):
-    """Run readelf -d on BINARY, returning (deps, rpaths)."""
-
-    deps = set()
-    rpaths = set()
-
-    # Read list of dynamic libraries, ignoring error messages that occur
-    # when we look at files that aren't actually libraries
-    f = os.popen("readelf -d '%s' 2>/dev/null" % binary)
-    for line in f:
-
-      # Grab dependencies
-      m = _SHARED_RE.search(line)
-      if m:
-        deps.add(m.group(1))
-
-      # Add RPATHs in our search path
-      m = _RPATH_RE.search(line)
-      if m:
-        for path in m.group(1).split(":"):
-          if path.startswith("$ORIGIN"):
-            rpaths.add(path.replace("$ORIGIN", os.path.dirname(binary)))
-          else:
-            rpaths.add(os.path.join(self._root, path[1:]))
-    f.close()
-
-    return (deps, rpaths)
-
-  def CheckDependencies(self, binary):
-    """Check whether the libs for BINARY can be found in our sysroot."""
-
-    good = True
-
-    deps, rpaths = self._ReadDependencies(binary)
-
-    if self._verbose:
-      for lib in self._libcache & deps:
-        print "Found %s" % lib
-
-    for lib in deps - self._libcache:
-      if lib[0] != "/":
-        for path in rpaths:
-          if os.path.exists(os.path.join(path, lib)):
-            if self._verbose:
-              print "Found %s" % lib
-            break
-        else:
-          print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib)
-          good = False
-      else:
-        full_path = os.path.join(self._root, lib[1:])
-        if os.path.exists(full_path):
-          if self._verbose: print "Found %s" % lib
-        else:
-          print >>sys.stderr, "Problem with %s: Can't find %s" % (binary, lib)
-          good = False
-
-    return good
-
-
-def main():
-  if len(sys.argv) < 3:
-    print "Usage: %s [-v] sysroot binary [ binary ... ]" % sys.argv[0]
-    sys.exit(1)
-
-  verbose = False
-  if sys.argv[1] == "-v":
-    verbose = True
-    sys.argv = sys.argv[0:1] + sys.argv[2:]
-
-  checker = CheckDependencies(sys.argv[1], verbose)
-  errors = False
-  for binary in sys.argv[2:]:
-    if verbose: print "Checking %s" % binary
-    if not checker.CheckDependencies(binary):
-      errors = True
-
-  if errors:
-    sys.exit(1)
-  else:
-    sys.exit(0)
-
-if __name__ == "__main__":
-  main()

build_library/check_root

@@ -0,0 +1,156 @@
+#!/usr/bin/python
+
+# Copyright (c) 2015 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import unicode_literals
+
+import sys
+
+import portage
+from portage import dep
+from portage import output
+from portage.dep.soname.SonameAtom import SonameAtom
+from portage.dep.soname.parse import parse_soname_deps
+
+VARDB = portage.db[portage.root]["vartree"].dbapi
+
+# TODO(marneam): possibly accept globs for arch and sonames
+IGNORE_MISSING = {
+    # /usr/lib/go/src/debug/elf/testdata/gcc-386-freebsd-exec
+    "dev-lang/go":              [SonameAtom("x86_32", "libc.so.6")],
+    "dev-lang/go-bootstrap":    [SonameAtom("x86_32", "libc.so.6")],
+
+    # https://bugs.gentoo.org/show_bug.cgi?id=554582
+    "net-firewall/ebtables":    [SonameAtom("x86_64", "libebt_802_3.so"),
+                                 SonameAtom("x86_64", "libebt_among.so"),
+                                 SonameAtom("x86_64", "libebt_arp.so"),
+                                 SonameAtom("x86_64", "libebt_arpreply.so"),
+                                 SonameAtom("x86_64", "libebt_ip.so"),
+                                 SonameAtom("x86_64", "libebt_ip6.so"),
+                                 SonameAtom("x86_64", "libebt_limit.so"),
+                                 SonameAtom("x86_64", "libebt_log.so"),
+                                 SonameAtom("x86_64", "libebt_mark.so"),
+                                 SonameAtom("x86_64", "libebt_mark_m.so"),
+                                 SonameAtom("x86_64", "libebt_nat.so"),
+                                 SonameAtom("x86_64", "libebt_nflog.so"),
+                                 SonameAtom("x86_64", "libebt_pkttype.so"),
+                                 SonameAtom("x86_64", "libebt_redirect.so"),
+                                 SonameAtom("x86_64", "libebt_standard.so"),
+                                 SonameAtom("x86_64", "libebt_stp.so"),
+                                 SonameAtom("x86_64", "libebt_ulog.so"),
+                                 SonameAtom("x86_64", "libebt_vlan.so"),
+                                 SonameAtom("x86_64", "libebtable_broute.so"),
+                                 SonameAtom("x86_64", "libebtable_filter.so"),
+                                 SonameAtom("x86_64", "libebtable_nat.so")],
+}
+
+USR_LINKS = ("/bin/", "/sbin/", "/lib/", "/lib32/", "/lib64/")
+
+def provided_sonames():
+    for cpv in VARDB.cpv_all():
+        raw = VARDB.aux_get(cpv, ["PROVIDES"])[0]
+        for atom in parse_soname_deps(raw):
+            yield atom
+
+    # soname.provided in PORTAGE_CONFIGROOT
+    for atom in VARDB.settings.soname_provided:
+        yield atom
+
+def ignore_sonames(cpv):
+    for key in dep.match_to_list(cpv, IGNORE_MISSING.iterkeys()):
+        for atom in IGNORE_MISSING[key]:
+            yield atom
+
+def missing_sonames():
+    provided = frozenset(provided_sonames())
+    for cpv in VARDB.cpv_all():
+        raw = VARDB.aux_get(cpv, ["REQUIRES"])[0]
+        requires = frozenset(parse_soname_deps(raw))
+        ignore = frozenset(ignore_sonames(cpv))
+        missing = requires - provided - ignore
+        if missing:
+            yield (cpv, missing)
+
+def usr_conflicts():
+    for cpv in VARDB.cpv_all():
+        raw = VARDB.aux_get(cpv, ["CONTENTS"])[0]
+        usr = set()
+        root = set()
+
+        # format is:
+        #  obj /path goo 123
+        #  dir /path/foo
+        #  sym /this -> that 123
+        # and so on
+        for line in raw.split("\n"):
+            if line[:4] != "obj " and line[:4] != "sym ":
+                continue
+
+            # yeah, hard to read, trying to make it fast...
+            i = line.find("/", 5)
+            topdir = line[4:i+1]
+            if topdir == "/usr/":
+                j = line.find("/", 9)
+                nextdir = line[8:j+1]
+                if nextdir in USR_LINKS:
+                    end = line.find(" ", 8)
+                    usr.add(line[8:end])
+            elif topdir in USR_LINKS:
+                end = line.find(" ", 4)
+                root.add(line[4:end])
+
+        conflicts = frozenset(root).intersection(usr)
+        if conflicts:
+            yield (cpv, conflicts)
+
+def check_libs():
+    ok = True
+    for cpv, sonames in missing_sonames():
+        error("%s is missing libraries:", cpv)
+        for soname in sonames:
+            error("\t%s", soname)
+        ok = False
+    return ok
+
+def check_usr():
+    ok = True
+    for cpv, conflicts in usr_conflicts():
+        error("%s has paths that conflict with /usr", cpv)
+        for path in conflicts:
+            error("\t%s", path)
+        ok = False
+    return ok
+
+def error(fmt, *args):
+    sys.stderr.write(output.red(fmt % args))
+    sys.stderr.write("\n")
+
+def main():
+    ok = True
+    check_funcs = {
+            "libs": check_libs,
+            "usr": check_usr,
+    }
+
+    if not sys.stderr.isatty():
+        output.nocolor()
+
+    checks = sys.argv[1:]
+    if not checks:
+        checks = check_funcs.keys()
+
+    for check in checks:
+        func = check_funcs.get(check)
+        if func:
+            ok = func() and ok
+        else:
+            error("Unknown test name '%s'", check)
+            error("Valid tests: %s", " ".join(check_funcs))
+            ok = False
+
+    return 0 if ok else 1
+
+if __name__ == "__main__":
+    sys.exit(main())

build_library/test_image_content.sh

@@ -6,30 +6,14 @@ test_image_content() {
   local root="$1"
   local returncode=0
 
-  if [[ -z "$BOARD" ]]; then
+  info "Checking $1"
-    die '$BOARD is undefined!'
+  local check_root="${BUILD_LIBRARY_DIR}/check_root"
-  fi
+  if ! ROOT="$root" "$check_root" libs; then
-  local portageq="portageq-$BOARD"
+    warn "test_image_content: Failed dependency check"
-
+    warn "This may be the result of having a long-lived SDK with binary"
-  local binaries=(
+    warn "packages that predate portage 2.2.18. If this is the case try:"
-    "$root/usr/boot/vmlinuz"
+    echo "    emerge-$BOARD -agkuDN --rebuilt-binaries=y -j9  @world"
-    "$root/bin/sed"
+    echo "    emerge-$BOARD -a --depclean"
-  )
-
-  for test_file in "${binaries[@]}"; do
-    if [ ! -f "$test_file" ]; then
-      error "test_image_content: Cannot find '$test_file'"
-      returncode=1
-    fi
-  done
-
-  local libs=( $(sudo find "$root" -type f -name '*.so*') )
-
-  # Check that all .so files, plus the binaries, have the appropriate
-  # dependencies.
-  local check_deps="${BUILD_LIBRARY_DIR}/check_deps"
-  if ! "$check_deps"  "$root" "${binaries[@]}" "${libs[@]}"; then
-    error "test_image_content: Failed dependency check"
     returncode=1
   fi
 
@@ -47,16 +31,10 @@ test_image_content() {
   done
 
   # Check that there are no conflicts between /* and /usr/*
-  local pkgdb=$(ROOT="${root}" $portageq vdb_path)
+  if ! ROOT="$root" "$check_root" usr; then
-  local files=$(awk '$2 ~ /^\/(bin|sbin|lib|lib32|lib64)\// {print $2}' \
+    error "test_image_content: Failed /usr conflict check"
-                "${pkgdb}"/*/*/CONTENTS)
+    returncode=1
-  local check_file
+  fi
-  for check_file in $files; do
-    if grep -q "^... /usr$check_file " "${pkgdb}"/*/*/CONTENTS; then
-      error "test_image_content: $check_file conflicts with /usr$check_file"
-      returncode=1
-    fi
-  done
 
   return $returncode
 }