Merge pull request #137 from marineam/check-pkgs

Improvements for image validation, add /usr conflict test

build_image

@@ -103,7 +103,8 @@ fi
 
 # Check that the build root is sane.
 if [[ ${skip_test_build_root} -ne 1 ]]; then
-  "${BUILD_LIBRARY_DIR}/test_build_root" --root="${BOARD_ROOT}"
+  info "Checking build root"
+  test_image_content "${BOARD_ROOT}"
 fi
 
 # Hack to fix bug where x86_64 CHOST line gets incorrectly added.

build_library/check_deps

@@ -29,7 +29,7 @@ class CheckDependencies(object):
       A list of valid libdirs.
     """
 
-    libdirs = []
+    libdirs = set()
 
     ld_so_conf = self._root + path
     if os.path.exists(ld_so_conf):
@@ -41,14 +41,14 @@ class CheckDependencies(object):
         if line.startswith("/"):
           libpath = self._root + line
           if os.path.exists(libpath):
-            libdirs.append(libpath)
+            libdirs.add(libpath)
 
         elif line.startswith("include "):
           # Includes are absolute or relative to the file itself.
           line = os.path.join(os.path.dirname(path), line[8:])
           for p in glob.glob(self._root + line):
             rel_p = "/%s" % os.path.relpath(p, self._root)
-            libdirs.extend(self._ReadLdSoConf(rel_p))
+            libdirs.update(self._ReadLdSoConf(rel_p))
 
       f.close()
 
@@ -66,36 +66,9 @@ class CheckDependencies(object):
     self._libcache = set()
     self._verbose = verbose
 
-    libdirs = []
-
-    # Add the native paths. Get the ELF interpreter from a known file
-    # and assume that the path it is in is our native libdir. So here
-    # we would get something like "/lib64/ld-linux-x86-64.so.2".
-    elf = "/bin/sh"
-    f = os.popen("scanelf -qF'%%i#p' %s/%s" % (root, elf))
-    native_libdir = os.path.dirname(f.readline().rstrip())
-    f.close()
-    if len(native_libdir) == 0:
-      print >>sys.stderr, "Problem with %s: can't find ELF interp" % elf
-      sys.exit(1)
-    elif native_libdir != "/lib":
-      libdirs.extend([
-        "%s/%s" % (root, native_libdir),
-        "%s/usr%s" % (root, native_libdir)
-      ])
-
-    # Insert some default directories into our library cache.
-    libdirs.extend([
-      "%s/lib" % root,
-      "%s/usr/lib" % root,
-      "%s/opt/google/o3d/lib" % root,
-      "%s/usr/lib/opengl/xorg-x11/lib" % root,
-      "%s/usr/local/lib/icedtea6/jre/lib/i386/client" % root,
-      "%s/usr/local/lib/icedtea6/jre/lib/i386/headless" % root
-    ])
-
-    # Read more directories from ld.so.conf.
-    libdirs.extend(self._ReadLdSoConf("/etc/ld.so.conf"))
+    libdirs = self._ReadLdSoConf("/etc/ld.so.conf")
+    if self._verbose:
+        print "Library search path: %s" % " ".join(sorted(libdirs))
 
     self._ReadLibs(libdirs, self._libcache)
 

build_library/test_build_root

@@ -6,12 +6,13 @@
 
 SCRIPT_ROOT=$(readlink -f $(dirname "$0")/..)
 . "${SCRIPT_ROOT}/common.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
 
 # We're invoked only by build_image, which runs in the chroot
 assert_inside_chroot
 
 # Flags
-DEFINE_string root ""      \
+DEFINE_string root "${DEFAULT_BOARD+/build/${DEFAULT_BOARD}}" \
   "The root file system to check."
 
 # Parse command line
@@ -21,26 +22,8 @@ eval set -- "${FLAGS_ARGV}"
 # Die on any errors
 switch_to_strict_mode
 
-# Check all parts of a pipe
-set -o pipefail
-
-ROOT="$FLAGS_root"
-if [[ ! -d "$ROOT" ]]; then
-  die_notrace "Root FS does not exist ($ROOT)"
+if [[ ! -d "$FLAGS_root" ]]; then
+  die_notrace "Root FS does not exist ($FLAGS_root)"
 fi
 
-ROOT_BINARIES=(
-  /boot/vmlinuz
-  /bin/sed
-)
-
-EXITCODE=0
-for i in "${BINARIES[@]}"; do
-  i="${ROOT}${i}"
-  if ! [[ -f $i ]]; then
-    error "cannot find $i"
-    EXITCODE=1
-  fi
-done
-
-exit $EXITCODE
+test_image_content "$FLAGS_root"

build_library/test_image_content.sh

@@ -2,6 +2,37 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# Files that are known to conflict with /usr but are OK because they
+# are already fixed by toggling the symlink-usr USE flag.
+USR_CONFLICT_WHITELIST=(
+    /bin/awk
+    /bin/basename
+    /bin/chroot
+    /bin/cut
+    /bin/dir
+    /bin/dirname
+    /bin/du
+    /bin/env
+    /bin/expr
+    /bin/gawk
+    /bin/head
+    /bin/mkfifo
+    /bin/mktemp
+    /bin/passwd
+    /bin/readlink
+    /bin/seq
+    /bin/sleep
+    /bin/sort
+    /bin/tail
+    /bin/touch
+    /bin/tr
+    /bin/tty
+    /bin/uname
+    /bin/vdir
+    /bin/wc
+    /bin/yes
+)
+
 test_image_content() {
   local root="$1"
   local returncode=0
@@ -55,5 +86,29 @@ test_image_content() {
     fi
   fi
 
+  # Check that there are no conflicts between /* and /usr/*
+  # TODO(marineam): Before symlinking to /usr this test will need to be
+  # rewritten to query the package database instead of the filesystem.
+  local check_dir
+  for check_dir in "${root}"/usr/*; do
+    if [[ ! -d "${check_dir}" || -h "${check_dir}" ]]; then
+      continue
+    fi
+    for check_file in "${check_dir}"/*; do
+      root_file="${root}${check_file##*/usr}"
+      trimmed_path="${root_file#${root}}"
+      local whitelist
+      for whitelist in "${USR_CONFLICT_WHITELIST[@]}"; do
+        if [[ "${trimmed_path}" == "${whitelist}" ]]; then
+          continue 2
+        fi
+      done
+      if [[ -e "${root_file}" || -h "${root_file}" ]]; then
+        # TODO(marineam): make fatal before switching to symlinks
+        warn "test_image_content: ${root_file#${root}} conflicts with /usr"
+      fi
+    done
+  done
+
   return $returncode
 }

build_packages

@@ -128,6 +128,7 @@ fi
 # set BOARD and BOARD_ROOT
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
 
 # Setup all the emerge command/flags.
 EMERGE_FLAGS=( -uDNv --backtrack=30 --select )
@@ -209,9 +210,11 @@ fi
 info "Merging board packages now"
 sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}"
 
+info "Checking build root"
+test_image_content "${BOARD_ROOT}"
+
 # upload packages if enabled
 upload_packages
 
-echo "Builds complete"
+info "Builds complete"
 command_completed
-echo "Done"

install_toolchain

@@ -116,17 +116,27 @@ install_toolchain_in_board() {
   # Install libc libraries.
   if [ "${CHOST}" != "$FLAGS_toolchain" ] ; then
     local libc_path="${PKGDIR}/cross-${FLAGS_toolchain}/${libc_tar}"
+    local libc_excludes=(
+      # Empty lib dirs, replaced by symlinks
+      'lib'
+      # Locales and info pages
+      usr/share/{i18n,info,locale}
+    )
     cmds=(
       "mkdir -p '${BOARD_ROOT}/lib64' '${BOARD_ROOT}/sbin'"
-      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}' --exclude=lib \
+      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}' \
           --exclude=${FLAGS_toolchain}/lib64 --exclude=${FLAGS_toolchain}/sbin \
+          ${libc_excludes[@]/#/--exclude=} \
           './usr/${FLAGS_toolchain}' --strip-components=3"
-      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/lib64' --exclude=lib \
+      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/lib64' \
+          ${libc_excludes[@]/#/--exclude=} \
           './usr/${FLAGS_toolchain}/lib64' --strip-components=4"
-      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/sbin' --exclude=lib \
+      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/sbin' \
+          ${libc_excludes[@]/#/--exclude=} \
           './usr/${FLAGS_toolchain}/sbin' --strip-components=4"
       "mkdir -p '${BOARD_ROOT}/usr/lib/debug'"
       "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/usr/lib/debug' \
+          ${libc_excludes[@]/#/--exclude=} \
           './usr/lib/debug/usr/${FLAGS_toolchain}' --strip-components=6 \
           || warn 'libc debug info not copied.'"
     )