From 1e8f44e3fce6b1557bc48786ed9839169dff20f9 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Tue, 26 Nov 2013 16:19:52 -0800 Subject: [PATCH 1/5] fix(check_deps): Depend exclusively on ld.so.conf for lib list The list in ld.so.conf needs to be complete so don't bother with including other hard-coded paths or the path to ld-linux-x86-64.so.2. --- build_library/check_deps | 39 ++++++--------------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/build_library/check_deps b/build_library/check_deps index 47257855f5..e72975c250 100755 --- a/build_library/check_deps +++ b/build_library/check_deps @@ -29,7 +29,7 @@ class CheckDependencies(object): A list of valid libdirs. """ - libdirs = [] + libdirs = set() ld_so_conf = self._root + path if os.path.exists(ld_so_conf): @@ -41,14 +41,14 @@ class CheckDependencies(object): if line.startswith("/"): libpath = self._root + line if os.path.exists(libpath): - libdirs.append(libpath) + libdirs.add(libpath) elif line.startswith("include "): # Includes are absolute or relative to the file itself. line = os.path.join(os.path.dirname(path), line[8:]) for p in glob.glob(self._root + line): rel_p = "/%s" % os.path.relpath(p, self._root) - libdirs.extend(self._ReadLdSoConf(rel_p)) + libdirs.update(self._ReadLdSoConf(rel_p)) f.close() @@ -66,36 +66,9 @@ class CheckDependencies(object): self._libcache = set() self._verbose = verbose - libdirs = [] - - # Add the native paths. Get the ELF interpreter from a known file - # and assume that the path it is in is our native libdir. So here - # we would get something like "/lib64/ld-linux-x86-64.so.2". - elf = "/bin/sh" - f = os.popen("scanelf -qF'%%i#p' %s/%s" % (root, elf)) - native_libdir = os.path.dirname(f.readline().rstrip()) - f.close() - if len(native_libdir) == 0: - print >>sys.stderr, "Problem with %s: can't find ELF interp" % elf - sys.exit(1) - elif native_libdir != "/lib": - libdirs.extend([ - "%s/%s" % (root, native_libdir), - "%s/usr%s" % (root, native_libdir) - ]) - - # Insert some default directories into our library cache. - libdirs.extend([ - "%s/lib" % root, - "%s/usr/lib" % root, - "%s/opt/google/o3d/lib" % root, - "%s/usr/lib/opengl/xorg-x11/lib" % root, - "%s/usr/local/lib/icedtea6/jre/lib/i386/client" % root, - "%s/usr/local/lib/icedtea6/jre/lib/i386/headless" % root - ]) - - # Read more directories from ld.so.conf. - libdirs.extend(self._ReadLdSoConf("/etc/ld.so.conf")) + libdirs = self._ReadLdSoConf("/etc/ld.so.conf") + if self._verbose: + print "Library search path: %s" % " ".join(sorted(libdirs)) self._ReadLibs(libdirs, self._libcache) From b4b7a2e2f8422bd4fe025e93857bc755762ccc61 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Tue, 26 Nov 2013 16:53:52 -0800 Subject: [PATCH 2/5] fix(install_toolchain): Exclude libc locale/info/etc from build root. These excludes are used in images and I want to use the same checking code as images do so the build root needs to behave the same way. This is temporary as I'll switch to installing libc via emerge soon. --- install_toolchain | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/install_toolchain b/install_toolchain index 29fdd21ffe..a827ea8d8e 100755 --- a/install_toolchain +++ b/install_toolchain @@ -116,17 +116,27 @@ install_toolchain_in_board() { # Install libc libraries. if [ "${CHOST}" != "$FLAGS_toolchain" ] ; then local libc_path="${PKGDIR}/cross-${FLAGS_toolchain}/${libc_tar}" + local libc_excludes=( + # Empty lib dirs, replaced by symlinks + 'lib' + # Locales and info pages + usr/share/{i18n,info,locale} + ) cmds=( "mkdir -p '${BOARD_ROOT}/lib64' '${BOARD_ROOT}/sbin'" - "tar jxpf '${libc_path}' -C '${BOARD_ROOT}' --exclude=lib \ + "tar jxpf '${libc_path}' -C '${BOARD_ROOT}' \ --exclude=${FLAGS_toolchain}/lib64 --exclude=${FLAGS_toolchain}/sbin \ + ${libc_excludes[@]/#/--exclude=} \ './usr/${FLAGS_toolchain}' --strip-components=3" - "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/lib64' --exclude=lib \ + "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/lib64' \ + ${libc_excludes[@]/#/--exclude=} \ './usr/${FLAGS_toolchain}/lib64' --strip-components=4" - "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/sbin' --exclude=lib \ + "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/sbin' \ + ${libc_excludes[@]/#/--exclude=} \ './usr/${FLAGS_toolchain}/sbin' --strip-components=4" "mkdir -p '${BOARD_ROOT}/usr/lib/debug'" "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/usr/lib/debug' \ + ${libc_excludes[@]/#/--exclude=} \ './usr/lib/debug/usr/${FLAGS_toolchain}' --strip-components=6 \ || warn 'libc debug info not copied.'" ) From 241d135126c4d12901ad88bd24224da18ef79a8c Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Tue, 26 Nov 2013 17:03:00 -0800 Subject: [PATCH 3/5] fix(test_build_root): Switch to using test_image_content Start using the more complete test_image_content test for the build root and add the test to the end of build_packages. --- build_image | 3 ++- build_library/test_build_root | 27 +++++---------------------- build_packages | 7 +++++-- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/build_image b/build_image index 86f9b0a08c..891aca3d28 100755 --- a/build_image +++ b/build_image @@ -103,7 +103,8 @@ fi # Check that the build root is sane. if [[ ${skip_test_build_root} -ne 1 ]]; then - "${BUILD_LIBRARY_DIR}/test_build_root" --root="${BOARD_ROOT}" + info "Checking build root" + test_image_content "${BOARD_ROOT}" fi # Hack to fix bug where x86_64 CHOST line gets incorrectly added. diff --git a/build_library/test_build_root b/build_library/test_build_root index cac339ea55..57aa6d72cd 100755 --- a/build_library/test_build_root +++ b/build_library/test_build_root @@ -6,12 +6,13 @@ SCRIPT_ROOT=$(readlink -f $(dirname "$0")/..) . "${SCRIPT_ROOT}/common.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 # We're invoked only by build_image, which runs in the chroot assert_inside_chroot # Flags -DEFINE_string root "" \ +DEFINE_string root "${DEFAULT_BOARD+/build/${DEFAULT_BOARD}}" \ "The root file system to check." # Parse command line @@ -21,26 +22,8 @@ eval set -- "${FLAGS_ARGV}" # Die on any errors switch_to_strict_mode -# Check all parts of a pipe -set -o pipefail - -ROOT="$FLAGS_root" -if [[ ! -d "$ROOT" ]]; then - die_notrace "Root FS does not exist ($ROOT)" +if [[ ! -d "$FLAGS_root" ]]; then + die_notrace "Root FS does not exist ($FLAGS_root)" fi -ROOT_BINARIES=( - /boot/vmlinuz - /bin/sed -) - -EXITCODE=0 -for i in "${BINARIES[@]}"; do - i="${ROOT}${i}" - if ! [[ -f $i ]]; then - error "cannot find $i" - EXITCODE=1 - fi -done - -exit $EXITCODE +test_image_content "$FLAGS_root" diff --git a/build_packages b/build_packages index 785b5306fd..81d3926cd1 100755 --- a/build_packages +++ b/build_packages @@ -128,6 +128,7 @@ fi # set BOARD and BOARD_ROOT . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 # Setup all the emerge command/flags. EMERGE_FLAGS=( -uDNv --backtrack=30 --select ) @@ -209,9 +210,11 @@ fi info "Merging board packages now" sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" +info "Checking build root" +test_image_content "${BOARD_ROOT}" + # upload packages if enabled upload_packages -echo "Builds complete" +info "Builds complete" command_completed -echo "Done" From fd7976e7fd1e52f321b50345f1b4efe9c84169c6 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 27 Nov 2013 14:43:35 -0800 Subject: [PATCH 4/5] fix(test_image_content): Add test for /usr file conflicts. --- build_library/test_image_content.sh | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/build_library/test_image_content.sh b/build_library/test_image_content.sh index 2a4c660db8..1389e936a7 100644 --- a/build_library/test_image_content.sh +++ b/build_library/test_image_content.sh @@ -2,6 +2,37 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# Files that are known to conflict with /usr but are OK because they +# are already fixed by toggling the symlink-usr USE flag. +USR_CONFLICT_WHITELIST=( + /bin/awk + /bin/basename + /bin/chroot + /bin/cut + /bin/dir + /bin/dirname + /bin/du + /bin/env + /bin/expr + /bin/gawk + /bin/head + /bin/mkfifo + /bin/mktemp + /bin/passwd + /bin/readlink + /bin/seq + /bin/sleep + /bin/sort + /bin/tail + /bin/touch + /bin/tr + /bin/tty + /bin/uname + /bin/vdir + /bin/wc + /bin/yes +) + test_image_content() { local root="$1" local returncode=0 @@ -55,5 +86,29 @@ test_image_content() { fi fi + # Check that there are no conflicts between /* and /usr/* + # TODO(marineam): Before symlinking to /usr this test will need to be + # rewritten to query the package database instead of the filesystem. + local check_dir + for check_dir in "${root}"/usr/*; do + if [[ ! -d "${check_dir}" || -h "${check_dir}" ]]; then + continue + fi + for check_file in "${check_dir}"/*; do + root_file="${root}${check_file##*/usr}" + trimmed_path="${root_file#${root}}" + local whitelist + for whitelist in "${USR_CONFLICT_WHITELIST[@]}"; do + if [[ "${trimmed_path}" == "${whitelist}" ]]; then + continue 2 + fi + done + if [[ -e "${root_file}" || -h "${root_file}" ]]; then + # TODO(marineam): make fatal before switching to symlinks + warn "test_image_content: ${root_file#${root}} conflicts with /usr" + fi + done + done + return $returncode } From 4e8bef60d784dc01816be7690134a02d6ae4fbfa Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 27 Nov 2013 14:54:13 -0800 Subject: [PATCH 5/5] fix(test_image_content): Clean out some stray tab chars --- build_library/test_image_content.sh | 56 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/build_library/test_image_content.sh b/build_library/test_image_content.sh index 1389e936a7..c624d4b7e2 100644 --- a/build_library/test_image_content.sh +++ b/build_library/test_image_content.sh @@ -5,32 +5,32 @@ # Files that are known to conflict with /usr but are OK because they # are already fixed by toggling the symlink-usr USE flag. USR_CONFLICT_WHITELIST=( - /bin/awk - /bin/basename - /bin/chroot - /bin/cut - /bin/dir - /bin/dirname - /bin/du - /bin/env - /bin/expr - /bin/gawk - /bin/head - /bin/mkfifo - /bin/mktemp - /bin/passwd - /bin/readlink - /bin/seq - /bin/sleep - /bin/sort - /bin/tail - /bin/touch - /bin/tr - /bin/tty - /bin/uname - /bin/vdir - /bin/wc - /bin/yes + /bin/awk + /bin/basename + /bin/chroot + /bin/cut + /bin/dir + /bin/dirname + /bin/du + /bin/env + /bin/expr + /bin/gawk + /bin/head + /bin/mkfifo + /bin/mktemp + /bin/passwd + /bin/readlink + /bin/seq + /bin/sleep + /bin/sort + /bin/tail + /bin/touch + /bin/tr + /bin/tty + /bin/uname + /bin/vdir + /bin/wc + /bin/yes ) test_image_content() { @@ -100,8 +100,8 @@ test_image_content() { local whitelist for whitelist in "${USR_CONFLICT_WHITELIST[@]}"; do if [[ "${trimmed_path}" == "${whitelist}" ]]; then - continue 2 - fi + continue 2 + fi done if [[ -e "${root_file}" || -h "${root_file}" ]]; then # TODO(marineam): make fatal before switching to symlinks