Fix delta generation to work with updated glibc version.

The new glibc caused a series of problems with Paygen based delta generation.

First, the new glibc is newer than what is used in workstations, or on
the Paygen servers. Since libc wasn't one of the libraries bundled up
by generate_au_zip.py, the executables bundled up (including delta_generator)
would fail because of unstatisfied library requirements at startup.

When the new libc was included by generate_au_zip.py, the delta_generator
executable started causing the dynamic loader to segfault during startup,
presumably because it was linked for a newer version. This means that
the new loader needs to be bundled and explicitly used when invoking
the new executables (thanks David James for figuring this out!).

Next, after including all of the new libraries, bash would crash at startup
with these libraries at the start of LD_LIBRARY_PATH. Since Paygen which
set LD_LIBRARY_PATH before invoking a shell script which invokes the delta
generator, this means delta generation would crash before it started.

If I modified Paygen to not set LD_LIBRARY_PATH, then older au_generator.zip
files could not be properly handled (and we do so regularly).

This change moves the required dynamic libraries into a subdir of the zip
file which will not be in Paygens LD_LIBRARY_PATH which allows bash to
operate correctly.

It also renames each dynamically linked executable from xxx to xxx.wrapped
and creates a shell script named xxx which invokes xxx.wrapped with the
new LD_LIBRARY_PATH and using ld-linux-x86-64.so.2 to do so.

This change also moves a number of constants from inline in various
functions to constants at the top of the script, and introduces a WHITE_LIST
to cause a build failure if ld-linux-x86-64.so.2 isn't present.
(Fixing chromium-os:32550 would help with that)

BUG=chromium-os:32542
TEST=Generated au_generator.zip by hand, extracted files in a test directory,
and generated a delta using Paygen's command line:

LD_LIBRARY_PATH=. PATH=.:$PATH ./cros_generate_update_payload --image ../chromiumos_test_image.bin --output fuzzy --outside_chroot

Change-Id: I90d18a6d17a8f9824b19a6ce480048e388832b56
Reviewed-on: https://gerrit.chromium.org/gerrit/27443
Reviewed-by: Jay Srinivasan <jaysri@chromium.org>
Tested-by: Don Garrett <dgarrett@chromium.org>
Commit-Ready: Don Garrett <dgarrett@chromium.org>

build_library/generate_au_zip.py

@@ -1,11 +1,11 @@
 #!/usr/bin/python
 
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
-  Script to generate a zip file of delta-generator and it's dependencies.
+  Script to generate a zip file of delta-generator and its dependencies.
 """
 import logging.handlers
 import optparse
@@ -17,6 +17,37 @@ import sys
 import tempfile
 
 # GLOBALS
+image_sign_dir = '~/trunk/src/platform/vboot_reference/scripts/image_signing'
+STATIC_FILES = ['/usr/bin/cgpt',
+                '~/trunk/src/scripts/common.sh',
+                '~/trunk/src/scripts/common_bash_backtraces.sh',
+                '/usr/bin/cros_generate_update_payload',
+                '~/trunk/src/scripts/chromeos-common.sh',
+                os.path.join(image_sign_dir, 'convert_recovery_to_ssd.sh'),
+                os.path.join(image_sign_dir, 'common_minimal.sh'),
+                ]
+
+DYNAMIC_EXECUTABLES = ['/usr/bin/delta_generator',
+                       '/usr/bin/bsdiff',
+                       '/usr/bin/bspatch']
+
+# These files will be ignored when present in the dependancy list.
+BLACK_LIST = [
+    # This library does not exist on disk, but is inserted into the
+    # executable's memory space when the executable is loaded by the kernel.
+    'linux-vdso.so',
+    ]
+
+# These files MUST be present in the dependancy list.
+WHITE_LIST = [
+    # Update WrapExecutableFiles if this file changes names
+    'ld-linux-x86-64.so.2',
+    ]
+
+LIB_DIR = 'lib.so'
+
+# We need directories to be copied recursively to a dest within tempdir
+RECURSE_DIRS = {'~/trunk/src/scripts/lib/shflags': 'lib/shflags'}
 
 logging_format = '%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s'
 date_format = '%Y/%m/%d %H:%M:%S'
@@ -57,7 +88,7 @@ def _SplitAndStrip(data):
   return return_list
 
 
-def DepsToCopy(ldd_files, black_list=[]):
+def DepsToCopy(ldd_files):
   """Returns a list of deps for a given dynamic executables list.
     Args:
       ldd_files: List of dynamic files that needs to have the deps evaluated
@@ -89,7 +120,9 @@ def DepsToCopy(ldd_files, black_list=[]):
 
     libs |= set(_SplitAndStrip(stdout_data))
 
-  return _ExcludeBlacklist(list(libs), black_list)
+  result = _ExcludeBlacklist(list(libs), BLACK_LIST)
+  _EnforceWhiteList(list(libs), WHITE_LIST)
+  return result
 
 
 def CopyRequiredFiles(dest_files_root):
@@ -100,26 +133,8 @@ def CopyRequiredFiles(dest_files_root):
   if not dest_files_root:
     logging.error('Invalid option passed for dest_files_root')
     sys.exit(1)
-  # Files that need to go through ldd
-  ldd_files = ['/usr/bin/delta_generator', '/usr/bin/bsdiff',
-               '/usr/bin/bspatch', '/usr/bin/cgpt']
-  # statically linked files and scripts etc.,
-  image_sign_dir = '~/trunk/src/platform/vboot_reference/scripts/image_signing'
-  static_files = ['~/trunk/src/scripts/common.sh',
-                  '/usr/bin/cros_generate_update_payload',
-                  '~/trunk/src/scripts/chromeos-common.sh',
-                  '%s/convert_recovery_to_ssd.sh' % image_sign_dir,
-                  '%s/common_minimal.sh' % image_sign_dir]
-  # We need directories to be copied recursively to a destination within tempdir
-  recurse_dirs = {'~/trunk/src/scripts/lib/shflags': 'lib/shflags'}
 
-  black_list = [
-    # This library does not exist on disk, but is inserted into the
-    # executable's memory space when the executable is loaded by the kernel.
-    'linux-vdso.so',
-    ]
-
-  all_files = ldd_files + static_files
+  all_files = DYNAMIC_EXECUTABLES + STATIC_FILES
   all_files = map(os.path.expanduser, all_files)
 
   for file_name in all_files:
@@ -128,12 +143,19 @@ def CopyRequiredFiles(dest_files_root):
       sys.exit(1)
 
   logging.debug('Given files that need to be copied = %s' % '' .join(all_files))
-  all_files += DepsToCopy(ldd_files=ldd_files,black_list=black_list)
+  all_files
   for file_name in all_files:
     logging.debug('Copying file  %s to %s', file_name, dest_files_root)
     shutil.copy2(file_name, dest_files_root)
 
-  for source_dir, target_dir in recurse_dirs.iteritems():
+  libraries = DepsToCopy(ldd_files=DYNAMIC_EXECUTABLES)
+  lib_dir = os.path.join(dest_files_root, LIB_DIR)
+  os.mkdir(lib_dir)
+  for file_name in libraries:
+    logging.debug('Copying file  %s to %s', file_name, lib_dir)
+    shutil.copy2(file_name, lib_dir)
+
+  for source_dir, target_dir in RECURSE_DIRS.iteritems():
     logging.debug('Processing directory %s', source_dir)
     full_path = os.path.expanduser(source_dir)
     if not os.path.isdir(full_path):
@@ -144,6 +166,34 @@ def CopyRequiredFiles(dest_files_root):
   logging.debug('Copying directory %s to %s.', full_path, target_dir)
   shutil.copytree(full_path, dest)
 
+
+def WrapExecutableFiles(dest_files_root):
+  """Our dynamically linked executalbes have to be invoked use the library
+     versions they were linked with inside the chroot (from libc on), as well
+     as the dynamic linker they were built with inside the chroot.
+
+     So, this code moves the execs to backup names, and then creates a shell
+     script wrapper which invokes them in the proper way.
+  """
+
+  for src_exec in DYNAMIC_EXECUTABLES:
+    base_exec = os.path.basename(src_exec)
+    local_exec = os.path.join(dest_files_root, base_exec)
+    local_exec_wrapped = local_exec + ".bin"
+    shutil.move(local_exec, local_exec_wrapped)
+
+    fd = os.open(local_exec, os.O_WRONLY | os.O_CREAT, 0733)
+    with os.fdopen(fd, 'w') as script:
+      script.write('#!/bin/sh\n')
+      script.write('# Auto-generated wrapper script\n')
+      script.write('thisdir="$(dirname "$0")"\n')
+      script.write('LD_LIBRARY_PATH=\n')
+      script.write('exec "$thisdir/%s/ld-linux-x86-64.so.2"'
+                   ' --library-path "$thisdir/%s"'
+                   ' "$thisdir/%s.bin" "$@"\n' %
+                   (LIB_DIR, LIB_DIR, base_exec))
+
+
 def CleanUp(temp_dir):
   """Cleans up the tempdir
     Args:
@@ -153,6 +203,7 @@ def CleanUp(temp_dir):
     shutil.rmtree(temp_dir, ignore_errors=True)
     logging.debug('Removed tempdir = %s', temp_dir)
 
+
 def GenerateZipFile(base_name, root_dir):
   """Returns true if able to generate zip file
     Args:
@@ -205,6 +256,31 @@ def _ExcludeBlacklist(library_list, black_list=[]):
   return return_list
 
 
+def _EnforceWhiteList(library_list, white_list=[]):
+  """Deletes the set of files from black_list from the library_list
+    Args:
+      library_list: List of the library names to filter through black_list
+      black_list: List of the black listed names to filter
+    Returns:
+      Filtered library_list
+  """
+
+  for white_item in white_list:
+    pattern = re.compile(white_item)
+
+    logging.debug('PATTERN: %s=', pattern)
+
+    found = False
+    for library in library_list:
+      if pattern.search(library):
+        found = True
+        break
+
+    if not found:
+      logging.error('Required WHITE_LIST items %s not found!!!' % white_item)
+      exit(1)
+
+
 def CopyZipToFinalDestination(output_dir, zip_file_name):
   """Copies the generated zip file to a final destination
   Args:
@@ -249,6 +325,7 @@ def main():
   dest_files_root = os.path.join(temp_dir, 'au-generator')
   os.makedirs(dest_files_root)
   CopyRequiredFiles(dest_files_root=dest_files_root)
+  WrapExecutableFiles(dest_files_root=dest_files_root)
   zip_file_name = os.path.join(temp_dir, options.zip_name)
   GenerateZipFile(zip_file_name, dest_files_root)
   CopyZipToFinalDestination(options.output_dir, zip_file_name)