From a7f251da4493fe3009196be9412cd930dac81b43 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Thu, 12 Aug 2021 15:31:00 +0200 Subject: [PATCH 1/3] build_library: use proper vocabulary in generate_au_zip Replace black list with deny list, white list with allow list. --- build_library/generate_au_zip.py | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 4096a16413..e4cca92a69 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -33,14 +33,14 @@ DYNAMIC_EXECUTABLES = ['/usr/bin/delta_generator', '/usr/bin/bspatch'] # These files will be ignored when present in the dependancy list. -BLACK_LIST = [ +DENY_LIST = [ # This library does not exist on disk, but is inserted into the # executable's memory space when the executable is loaded by the kernel. 'linux-vdso.so', ] # These files MUST be present in the dependancy list. -WHITE_LIST = [ +ALLOW_LIST = [ # Update WrapExecutableFiles if this file changes names 'ld-linux-x86-64.so.2', ] @@ -97,7 +97,7 @@ def DepsToCopy(ldd_files): """Returns a list of deps for a given dynamic executables list. Args: ldd_files: List of dynamic files that needs to have the deps evaluated - black_list: List of files that we should ignore + deny_list: List of files that we should ignore Returns: List of files that are dependencies """ @@ -129,8 +129,8 @@ def DepsToCopy(ldd_files): logging.error("ldd for %s failed: %s", file_name, ex) sys.exit(1) - result = _ExcludeBlacklist(list(libs), BLACK_LIST) - _EnforceWhiteList(list(libs), WHITE_LIST) + result = _ExcludeDenylist(list(libs), DENY_LIST) + _EnforceAllowList(list(libs), ALLOW_LIST) return result @@ -249,26 +249,26 @@ def GenerateZipFile(base_name, root_dir): return True -def _ExcludeBlacklist(library_list, black_list=[]): - """Deletes the set of files from black_list from the library_list +def _ExcludeDenylist(library_list, deny_list=[]): + """Deletes the set of files from deny_list from the library_list Args: - library_list: List of the library names to filter through black_list - black_list: List of the black listed names to filter + library_list: List of the library names to filter through deny_list + deny_list: List of the deny listed names to filter Returns: Filtered library_list """ - if not black_list: + if not deny_list: return library_list return_list = [] - pattern = re.compile(r'|'.join(black_list)) + pattern = re.compile(r'|'.join(deny_list)) logging.debug('PATTERN: %s=', pattern) for library in library_list: if pattern.search(library): - logging.debug('BLACK-LISTED = %s=', library) + logging.debug('DENY-LISTED = %s=', library) continue return_list.append(library) @@ -277,17 +277,17 @@ def _ExcludeBlacklist(library_list, black_list=[]): return return_list -def _EnforceWhiteList(library_list, white_list=[]): - """Deletes the set of files from black_list from the library_list +def _EnforceAllowList(library_list, allow_list=[]): + """Deletes the set of files from deny_list from the library_list Args: - library_list: List of the library names to filter through black_list - black_list: List of the black listed names to filter + library_list: List of the library names to filter through deny_list + deny_list: List of the deny listed names to filter Returns: Filtered library_list """ - for white_item in white_list: - pattern = re.compile(white_item) + for allow_item in allow_list: + pattern = re.compile(allow_item) logging.debug('PATTERN: %s=', pattern) @@ -298,7 +298,7 @@ def _EnforceWhiteList(library_list, white_list=[]): break if not found: - logging.error('Required WHITE_LIST items %s not found!!!' % white_item) + logging.error('Required ALLOW_LIST items %s not found!!!' % allow_item) exit(1) From 34cb6d305ad14a0c3d06fd9e4ff1dc8c159eb878 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Thu, 12 Aug 2021 16:04:28 +0200 Subject: [PATCH 2/3] build_library: support multi-arch in generate_au_zip To be able to support arm64 native SDK without cross builds, we should make generate_au_zip support both architectures, amd64 and arm64. Without doing that, `build_image` fails with `ERROR : Required WHITE_LIST items ld-linux-x86-64.so.2 not found!!!`, because the script recognizes only amd64 libs in WHITE_LIST. We should first determine the architecture in build_image, before running generate_au_zip, and pass the architecture, either amd64 or arm64. Also add allow_list and ld_linux parameters to necessary functions. --- build_library/build_image_util.sh | 2 +- build_library/generate_au_zip.py | 42 ++++++++++++++++++++++--------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 8c443e161f..55f8f783d4 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -83,7 +83,7 @@ zip_update_tools() { # Make sure some vars this script needs are exported export REPO_MANIFESTS_DIR SCRIPTS_DIR "${BUILD_LIBRARY_DIR}/generate_au_zip.py" \ - --output-dir "${BUILD_DIR}" --zip-name "${update_zip}" + --arch "$(get_sdk_arch)" --output-dir "${BUILD_DIR}" --zip-name "${update_zip}" upload_image "${BUILD_DIR}/${update_zip}" } diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index e4cca92a69..8ba4f3d141 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -32,6 +32,9 @@ DYNAMIC_EXECUTABLES = ['/usr/bin/delta_generator', '/usr/bin/bsdiff', '/usr/bin/bspatch'] +LD_LINUX_AMD64 = 'ld-linux-x86-64.so.2' +LD_LINUX_ARM64 = 'ld-linux-aarch64.so.1' + # These files will be ignored when present in the dependancy list. DENY_LIST = [ # This library does not exist on disk, but is inserted into the @@ -40,9 +43,14 @@ DENY_LIST = [ ] # These files MUST be present in the dependancy list. -ALLOW_LIST = [ - # Update WrapExecutableFiles if this file changes names - 'ld-linux-x86-64.so.2', +# Update WrapExecutableFiles if this file changes names. +# Each architecture requires different allow list libs. +ALLOW_LIST_AMD64 = [ + LD_LINUX_AMD64, + ] + +ALLOW_LIST_ARM64 = [ + LD_LINUX_ARM64, ] LIB_DIR = 'lib.so' @@ -93,7 +101,7 @@ def _SplitAndStrip(data): return return_list -def DepsToCopy(ldd_files): +def DepsToCopy(ldd_files, allow_list): """Returns a list of deps for a given dynamic executables list. Args: ldd_files: List of dynamic files that needs to have the deps evaluated @@ -130,11 +138,11 @@ def DepsToCopy(ldd_files): sys.exit(1) result = _ExcludeDenylist(list(libs), DENY_LIST) - _EnforceAllowList(list(libs), ALLOW_LIST) + _EnforceAllowList(list(libs), allow_list=allow_list) return result -def CopyRequiredFiles(dest_files_root): +def CopyRequiredFiles(dest_files_root, allow_list): """Generates a list of files that are required for au-generator zip file Args: dest_files_root: location of the directory where we should copy the files @@ -161,7 +169,7 @@ def CopyRequiredFiles(dest_files_root): logging.exception("Copying '%s' to %s failed", file_name, dest_files_root) sys.exit(1) - libraries = DepsToCopy(ldd_files=DYNAMIC_EXECUTABLES) + libraries = DepsToCopy(ldd_files=DYNAMIC_EXECUTABLES, allow_list=allow_list) lib_dir = os.path.join(dest_files_root, LIB_DIR) os.mkdir(lib_dir) for file_name in libraries: @@ -188,7 +196,7 @@ def CopyRequiredFiles(dest_files_root): sys.exit(1) -def WrapExecutableFiles(dest_files_root): +def WrapExecutableFiles(dest_files_root, ld_linux): """Our dynamically linked executalbes have to be invoked use the library versions they were linked with inside the chroot (from libc on), as well as the dynamic linker they were built with inside the chroot. @@ -209,10 +217,10 @@ def WrapExecutableFiles(dest_files_root): script.write('# Auto-generated wrapper script\n') script.write('thisdir="$(dirname "$0")"\n') script.write('LD_LIBRARY_PATH=\n') - script.write('exec "$thisdir/%s/ld-linux-x86-64.so.2"' + script.write('exec "$thisdir/%s/%s"' ' --library-path "$thisdir/%s"' ' "$thisdir/%s.bin" "$@"\n' % - (LIB_DIR, LIB_DIR, base_exec)) + (LIB_DIR, ld_linux, LIB_DIR, base_exec)) def CleanUp(temp_dir): @@ -335,6 +343,8 @@ def main(): default='au-generator.zip', help='Name of the zip file') parser.add_option('-k', '--keep-temp', dest='keep_temp', default=False, action='store_true', help='Keep the temp files...',) + parser.add_option('-a', '--arch', dest='arch', + default='amd64', help='Arch amd64/arm64. Default: amd64',) (options, args) = parser.parse_args() if options.debug: @@ -345,8 +355,16 @@ def main(): temp_dir = CreateTempDir() dest_files_root = os.path.join(temp_dir, 'au-generator') os.makedirs(dest_files_root) - CopyRequiredFiles(dest_files_root=dest_files_root) - WrapExecutableFiles(dest_files_root=dest_files_root) + + if options.arch == 'arm64': + ld_linux = LD_LINUX_ARM64 + allow_list = ALLOW_LIST_ARM64 + else: + ld_linux = LD_LINUX_AMD64 + allow_list = ALLOW_LIST_AMD64 + + CopyRequiredFiles(dest_files_root=dest_files_root, allow_list=allow_list) + WrapExecutableFiles(dest_files_root=dest_files_root, ld_linux=ld_linux) zip_file_name = os.path.join(temp_dir, options.zip_name) GenerateZipFile(zip_file_name, dest_files_root) CopyZipToFinalDestination(options.output_dir, zip_file_name) From 04d9fea0c1e6296ece0b894603430634a91fa9fe Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Mon, 16 Aug 2021 10:39:26 +0200 Subject: [PATCH 3/3] build_library: fix docstring issues Fix several docstring issues, so that the docstrings could match with the actual code. --- build_library/generate_au_zip.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 8ba4f3d141..6db1a1bae3 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -105,7 +105,7 @@ def DepsToCopy(ldd_files, allow_list): """Returns a list of deps for a given dynamic executables list. Args: ldd_files: List of dynamic files that needs to have the deps evaluated - deny_list: List of files that we should ignore + allow_list: List of files that we should allow Returns: List of files that are dependencies """ @@ -146,6 +146,7 @@ def CopyRequiredFiles(dest_files_root, allow_list): """Generates a list of files that are required for au-generator zip file Args: dest_files_root: location of the directory where we should copy the files + allow_list: List of files that we should allow """ if not dest_files_root: logging.error('Invalid option passed for dest_files_root') @@ -286,12 +287,10 @@ def _ExcludeDenylist(library_list, deny_list=[]): def _EnforceAllowList(library_list, allow_list=[]): - """Deletes the set of files from deny_list from the library_list + """Ensures that library_list contains all the items from allow_list Args: - library_list: List of the library names to filter through deny_list - deny_list: List of the deny listed names to filter - Returns: - Filtered library_list + library_list: List of the library names to check + allow_list: List of the items that ought to be in the library_list """ for allow_item in allow_list: