From bd5c0c6a0e1747eba454a049ed2ec2236cad5af5 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 13 Aug 2021 18:15:42 +0200 Subject: [PATCH 1/8] check_out_of_date.py: Drop unused script The script is potentially useful, but it seems to be unused anyway. We can bring it back later if there's a need. Note that this will need updating it to python3 first. Which is why I'm dropping it currently - it's one python2 script less to port. --- check_out_of_date.py | 188 ------------------------------------------- 1 file changed, 188 deletions(-) delete mode 100755 check_out_of_date.py diff --git a/check_out_of_date.py b/check_out_of_date.py deleted file mode 100755 index 91785525ba..0000000000 --- a/check_out_of_date.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/python2 -# TODO: This script needs porting to python3, since portage is also in python3 now. - -# Prints out a list of all packages in portage-stable and how they stand relative to gentoo upstream - -import argparse -import json -import os -import subprocess -import sys - -import portage.versions - - -def split_package(p): - # split into cat/package,ver-rev - split = portage.versions.catpkgsplit(p.strip()) - return (split[0] + "/" + split[1], split[2] + "-" + split[3]) - - -def build_pkg_map(pkgs): - pkgs = map(split_package, pkgs) - package_map = dict() - for pkg, ver in pkgs: - if pkg not in package_map: - package_map[pkg] = [ver] - else: - package_map[pkg].append(ver) - return package_map - - -def exec_command_strict(cmd): - """ Wraps check_output splitting the input and string'ing the output""" - return bytes.decode(subprocess.check_output(cmd.split())) - - -def exec_command(cmd): - """ Like exec_command_strict but returns the output even if the command exited unsuccessfully""" - try: - return exec_command_strict(cmd) - except subprocess.CalledProcessError as e: - return bytes.decode(e.output) - - -def get_portage_tree_packages(tree_path): - """ returns a list of all packages in a portage tree/overlay in the form of cat/pkg-ver""" - pkgs = exec_command_strict("find -L {} -maxdepth 3 -type f -name *.ebuild -not -name skel.ebuild -printf %P\\n".format(tree_path)) - - def process_line(line): - # cat/pkg/pkg-ver.ebuild -> cat/pkg-ver - chunks = line.split("/") - end = chunks[2].replace(".ebuild", "") - return chunks[0] + "/" + end - return build_pkg_map(map(process_line, pkgs.splitlines())) - - -def process_emerge_output(eout): - """ transform from emerge --unordered-dispaly to cat/pkg-ver""" - def process_line(line): - return line.strip().split("] ")[1].split(":")[0] - - def is_package(line): - # none of the header line have a / - return "/" in line - - return map(process_line, filter(is_package, eout.splitlines())) - - -def get_board_packages(board): - """ gets a list of packages used by a board. valid boards are {arm,amd}64-usr, sdk, and bootstrap""" - emerge_args = "--emptytree --pretend --verbose --unordered-display" - if board == "sdk": - cmd = "emerge {} @system sdk-depends sdk-extras".format(emerge_args) - elif board == "amd64-usr" or board == "arm64-usr": - cmd = "emerge-{} {} @system board-packages".format(board, emerge_args) - elif board == "bootstrap": - pkgs = exec_command_strict("/usr/lib64/catalyst/targets/stage1/build.py") - cmd = "emerge {} {}".format(emerge_args, pkgs) - elif board == "image": - cmd = "emerge-amd64-usr {} --usepkgonly board-packages".format(emerge_args) - else: - raise "invalid board" - return build_pkg_map(process_emerge_output(exec_command(cmd))) - - -def print_table(report, head, line_head, line_tail, tail, joiner, pkg_joiner): - print(head) - # metapackage that acts as the header - report.insert(0, {"name": "Package", - "common": ["Common"], - "ours": ["Ours"], - "upstream": ["Upstream"], - "tag": "Tag", - "sdk": ["sdk"], - "arm64-usr": ["arm64-usr"], - "amd64-usr": ["amd64-usr"], - "bootstrap": ["bootstrap"], - "modified": "Modified"}) - for entry in report: - print(line_head + joiner.join([entry.get("name",""), - pkg_joiner.join(entry.get("common",[])), - pkg_joiner.join(entry.get("ours",[])), - pkg_joiner.join(entry.get("upstream",[])), - entry.get("tag",""), - pkg_joiner.join(entry.get("sdk", [])), - pkg_joiner.join(entry.get("arm64-usr", [])), - pkg_joiner.join(entry.get("amd64-usr", [])), - pkg_joiner.join(entry.get("bootstrap", [])), - entry.get("modified","")]) + line_tail) - print(tail) - - -def print_table_human(report): - print_table(report, "", "", "", "", "\t", " ") - - -def print_html_table(report): - print_table(report, "", "", "
", "
", "", "
") - - -def get_date(pkg, repo_root, fmt): - return exec_command_strict("git -C {} --no-pager log -1 --pretty=%ad --date={} {}".format(repo_root, fmt, pkg)).strip() - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--update-upstream", help="run git-pull in the gentoo mirror repo first", action="store_true") - parser.add_argument("--upstream-git", help="git uri to clone for upstream", default="https://github.com/gentoo/gentoo.git") - parser.add_argument("--upstream-path", help="path to gentoo tree", default="/mnt/host/source/src/gentoo-portage") - parser.add_argument("--portage-stable-path", help="path to portage-stable", default="/mnt/host/source/src/third_party/portage-stable") - parser.add_argument("--date-fmt", help="format for git-date to use", default="relative") - parser.add_argument("--output", help="output format, json, table, and html are accepted", default="json") - args = parser.parse_args() - - if not os.path.exists(args.upstream_path): - os.makedirs(args.upstream_path) - subprocess.check_call(["git", "clone", args.upstream_git, args.upstream_path]) - elif args.update_upstream: - # elif to not pull if we just cloned - subprocess.check_call(["git", "-C", args.upstream_path, "pull"]) - - pkg_lists = {} - sources = ["sdk", "bootstrap", "amd64-usr", "arm64-usr", "image"] - for i in sources: - pkg_lists[i] = get_board_packages(i) - - gentoo_packages = get_portage_tree_packages(args.upstream_path) - packages = get_portage_tree_packages(args.portage_stable_path) - - # time to make the report - report = [] - for pkg, vers in packages.iteritems(): - upstream = gentoo_packages.get(pkg, []) - - entry = { - "name": pkg, - "common": list(set(vers).intersection(upstream)), - "ours": list(set(vers).difference(upstream)), - "upstream": list(set(upstream).difference(vers)), - "modified": get_date(pkg, args.portage_stable_path, args.date_fmt) - } - if not entry["upstream"]: - entry["tag"] = "updated" - elif entry["common"]: - entry["tag"] = "has_update" - elif pkg in gentoo_packages: - entry["tag"] = "no_ebuild_upstream" - else: - entry["tag"] = "deleted_upstream" - - for src in sources: - if pkg in pkg_lists[src]: - entry[src] = pkg_lists[src][pkg] - report.append(entry) - - if args.output == "json": - print(json.dumps(report)) - elif args.output == "table": - print_table_human(report) - elif args.output == "html": - print_html_table(report) - else: - print("Unknown output type. Dying.") - sys.exit(2) - - -if __name__ == "__main__": - main() From df63498a915735a90684cf2c446ea371ead89eae Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 13 Aug 2021 18:17:52 +0200 Subject: [PATCH 2/8] common: Replace the use of python2 with python3 We want to move to python3 eventually so update the script snippet. We can be sure that python3 is installed, because portage already requires it. --- common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common.sh b/common.sh index a97c560fe8..6f6aa00435 100644 --- a/common.sh +++ b/common.sh @@ -843,8 +843,8 @@ reinterpret_path_for_chroot() { # Get the relative path between two locations. Handy for printing paths to # the user that will usually make sense both inside and outside the chroot. relpath() { - local py='import sys, os; print os.path.relpath(sys.argv[1], sys.argv[2])' - python2 -c "${py}" "${1}" "${2:-.}" + local py='import sys, os; print(os.path.relpath(sys.argv[1], sys.argv[2]))' + python3 -c "${py}" "${1}" "${2:-.}" } enable_strict_sudo() { From 336a96794106b42540b5cfc118a383213c45a71f Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 13 Aug 2021 18:46:17 +0200 Subject: [PATCH 3/8] build_library: Convert python2 scripts to python3 This is just a conversion done by 2to3 with a manual updates of shebangs to mention python3 explicitly. The fixups for bytearray vs string issues will follow up. --- build_library/disk_util | 84 +++++++++++++-------------- build_library/gen_tmpfiles.py | 4 +- build_library/generate_au_zip.py | 12 ++-- build_library/generate_grub_hashes.py | 4 +- build_library/generate_kernel_hash.sh | 4 +- 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 267ebb32ac..8505c0db77 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -66,9 +66,9 @@ def LoadPartitionConfig(options): raise InvalidLayout('Metadata is missing required entries: %s' % e) def VerifyLayout(layout_name, layout, base=None): - for part_num, part in layout.iteritems(): + for part_num, part in layout.items(): part['num'] = int(part_num) - part_keys = set(part.iterkeys()) + part_keys = set(part.keys()) unknown_keys = part_keys - valid_layout_keys if unknown_keys: raise InvalidLayout('Unknown items in partition %s %s: %r' % @@ -79,7 +79,7 @@ def LoadPartitionConfig(options): if base: part_base = base.get(part_num, {}) - part_keys.update(part_base.iterkeys()) + part_keys.update(iter(part_base.keys())) if part.get('type', None) == 'blank': continue @@ -122,12 +122,12 @@ def LoadPartitionConfig(options): # Fill in default values from base, # dict doesn't have a update()+setdefault() method so this looks tedious if base: - for part_num, base_part in base.iteritems(): + for part_num, base_part in base.items(): part = layout.setdefault(part_num, {}) - for base_key, base_value in base_part.iteritems(): + for base_key, base_value in base_part.items(): part.setdefault(base_key, base_value) - for part_num, part in sorted(layout.iteritems(), key=lambda t: int(t[0])): + for part_num, part in sorted(iter(layout.items()), key=lambda t: int(t[0])): if part['type'] == 'blank': continue @@ -163,7 +163,7 @@ def LoadPartitionConfig(options): # Verify 'base' before other layouts because it is inherited by the others # Fill in extra/default values in base last so they aren't inherited VerifyLayout('base', base) - for layout_name, layout in config['layouts'].iteritems(): + for layout_name, layout in config['layouts'].items(): if layout_name == 'base': continue VerifyLayout(layout_name, layout, base) @@ -225,7 +225,7 @@ def GetPartitionTableFromImage(options, config, partitions): else: part['image_compat'] = False - for part in partitions.itervalues(): + for part in partitions.values(): if part.get('type', 'blank') == 'blank': continue if not part.get('image_exists', False): @@ -240,7 +240,7 @@ def GetPartitionTableFromImage(options, config, partitions): part['image_fs_type'] = None # Set compat flags for any partition not in the image - for part in partitions.itervalues(): + for part in partitions.values(): part.setdefault('image_exists', False) if part.get('type', 'blank') == 'blank': part.setdefault('image_compat', True) @@ -269,7 +269,7 @@ def WritePartitionTable(options, config=None, partitions=None): else: # If we are not creating a fresh image all partitions must be compatible. GetPartitionTableFromImage(options, config, partitions) - if not all(p['image_compat'] for p in partitions.itervalues()): + if not all(p['image_compat'] for p in partitions.values()): raise InvalidLayout("New disk layout is incompatible existing image") # Extend the disk image size as needed @@ -279,7 +279,7 @@ def WritePartitionTable(options, config=None, partitions=None): hybrid = None prioritize = [] - for partition in partitions.itervalues(): + for partition in partitions.values(): if partition['type'] != 'blank': Cgpt('add', '-i', partition['num'], '-b', partition['first_block'], @@ -469,8 +469,8 @@ def PartitionLoop(options, partition): def FormatPartition(options, part): - print "Formatting partition %s (%s) as %s" % ( - part['num'], part['label'], part['fs_type']) + print("Formatting partition %s (%s) as %s" % ( + part['num'], part['label'], part['fs_type'])) with PartitionLoop(options, part) as loop_dev: if part['fs_type'] in ('ext2', 'ext4'): @@ -497,7 +497,7 @@ def Format(options): config, partitions = LoadPartitionConfig(options) WritePartitionTable(options, config, partitions) - for part in partitions.itervalues(): + for part in partitions.values(): if part['type'] == 'blank' or 'fs_type' not in part: continue @@ -542,13 +542,13 @@ def Update(options): config, partitions = LoadPartitionConfig(options) WritePartitionTable(options, config, partitions) - for part in partitions.itervalues(): + for part in partitions.values(): if not part.get('fs_type', None): continue elif not part['image_fs_type']: FormatPartition(options, part) - for part in partitions.itervalues(): + for part in partitions.values(): resize_func = None if not part.get('fs_type', None): continue @@ -561,8 +561,8 @@ def Update(options): else: continue - print "Resizing partition %s (%s) to %s bytes" % ( - part['num'], part['label'], part['fs_bytes']) + print("Resizing partition %s (%s) to %s bytes" % ( + part['num'], part['label'], part['fs_bytes'])) with PartitionLoop(options, part) as loop_dev: resize_func(part, loop_dev) @@ -582,7 +582,7 @@ def Mount(options): GetPartitionTableFromImage(options, config, partitions) mounts = {} - for part_num, part in partitions.iteritems(): + for part_num, part in partitions.items(): path = part.get('mount', None) if not path or not path.startswith('/'): continue @@ -625,7 +625,7 @@ def Mount(options): if err is not None: raise err - for src, dst in mount.get('binds', {}).iteritems(): + for src, dst in mount.get('binds', {}).items(): # src may be relative or absolute, os.path.join handles this. full_src = os.path.realpath( options.mount_dir + os.path.join(mount['mount'], src)) @@ -650,11 +650,11 @@ def ReadWriteSubvol(options, partition, disable_rw): """ if disable_rw: - print "Disabling read-write on default subvolume of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Disabling read-write on default subvolume of partition %s (%s)" % ( + partition['num'], partition['label'])) else: - print "Enabling read-write on default subvolume of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Enabling read-write on default subvolume of partition %s (%s)" % ( + partition['num'], partition['label'])) with PartitionLoop(options, partition) as loop_dev: btrfs_mount = tempfile.mkdtemp() @@ -701,11 +701,11 @@ def Tune2fsReadWrite(options, partition, disable_rw): """ if disable_rw: - print "Disabling read-write mounting of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Disabling read-write mounting of partition %s (%s)" % ( + partition['num'], partition['label'])) else: - print "Enabling read-write mounting of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Enabling read-write mounting of partition %s (%s)" % ( + partition['num'], partition['label'])) # offset of ro_compat, highest order byte (le 32 bit field) flag_offset = 0x464 + 3 @@ -767,7 +767,7 @@ def Verity(options): config, partitions = LoadPartitionConfig(options) GetPartitionTableFromImage(options, config, partitions) - for part_num, part in partitions.iteritems(): + for part_num, part in partitions.items(): if 'verity' not in part.get('features', []): continue @@ -848,7 +848,7 @@ def GetPartitionByLabel(partitions, label): Returns: An object for the selected partition """ - for partition in partitions.itervalues(): + for partition in partitions.values(): if partition['type'] == 'blank': continue elif partition['label'] == label: @@ -882,7 +882,7 @@ def GetBlockSize(options): """ config, partitions = LoadPartitionConfig(options) - print config['metadata']['block_size'] + print(config['metadata']['block_size']) def GetFilesystemBlockSize(options): @@ -897,7 +897,7 @@ def GetFilesystemBlockSize(options): """ config, partitions = LoadPartitionConfig(options) - print config['metadata']['fs_block_size'] + print(config['metadata']['fs_block_size']) def GetPartitionSize(options): @@ -911,7 +911,7 @@ def GetPartitionSize(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByNumber(partitions, options.partition_num) - print partition['bytes'] + print(partition['bytes']) def GetFilesystemSize(options): @@ -927,7 +927,7 @@ def GetFilesystemSize(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByNumber(partitions, options.partition_num) - print partition.get('fs_bytes', partition['bytes']) + print(partition.get('fs_bytes', partition['bytes'])) def GetLabel(options): @@ -941,7 +941,7 @@ def GetLabel(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByNumber(partitions, options.partition_num) - print partition.get('label', 'UNTITLED') + print(partition.get('label', 'UNTITLED')) def GetNum(options): @@ -955,7 +955,7 @@ def GetNum(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByLabel(partitions, options.label) - print partition.get('num', '-1') + print(partition.get('num', '-1')) def GetUuid(options): @@ -969,7 +969,7 @@ def GetUuid(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByLabel(partitions, options.label) - print partition.get('uuid', '') + print(partition.get('uuid', '')) def DoDebugOutput(options): @@ -983,7 +983,7 @@ def DoDebugOutput(options): """ partitions = GetPartitionTableFromConfig(options) - for num, partition in sorted(partitions.iteritems()): + for num, partition in sorted(partitions.items()): if partition['type'] != 'blank': if partition['bytes'] < 1024 * 1024: size = '%d bytes' % partition['bytes'] @@ -994,11 +994,11 @@ def DoDebugOutput(options): fs_size = '%d bytes' % partition['fs_bytes'] else: fs_size = '%d MB' % (partition['fs_bytes'] / 1024 / 1024) - print '%s: %s - %s/%s' % (num, partition['label'], fs_size, size) + print('%s: %s - %s/%s' % (num, partition['label'], fs_size, size)) else: - print '%s: %s - %s' % (num, partition['label'], size) + print('%s: %s - %s' % (num, partition['label'], size)) else: - print '%s: blank' % num + print('%s: blank' % num) def DoParseOnly(options): diff --git a/build_library/gen_tmpfiles.py b/build_library/gen_tmpfiles.py index c0133f6dbd..a4b4134ba1 100755 --- a/build_library/gen_tmpfiles.py +++ b/build_library/gen_tmpfiles.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 '''Scan an existing directory tree and record installed directories. During build a number of directories under /var are created in the state @@ -82,7 +82,7 @@ def main(): fd.write('\n'.join(config)+'\n') fd.close() else: - print '\n'.join(config) + print('\n'.join(config)) if __name__ == '__main__': main() diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 6db1a1bae3..437ccce112 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -120,7 +120,7 @@ def DepsToCopy(ldd_files, allow_list): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout_data, stderr_data) = proc.communicate(input=None) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: logging.error('Command %s failed', cmd) logging.error('error code %s', e.returncode) logging.error('ouput %s', e.output) @@ -153,7 +153,7 @@ def CopyRequiredFiles(dest_files_root, allow_list): sys.exit(1) all_files = DYNAMIC_EXECUTABLES + STATIC_FILES - all_files = map(os.path.expanduser, all_files) + all_files = list(map(os.path.expanduser, all_files)) for file_name in all_files: if not os.path.isfile(file_name): @@ -181,7 +181,7 @@ def CopyRequiredFiles(dest_files_root, allow_list): logging.exception("Copying '%s' to %s failed", file_name, lib_dir) sys.exit(1) - for source_dir, target_dir in RECURSE_DIRS.iteritems(): + for source_dir, target_dir in RECURSE_DIRS.items(): logging.debug('Processing directory %s', source_dir) full_path = os.path.expanduser(source_dir) if not os.path.isdir(full_path): @@ -212,7 +212,7 @@ def WrapExecutableFiles(dest_files_root, ld_linux): local_exec_wrapped = local_exec + ".bin" shutil.move(local_exec, local_exec_wrapped) - fd = os.open(local_exec, os.O_WRONLY | os.O_CREAT, 0733) + fd = os.open(local_exec, os.O_WRONLY | os.O_CREAT, 0o733) with os.fdopen(fd, 'w') as script: script.write('#!/bin/sh\n') script.write('# Auto-generated wrapper script\n') @@ -249,7 +249,7 @@ def GenerateZipFile(base_name, root_dir): try: subprocess.Popen(['zip', '-r', '-9', base_name, '.'], stdout=subprocess.PIPE).communicate()[0] - except OSError, e: + except OSError as e: logging.error('Execution failed:%s', e.strerror) return False finally: diff --git a/build_library/generate_grub_hashes.py b/build_library/generate_grub_hashes.py index 419e62e401..51660c26c7 100755 --- a/build_library/generate_grub_hashes.py +++ b/build_library/generate_grub_hashes.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import hashlib import json @@ -7,7 +7,7 @@ import string import subprocess import sys -filename = sys.argv[1] +filename = sys.argv[1] grubdir = sys.argv[2] outputdir = sys.argv[3] version = sys.argv[4] diff --git a/build_library/generate_kernel_hash.sh b/build_library/generate_kernel_hash.sh index 429fe9cb7d..6e7549fe93 100755 --- a/build_library/generate_kernel_hash.sh +++ b/build_library/generate_kernel_hash.sh @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import hashlib import json @@ -10,4 +10,4 @@ version=sys.argv[2] with open(path, "rb") as f: kernel = f.read() - print json.dumps({"9": {"binaryvalues": [{"prefix": "grub_linux", "values": [{"value": hashlib.sha1(kernel).hexdigest(), "description": "flatcar-%s" % version}]}]}}) + print(json.dumps({"9": {"binaryvalues": [{"prefix": "grub_linux", "values": [{"value": hashlib.sha1(kernel).hexdigest(), "description": "flatcar-%s" % version}]}]}})) From b6c6a054044afbf23405bef810594d46d6d1e619 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 13 Aug 2021 18:47:37 +0200 Subject: [PATCH 4/8] build_library: Fix python script extension It's not a shell script. It never was a shell script. Must have been a mistake to name it as such. --- build_library/build_image_util.sh | 2 +- .../{generate_kernel_hash.sh => generate_kernel_hash.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename build_library/{generate_kernel_hash.sh => generate_kernel_hash.py} (100%) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 55f8f783d4..bbb80dcb6d 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -693,7 +693,7 @@ EOF if [[ -n "${pcr_policy}" ]]; then mkdir -p "${BUILD_DIR}/pcrs" - ${BUILD_LIBRARY_DIR}/generate_kernel_hash.sh \ + ${BUILD_LIBRARY_DIR}/generate_kernel_hash.py \ "${root_fs_dir}/boot/flatcar/vmlinuz-a" ${FLATCAR_VERSION} \ >"${BUILD_DIR}/pcrs/kernel.config" fi diff --git a/build_library/generate_kernel_hash.sh b/build_library/generate_kernel_hash.py similarity index 100% rename from build_library/generate_kernel_hash.sh rename to build_library/generate_kernel_hash.py From ea0f478cfa9b225fb7db5928a99ee23b77c51062 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 17 Aug 2021 11:33:07 +0200 Subject: [PATCH 5/8] build_library: Fix some string vs bytes issues This is some fallout from converting scripts from python2 to python3. Output received from the functions in subprocess module now return bytearrays, but we operate on them as if they were a text. So decode the bytearrays to strings. Otherwise we are either getting some junk values passed to the command line utilities (for example: `b'/dev/loop2'` instead of `/dev/loop2`), or exceptions are thrown, because a function expected a string. --- build_library/disk_util | 7 ++++--- build_library/generate_au_zip.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 8505c0db77..643b159667 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -198,7 +198,7 @@ def GetPartitionTableFromImage(options, config, partitions): """ block_size = config['metadata']['block_size'] cgpt_show = subprocess.check_output( - ['cgpt', 'show', '-q', options.disk_image]) + ['cgpt', 'show', '-q', options.disk_image]).decode('utf8') for line in cgpt_show.split('\n'): if not line.strip(): continue @@ -235,7 +235,7 @@ def GetPartitionTableFromImage(options, config, partitions): with PartitionLoop(options, part) as loop_dev: try: part['image_fs_type'] = subprocess.check_output( - ['sudo', 'blkid', '-o', 'value', '-s', 'TYPE', loop_dev]).strip() + ['sudo', 'blkid', '-o', 'value', '-s', 'TYPE', loop_dev]).strip().decode('utf8') except subprocess.CalledProcessError: part['image_fs_type'] = None @@ -452,6 +452,7 @@ def PartitionLoop(options, partition): '--sizelimit', str(partition['bytes']), '--find', '--show', options.disk_image]) loop_dev = loop_dev.strip() + loop_dev = loop_dev.decode('utf8') err = None break except subprocess.CalledProcessError as error: @@ -785,7 +786,7 @@ def Verity(options): '--hash-block-size', part['fs_block_size'], '--data-blocks', part['fs_blocks'], '--hash-offset', part['fs_bytes'], - loop_dev, loop_dev]) + loop_dev, loop_dev]).decode('utf8') print(verityout.strip()) m = re.search("Root hash:\s+([a-f0-9]{64})$", verityout, re.IGNORECASE|re.MULTILINE) if not m: diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 437ccce112..2a660a91ef 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -128,6 +128,8 @@ def DepsToCopy(ldd_files, allow_list): if not stdout_data: continue + stdout_data = stdout_data.decode('utf8') + stderr_data = stderr_data.decode('utf8') logging.debug('ldd for %s = stdout = %s stderr =%s', file_name, stdout_data, stderr_data) From a314348a38fa8631037a8ffb1009029f1f544456 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 18 Aug 2021 10:00:57 +0200 Subject: [PATCH 6/8] build_library: Fix getting integer from string `string.atoi` was deprecated since at least the first release of python2, and looks like it was dropped in python3. --- build_library/generate_grub_hashes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/generate_grub_hashes.py b/build_library/generate_grub_hashes.py index 51660c26c7..bd9ab9e838 100755 --- a/build_library/generate_grub_hashes.py +++ b/build_library/generate_grub_hashes.py @@ -11,7 +11,7 @@ filename = sys.argv[1] grubdir = sys.argv[2] outputdir = sys.argv[3] version = sys.argv[4] -bootoffset = string.atoi(subprocess.check_output(['cgpt', 'show', '-i', '2', '-b', filename])) * 512 +bootoffset = int(subprocess.check_output(['cgpt', 'show', '-i', '2', '-b', filename])) * 512 with open(filename, "rb") as f: boot = f.read(440) f.seek(bootoffset) From 3f9c073b4be0c827d5f328ef63c69ffe7341572c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 18 Aug 2021 13:57:29 +0200 Subject: [PATCH 7/8] build_library: Fix some typos --- build_library/generate_au_zip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 2a660a91ef..f047d10af9 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -82,7 +82,7 @@ def _SplitAndStrip(data): Args: data: list of libraries from ldd output Returns: - list of libararies that we should copy + list of libraries that we should copy """ return_list = [] @@ -176,7 +176,7 @@ def CopyRequiredFiles(dest_files_root, allow_list): lib_dir = os.path.join(dest_files_root, LIB_DIR) os.mkdir(lib_dir) for file_name in libraries: - logging.debug('Copying file %s to %s', file_name, lib_dir) + logging.debug('Copying file %s to %s', file_name, lib_dir) try: shutil.copy2(file_name, lib_dir) except EnvironmentError: From f143645215fe4be8ee31c227a01354c669cc2995 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 24 Aug 2021 16:08:15 +0200 Subject: [PATCH 8/8] generate_au_zip: Remove useless line --- build_library/generate_au_zip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index f047d10af9..3e84a36ae0 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -163,7 +163,6 @@ def CopyRequiredFiles(dest_files_root, allow_list): sys.exit(1) logging.debug('Given files that need to be copied = %s' % '' .join(all_files)) - all_files for file_name in all_files: logging.debug('Copying file %s to %s', file_name, dest_files_root) try: