diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 55f8f783d4..bbb80dcb6d 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -693,7 +693,7 @@ EOF if [[ -n "${pcr_policy}" ]]; then mkdir -p "${BUILD_DIR}/pcrs" - ${BUILD_LIBRARY_DIR}/generate_kernel_hash.sh \ + ${BUILD_LIBRARY_DIR}/generate_kernel_hash.py \ "${root_fs_dir}/boot/flatcar/vmlinuz-a" ${FLATCAR_VERSION} \ >"${BUILD_DIR}/pcrs/kernel.config" fi diff --git a/build_library/disk_util b/build_library/disk_util index 267ebb32ac..643b159667 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -66,9 +66,9 @@ def LoadPartitionConfig(options): raise InvalidLayout('Metadata is missing required entries: %s' % e) def VerifyLayout(layout_name, layout, base=None): - for part_num, part in layout.iteritems(): + for part_num, part in layout.items(): part['num'] = int(part_num) - part_keys = set(part.iterkeys()) + part_keys = set(part.keys()) unknown_keys = part_keys - valid_layout_keys if unknown_keys: raise InvalidLayout('Unknown items in partition %s %s: %r' % @@ -79,7 +79,7 @@ def LoadPartitionConfig(options): if base: part_base = base.get(part_num, {}) - part_keys.update(part_base.iterkeys()) + part_keys.update(iter(part_base.keys())) if part.get('type', None) == 'blank': continue @@ -122,12 +122,12 @@ def LoadPartitionConfig(options): # Fill in default values from base, # dict doesn't have a update()+setdefault() method so this looks tedious if base: - for part_num, base_part in base.iteritems(): + for part_num, base_part in base.items(): part = layout.setdefault(part_num, {}) - for base_key, base_value in base_part.iteritems(): + for base_key, base_value in base_part.items(): part.setdefault(base_key, base_value) - for part_num, part in sorted(layout.iteritems(), key=lambda t: int(t[0])): + for part_num, part in sorted(iter(layout.items()), key=lambda t: int(t[0])): if part['type'] == 'blank': continue @@ -163,7 +163,7 @@ def LoadPartitionConfig(options): # Verify 'base' before other layouts because it is inherited by the others # Fill in extra/default values in base last so they aren't inherited VerifyLayout('base', base) - for layout_name, layout in config['layouts'].iteritems(): + for layout_name, layout in config['layouts'].items(): if layout_name == 'base': continue VerifyLayout(layout_name, layout, base) @@ -198,7 +198,7 @@ def GetPartitionTableFromImage(options, config, partitions): """ block_size = config['metadata']['block_size'] cgpt_show = subprocess.check_output( - ['cgpt', 'show', '-q', options.disk_image]) + ['cgpt', 'show', '-q', options.disk_image]).decode('utf8') for line in cgpt_show.split('\n'): if not line.strip(): continue @@ -225,7 +225,7 @@ def GetPartitionTableFromImage(options, config, partitions): else: part['image_compat'] = False - for part in partitions.itervalues(): + for part in partitions.values(): if part.get('type', 'blank') == 'blank': continue if not part.get('image_exists', False): @@ -235,12 +235,12 @@ def GetPartitionTableFromImage(options, config, partitions): with PartitionLoop(options, part) as loop_dev: try: part['image_fs_type'] = subprocess.check_output( - ['sudo', 'blkid', '-o', 'value', '-s', 'TYPE', loop_dev]).strip() + ['sudo', 'blkid', '-o', 'value', '-s', 'TYPE', loop_dev]).strip().decode('utf8') except subprocess.CalledProcessError: part['image_fs_type'] = None # Set compat flags for any partition not in the image - for part in partitions.itervalues(): + for part in partitions.values(): part.setdefault('image_exists', False) if part.get('type', 'blank') == 'blank': part.setdefault('image_compat', True) @@ -269,7 +269,7 @@ def WritePartitionTable(options, config=None, partitions=None): else: # If we are not creating a fresh image all partitions must be compatible. GetPartitionTableFromImage(options, config, partitions) - if not all(p['image_compat'] for p in partitions.itervalues()): + if not all(p['image_compat'] for p in partitions.values()): raise InvalidLayout("New disk layout is incompatible existing image") # Extend the disk image size as needed @@ -279,7 +279,7 @@ def WritePartitionTable(options, config=None, partitions=None): hybrid = None prioritize = [] - for partition in partitions.itervalues(): + for partition in partitions.values(): if partition['type'] != 'blank': Cgpt('add', '-i', partition['num'], '-b', partition['first_block'], @@ -452,6 +452,7 @@ def PartitionLoop(options, partition): '--sizelimit', str(partition['bytes']), '--find', '--show', options.disk_image]) loop_dev = loop_dev.strip() + loop_dev = loop_dev.decode('utf8') err = None break except subprocess.CalledProcessError as error: @@ -469,8 +470,8 @@ def PartitionLoop(options, partition): def FormatPartition(options, part): - print "Formatting partition %s (%s) as %s" % ( - part['num'], part['label'], part['fs_type']) + print("Formatting partition %s (%s) as %s" % ( + part['num'], part['label'], part['fs_type'])) with PartitionLoop(options, part) as loop_dev: if part['fs_type'] in ('ext2', 'ext4'): @@ -497,7 +498,7 @@ def Format(options): config, partitions = LoadPartitionConfig(options) WritePartitionTable(options, config, partitions) - for part in partitions.itervalues(): + for part in partitions.values(): if part['type'] == 'blank' or 'fs_type' not in part: continue @@ -542,13 +543,13 @@ def Update(options): config, partitions = LoadPartitionConfig(options) WritePartitionTable(options, config, partitions) - for part in partitions.itervalues(): + for part in partitions.values(): if not part.get('fs_type', None): continue elif not part['image_fs_type']: FormatPartition(options, part) - for part in partitions.itervalues(): + for part in partitions.values(): resize_func = None if not part.get('fs_type', None): continue @@ -561,8 +562,8 @@ def Update(options): else: continue - print "Resizing partition %s (%s) to %s bytes" % ( - part['num'], part['label'], part['fs_bytes']) + print("Resizing partition %s (%s) to %s bytes" % ( + part['num'], part['label'], part['fs_bytes'])) with PartitionLoop(options, part) as loop_dev: resize_func(part, loop_dev) @@ -582,7 +583,7 @@ def Mount(options): GetPartitionTableFromImage(options, config, partitions) mounts = {} - for part_num, part in partitions.iteritems(): + for part_num, part in partitions.items(): path = part.get('mount', None) if not path or not path.startswith('/'): continue @@ -625,7 +626,7 @@ def Mount(options): if err is not None: raise err - for src, dst in mount.get('binds', {}).iteritems(): + for src, dst in mount.get('binds', {}).items(): # src may be relative or absolute, os.path.join handles this. full_src = os.path.realpath( options.mount_dir + os.path.join(mount['mount'], src)) @@ -650,11 +651,11 @@ def ReadWriteSubvol(options, partition, disable_rw): """ if disable_rw: - print "Disabling read-write on default subvolume of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Disabling read-write on default subvolume of partition %s (%s)" % ( + partition['num'], partition['label'])) else: - print "Enabling read-write on default subvolume of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Enabling read-write on default subvolume of partition %s (%s)" % ( + partition['num'], partition['label'])) with PartitionLoop(options, partition) as loop_dev: btrfs_mount = tempfile.mkdtemp() @@ -701,11 +702,11 @@ def Tune2fsReadWrite(options, partition, disable_rw): """ if disable_rw: - print "Disabling read-write mounting of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Disabling read-write mounting of partition %s (%s)" % ( + partition['num'], partition['label'])) else: - print "Enabling read-write mounting of partition %s (%s)" % ( - partition['num'], partition['label']) + print("Enabling read-write mounting of partition %s (%s)" % ( + partition['num'], partition['label'])) # offset of ro_compat, highest order byte (le 32 bit field) flag_offset = 0x464 + 3 @@ -767,7 +768,7 @@ def Verity(options): config, partitions = LoadPartitionConfig(options) GetPartitionTableFromImage(options, config, partitions) - for part_num, part in partitions.iteritems(): + for part_num, part in partitions.items(): if 'verity' not in part.get('features', []): continue @@ -785,7 +786,7 @@ def Verity(options): '--hash-block-size', part['fs_block_size'], '--data-blocks', part['fs_blocks'], '--hash-offset', part['fs_bytes'], - loop_dev, loop_dev]) + loop_dev, loop_dev]).decode('utf8') print(verityout.strip()) m = re.search("Root hash:\s+([a-f0-9]{64})$", verityout, re.IGNORECASE|re.MULTILINE) if not m: @@ -848,7 +849,7 @@ def GetPartitionByLabel(partitions, label): Returns: An object for the selected partition """ - for partition in partitions.itervalues(): + for partition in partitions.values(): if partition['type'] == 'blank': continue elif partition['label'] == label: @@ -882,7 +883,7 @@ def GetBlockSize(options): """ config, partitions = LoadPartitionConfig(options) - print config['metadata']['block_size'] + print(config['metadata']['block_size']) def GetFilesystemBlockSize(options): @@ -897,7 +898,7 @@ def GetFilesystemBlockSize(options): """ config, partitions = LoadPartitionConfig(options) - print config['metadata']['fs_block_size'] + print(config['metadata']['fs_block_size']) def GetPartitionSize(options): @@ -911,7 +912,7 @@ def GetPartitionSize(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByNumber(partitions, options.partition_num) - print partition['bytes'] + print(partition['bytes']) def GetFilesystemSize(options): @@ -927,7 +928,7 @@ def GetFilesystemSize(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByNumber(partitions, options.partition_num) - print partition.get('fs_bytes', partition['bytes']) + print(partition.get('fs_bytes', partition['bytes'])) def GetLabel(options): @@ -941,7 +942,7 @@ def GetLabel(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByNumber(partitions, options.partition_num) - print partition.get('label', 'UNTITLED') + print(partition.get('label', 'UNTITLED')) def GetNum(options): @@ -955,7 +956,7 @@ def GetNum(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByLabel(partitions, options.label) - print partition.get('num', '-1') + print(partition.get('num', '-1')) def GetUuid(options): @@ -969,7 +970,7 @@ def GetUuid(options): partitions = GetPartitionTableFromConfig(options) partition = GetPartitionByLabel(partitions, options.label) - print partition.get('uuid', '') + print(partition.get('uuid', '')) def DoDebugOutput(options): @@ -983,7 +984,7 @@ def DoDebugOutput(options): """ partitions = GetPartitionTableFromConfig(options) - for num, partition in sorted(partitions.iteritems()): + for num, partition in sorted(partitions.items()): if partition['type'] != 'blank': if partition['bytes'] < 1024 * 1024: size = '%d bytes' % partition['bytes'] @@ -994,11 +995,11 @@ def DoDebugOutput(options): fs_size = '%d bytes' % partition['fs_bytes'] else: fs_size = '%d MB' % (partition['fs_bytes'] / 1024 / 1024) - print '%s: %s - %s/%s' % (num, partition['label'], fs_size, size) + print('%s: %s - %s/%s' % (num, partition['label'], fs_size, size)) else: - print '%s: %s - %s' % (num, partition['label'], size) + print('%s: %s - %s' % (num, partition['label'], size)) else: - print '%s: blank' % num + print('%s: blank' % num) def DoParseOnly(options): diff --git a/build_library/gen_tmpfiles.py b/build_library/gen_tmpfiles.py index c0133f6dbd..a4b4134ba1 100755 --- a/build_library/gen_tmpfiles.py +++ b/build_library/gen_tmpfiles.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 '''Scan an existing directory tree and record installed directories. During build a number of directories under /var are created in the state @@ -82,7 +82,7 @@ def main(): fd.write('\n'.join(config)+'\n') fd.close() else: - print '\n'.join(config) + print('\n'.join(config)) if __name__ == '__main__': main() diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 6db1a1bae3..3e84a36ae0 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -82,7 +82,7 @@ def _SplitAndStrip(data): Args: data: list of libraries from ldd output Returns: - list of libararies that we should copy + list of libraries that we should copy """ return_list = [] @@ -120,7 +120,7 @@ def DepsToCopy(ldd_files, allow_list): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout_data, stderr_data) = proc.communicate(input=None) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: logging.error('Command %s failed', cmd) logging.error('error code %s', e.returncode) logging.error('ouput %s', e.output) @@ -128,6 +128,8 @@ def DepsToCopy(ldd_files, allow_list): if not stdout_data: continue + stdout_data = stdout_data.decode('utf8') + stderr_data = stderr_data.decode('utf8') logging.debug('ldd for %s = stdout = %s stderr =%s', file_name, stdout_data, stderr_data) @@ -153,7 +155,7 @@ def CopyRequiredFiles(dest_files_root, allow_list): sys.exit(1) all_files = DYNAMIC_EXECUTABLES + STATIC_FILES - all_files = map(os.path.expanduser, all_files) + all_files = list(map(os.path.expanduser, all_files)) for file_name in all_files: if not os.path.isfile(file_name): @@ -161,7 +163,6 @@ def CopyRequiredFiles(dest_files_root, allow_list): sys.exit(1) logging.debug('Given files that need to be copied = %s' % '' .join(all_files)) - all_files for file_name in all_files: logging.debug('Copying file %s to %s', file_name, dest_files_root) try: @@ -174,14 +175,14 @@ def CopyRequiredFiles(dest_files_root, allow_list): lib_dir = os.path.join(dest_files_root, LIB_DIR) os.mkdir(lib_dir) for file_name in libraries: - logging.debug('Copying file %s to %s', file_name, lib_dir) + logging.debug('Copying file %s to %s', file_name, lib_dir) try: shutil.copy2(file_name, lib_dir) except EnvironmentError: logging.exception("Copying '%s' to %s failed", file_name, lib_dir) sys.exit(1) - for source_dir, target_dir in RECURSE_DIRS.iteritems(): + for source_dir, target_dir in RECURSE_DIRS.items(): logging.debug('Processing directory %s', source_dir) full_path = os.path.expanduser(source_dir) if not os.path.isdir(full_path): @@ -212,7 +213,7 @@ def WrapExecutableFiles(dest_files_root, ld_linux): local_exec_wrapped = local_exec + ".bin" shutil.move(local_exec, local_exec_wrapped) - fd = os.open(local_exec, os.O_WRONLY | os.O_CREAT, 0733) + fd = os.open(local_exec, os.O_WRONLY | os.O_CREAT, 0o733) with os.fdopen(fd, 'w') as script: script.write('#!/bin/sh\n') script.write('# Auto-generated wrapper script\n') @@ -249,7 +250,7 @@ def GenerateZipFile(base_name, root_dir): try: subprocess.Popen(['zip', '-r', '-9', base_name, '.'], stdout=subprocess.PIPE).communicate()[0] - except OSError, e: + except OSError as e: logging.error('Execution failed:%s', e.strerror) return False finally: diff --git a/build_library/generate_grub_hashes.py b/build_library/generate_grub_hashes.py index 419e62e401..bd9ab9e838 100755 --- a/build_library/generate_grub_hashes.py +++ b/build_library/generate_grub_hashes.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import hashlib import json @@ -7,11 +7,11 @@ import string import subprocess import sys -filename = sys.argv[1] +filename = sys.argv[1] grubdir = sys.argv[2] outputdir = sys.argv[3] version = sys.argv[4] -bootoffset = string.atoi(subprocess.check_output(['cgpt', 'show', '-i', '2', '-b', filename])) * 512 +bootoffset = int(subprocess.check_output(['cgpt', 'show', '-i', '2', '-b', filename])) * 512 with open(filename, "rb") as f: boot = f.read(440) f.seek(bootoffset) diff --git a/build_library/generate_kernel_hash.sh b/build_library/generate_kernel_hash.py similarity index 61% rename from build_library/generate_kernel_hash.sh rename to build_library/generate_kernel_hash.py index 429fe9cb7d..6e7549fe93 100755 --- a/build_library/generate_kernel_hash.sh +++ b/build_library/generate_kernel_hash.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import hashlib import json @@ -10,4 +10,4 @@ version=sys.argv[2] with open(path, "rb") as f: kernel = f.read() - print json.dumps({"9": {"binaryvalues": [{"prefix": "grub_linux", "values": [{"value": hashlib.sha1(kernel).hexdigest(), "description": "flatcar-%s" % version}]}]}}) + print(json.dumps({"9": {"binaryvalues": [{"prefix": "grub_linux", "values": [{"value": hashlib.sha1(kernel).hexdigest(), "description": "flatcar-%s" % version}]}]}})) diff --git a/check_out_of_date.py b/check_out_of_date.py deleted file mode 100755 index 91785525ba..0000000000 --- a/check_out_of_date.py +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/python2 -# TODO: This script needs porting to python3, since portage is also in python3 now. - -# Prints out a list of all packages in portage-stable and how they stand relative to gentoo upstream - -import argparse -import json -import os -import subprocess -import sys - -import portage.versions - - -def split_package(p): - # split into cat/package,ver-rev - split = portage.versions.catpkgsplit(p.strip()) - return (split[0] + "/" + split[1], split[2] + "-" + split[3]) - - -def build_pkg_map(pkgs): - pkgs = map(split_package, pkgs) - package_map = dict() - for pkg, ver in pkgs: - if pkg not in package_map: - package_map[pkg] = [ver] - else: - package_map[pkg].append(ver) - return package_map - - -def exec_command_strict(cmd): - """ Wraps check_output splitting the input and string'ing the output""" - return bytes.decode(subprocess.check_output(cmd.split())) - - -def exec_command(cmd): - """ Like exec_command_strict but returns the output even if the command exited unsuccessfully""" - try: - return exec_command_strict(cmd) - except subprocess.CalledProcessError as e: - return bytes.decode(e.output) - - -def get_portage_tree_packages(tree_path): - """ returns a list of all packages in a portage tree/overlay in the form of cat/pkg-ver""" - pkgs = exec_command_strict("find -L {} -maxdepth 3 -type f -name *.ebuild -not -name skel.ebuild -printf %P\\n".format(tree_path)) - - def process_line(line): - # cat/pkg/pkg-ver.ebuild -> cat/pkg-ver - chunks = line.split("/") - end = chunks[2].replace(".ebuild", "") - return chunks[0] + "/" + end - return build_pkg_map(map(process_line, pkgs.splitlines())) - - -def process_emerge_output(eout): - """ transform from emerge --unordered-dispaly to cat/pkg-ver""" - def process_line(line): - return line.strip().split("] ")[1].split(":")[0] - - def is_package(line): - # none of the header line have a / - return "/" in line - - return map(process_line, filter(is_package, eout.splitlines())) - - -def get_board_packages(board): - """ gets a list of packages used by a board. valid boards are {arm,amd}64-usr, sdk, and bootstrap""" - emerge_args = "--emptytree --pretend --verbose --unordered-display" - if board == "sdk": - cmd = "emerge {} @system sdk-depends sdk-extras".format(emerge_args) - elif board == "amd64-usr" or board == "arm64-usr": - cmd = "emerge-{} {} @system board-packages".format(board, emerge_args) - elif board == "bootstrap": - pkgs = exec_command_strict("/usr/lib64/catalyst/targets/stage1/build.py") - cmd = "emerge {} {}".format(emerge_args, pkgs) - elif board == "image": - cmd = "emerge-amd64-usr {} --usepkgonly board-packages".format(emerge_args) - else: - raise "invalid board" - return build_pkg_map(process_emerge_output(exec_command(cmd))) - - -def print_table(report, head, line_head, line_tail, tail, joiner, pkg_joiner): - print(head) - # metapackage that acts as the header - report.insert(0, {"name": "Package", - "common": ["Common"], - "ours": ["Ours"], - "upstream": ["Upstream"], - "tag": "Tag", - "sdk": ["sdk"], - "arm64-usr": ["arm64-usr"], - "amd64-usr": ["amd64-usr"], - "bootstrap": ["bootstrap"], - "modified": "Modified"}) - for entry in report: - print(line_head + joiner.join([entry.get("name",""), - pkg_joiner.join(entry.get("common",[])), - pkg_joiner.join(entry.get("ours",[])), - pkg_joiner.join(entry.get("upstream",[])), - entry.get("tag",""), - pkg_joiner.join(entry.get("sdk", [])), - pkg_joiner.join(entry.get("arm64-usr", [])), - pkg_joiner.join(entry.get("amd64-usr", [])), - pkg_joiner.join(entry.get("bootstrap", [])), - entry.get("modified","")]) + line_tail) - print(tail) - - -def print_table_human(report): - print_table(report, "", "", "", "", "\t", " ") - - -def print_html_table(report): - print_table(report, "
", " |