Merge pull request #142 from kinvolk/krnowak/python3-porting

Port some stuff to python3
This commit is contained in:
Krzesimir Nowak 2021-08-24 16:10:08 +02:00 committed by GitHub
commit 344afd98f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 66 additions and 252 deletions

View File

@ -693,7 +693,7 @@ EOF
if [[ -n "${pcr_policy}" ]]; then if [[ -n "${pcr_policy}" ]]; then
mkdir -p "${BUILD_DIR}/pcrs" mkdir -p "${BUILD_DIR}/pcrs"
${BUILD_LIBRARY_DIR}/generate_kernel_hash.sh \ ${BUILD_LIBRARY_DIR}/generate_kernel_hash.py \
"${root_fs_dir}/boot/flatcar/vmlinuz-a" ${FLATCAR_VERSION} \ "${root_fs_dir}/boot/flatcar/vmlinuz-a" ${FLATCAR_VERSION} \
>"${BUILD_DIR}/pcrs/kernel.config" >"${BUILD_DIR}/pcrs/kernel.config"
fi fi

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/python3
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be # Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file. # found in the LICENSE file.
@ -66,9 +66,9 @@ def LoadPartitionConfig(options):
raise InvalidLayout('Metadata is missing required entries: %s' % e) raise InvalidLayout('Metadata is missing required entries: %s' % e)
def VerifyLayout(layout_name, layout, base=None): def VerifyLayout(layout_name, layout, base=None):
for part_num, part in layout.iteritems(): for part_num, part in layout.items():
part['num'] = int(part_num) part['num'] = int(part_num)
part_keys = set(part.iterkeys()) part_keys = set(part.keys())
unknown_keys = part_keys - valid_layout_keys unknown_keys = part_keys - valid_layout_keys
if unknown_keys: if unknown_keys:
raise InvalidLayout('Unknown items in partition %s %s: %r' % raise InvalidLayout('Unknown items in partition %s %s: %r' %
@ -79,7 +79,7 @@ def LoadPartitionConfig(options):
if base: if base:
part_base = base.get(part_num, {}) part_base = base.get(part_num, {})
part_keys.update(part_base.iterkeys()) part_keys.update(iter(part_base.keys()))
if part.get('type', None) == 'blank': if part.get('type', None) == 'blank':
continue continue
@ -122,12 +122,12 @@ def LoadPartitionConfig(options):
# Fill in default values from base, # Fill in default values from base,
# dict doesn't have a update()+setdefault() method so this looks tedious # dict doesn't have a update()+setdefault() method so this looks tedious
if base: if base:
for part_num, base_part in base.iteritems(): for part_num, base_part in base.items():
part = layout.setdefault(part_num, {}) part = layout.setdefault(part_num, {})
for base_key, base_value in base_part.iteritems(): for base_key, base_value in base_part.items():
part.setdefault(base_key, base_value) part.setdefault(base_key, base_value)
for part_num, part in sorted(layout.iteritems(), key=lambda t: int(t[0])): for part_num, part in sorted(iter(layout.items()), key=lambda t: int(t[0])):
if part['type'] == 'blank': if part['type'] == 'blank':
continue continue
@ -163,7 +163,7 @@ def LoadPartitionConfig(options):
# Verify 'base' before other layouts because it is inherited by the others # Verify 'base' before other layouts because it is inherited by the others
# Fill in extra/default values in base last so they aren't inherited # Fill in extra/default values in base last so they aren't inherited
VerifyLayout('base', base) VerifyLayout('base', base)
for layout_name, layout in config['layouts'].iteritems(): for layout_name, layout in config['layouts'].items():
if layout_name == 'base': if layout_name == 'base':
continue continue
VerifyLayout(layout_name, layout, base) VerifyLayout(layout_name, layout, base)
@ -198,7 +198,7 @@ def GetPartitionTableFromImage(options, config, partitions):
""" """
block_size = config['metadata']['block_size'] block_size = config['metadata']['block_size']
cgpt_show = subprocess.check_output( cgpt_show = subprocess.check_output(
['cgpt', 'show', '-q', options.disk_image]) ['cgpt', 'show', '-q', options.disk_image]).decode('utf8')
for line in cgpt_show.split('\n'): for line in cgpt_show.split('\n'):
if not line.strip(): if not line.strip():
continue continue
@ -225,7 +225,7 @@ def GetPartitionTableFromImage(options, config, partitions):
else: else:
part['image_compat'] = False part['image_compat'] = False
for part in partitions.itervalues(): for part in partitions.values():
if part.get('type', 'blank') == 'blank': if part.get('type', 'blank') == 'blank':
continue continue
if not part.get('image_exists', False): if not part.get('image_exists', False):
@ -235,12 +235,12 @@ def GetPartitionTableFromImage(options, config, partitions):
with PartitionLoop(options, part) as loop_dev: with PartitionLoop(options, part) as loop_dev:
try: try:
part['image_fs_type'] = subprocess.check_output( part['image_fs_type'] = subprocess.check_output(
['sudo', 'blkid', '-o', 'value', '-s', 'TYPE', loop_dev]).strip() ['sudo', 'blkid', '-o', 'value', '-s', 'TYPE', loop_dev]).strip().decode('utf8')
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
part['image_fs_type'] = None part['image_fs_type'] = None
# Set compat flags for any partition not in the image # Set compat flags for any partition not in the image
for part in partitions.itervalues(): for part in partitions.values():
part.setdefault('image_exists', False) part.setdefault('image_exists', False)
if part.get('type', 'blank') == 'blank': if part.get('type', 'blank') == 'blank':
part.setdefault('image_compat', True) part.setdefault('image_compat', True)
@ -269,7 +269,7 @@ def WritePartitionTable(options, config=None, partitions=None):
else: else:
# If we are not creating a fresh image all partitions must be compatible. # If we are not creating a fresh image all partitions must be compatible.
GetPartitionTableFromImage(options, config, partitions) GetPartitionTableFromImage(options, config, partitions)
if not all(p['image_compat'] for p in partitions.itervalues()): if not all(p['image_compat'] for p in partitions.values()):
raise InvalidLayout("New disk layout is incompatible existing image") raise InvalidLayout("New disk layout is incompatible existing image")
# Extend the disk image size as needed # Extend the disk image size as needed
@ -279,7 +279,7 @@ def WritePartitionTable(options, config=None, partitions=None):
hybrid = None hybrid = None
prioritize = [] prioritize = []
for partition in partitions.itervalues(): for partition in partitions.values():
if partition['type'] != 'blank': if partition['type'] != 'blank':
Cgpt('add', '-i', partition['num'], Cgpt('add', '-i', partition['num'],
'-b', partition['first_block'], '-b', partition['first_block'],
@ -452,6 +452,7 @@ def PartitionLoop(options, partition):
'--sizelimit', str(partition['bytes']), '--sizelimit', str(partition['bytes']),
'--find', '--show', options.disk_image]) '--find', '--show', options.disk_image])
loop_dev = loop_dev.strip() loop_dev = loop_dev.strip()
loop_dev = loop_dev.decode('utf8')
err = None err = None
break break
except subprocess.CalledProcessError as error: except subprocess.CalledProcessError as error:
@ -469,8 +470,8 @@ def PartitionLoop(options, partition):
def FormatPartition(options, part): def FormatPartition(options, part):
print "Formatting partition %s (%s) as %s" % ( print("Formatting partition %s (%s) as %s" % (
part['num'], part['label'], part['fs_type']) part['num'], part['label'], part['fs_type']))
with PartitionLoop(options, part) as loop_dev: with PartitionLoop(options, part) as loop_dev:
if part['fs_type'] in ('ext2', 'ext4'): if part['fs_type'] in ('ext2', 'ext4'):
@ -497,7 +498,7 @@ def Format(options):
config, partitions = LoadPartitionConfig(options) config, partitions = LoadPartitionConfig(options)
WritePartitionTable(options, config, partitions) WritePartitionTable(options, config, partitions)
for part in partitions.itervalues(): for part in partitions.values():
if part['type'] == 'blank' or 'fs_type' not in part: if part['type'] == 'blank' or 'fs_type' not in part:
continue continue
@ -542,13 +543,13 @@ def Update(options):
config, partitions = LoadPartitionConfig(options) config, partitions = LoadPartitionConfig(options)
WritePartitionTable(options, config, partitions) WritePartitionTable(options, config, partitions)
for part in partitions.itervalues(): for part in partitions.values():
if not part.get('fs_type', None): if not part.get('fs_type', None):
continue continue
elif not part['image_fs_type']: elif not part['image_fs_type']:
FormatPartition(options, part) FormatPartition(options, part)
for part in partitions.itervalues(): for part in partitions.values():
resize_func = None resize_func = None
if not part.get('fs_type', None): if not part.get('fs_type', None):
continue continue
@ -561,8 +562,8 @@ def Update(options):
else: else:
continue continue
print "Resizing partition %s (%s) to %s bytes" % ( print("Resizing partition %s (%s) to %s bytes" % (
part['num'], part['label'], part['fs_bytes']) part['num'], part['label'], part['fs_bytes']))
with PartitionLoop(options, part) as loop_dev: with PartitionLoop(options, part) as loop_dev:
resize_func(part, loop_dev) resize_func(part, loop_dev)
@ -582,7 +583,7 @@ def Mount(options):
GetPartitionTableFromImage(options, config, partitions) GetPartitionTableFromImage(options, config, partitions)
mounts = {} mounts = {}
for part_num, part in partitions.iteritems(): for part_num, part in partitions.items():
path = part.get('mount', None) path = part.get('mount', None)
if not path or not path.startswith('/'): if not path or not path.startswith('/'):
continue continue
@ -625,7 +626,7 @@ def Mount(options):
if err is not None: if err is not None:
raise err raise err
for src, dst in mount.get('binds', {}).iteritems(): for src, dst in mount.get('binds', {}).items():
# src may be relative or absolute, os.path.join handles this. # src may be relative or absolute, os.path.join handles this.
full_src = os.path.realpath( full_src = os.path.realpath(
options.mount_dir + os.path.join(mount['mount'], src)) options.mount_dir + os.path.join(mount['mount'], src))
@ -650,11 +651,11 @@ def ReadWriteSubvol(options, partition, disable_rw):
""" """
if disable_rw: if disable_rw:
print "Disabling read-write on default subvolume of partition %s (%s)" % ( print("Disabling read-write on default subvolume of partition %s (%s)" % (
partition['num'], partition['label']) partition['num'], partition['label']))
else: else:
print "Enabling read-write on default subvolume of partition %s (%s)" % ( print("Enabling read-write on default subvolume of partition %s (%s)" % (
partition['num'], partition['label']) partition['num'], partition['label']))
with PartitionLoop(options, partition) as loop_dev: with PartitionLoop(options, partition) as loop_dev:
btrfs_mount = tempfile.mkdtemp() btrfs_mount = tempfile.mkdtemp()
@ -701,11 +702,11 @@ def Tune2fsReadWrite(options, partition, disable_rw):
""" """
if disable_rw: if disable_rw:
print "Disabling read-write mounting of partition %s (%s)" % ( print("Disabling read-write mounting of partition %s (%s)" % (
partition['num'], partition['label']) partition['num'], partition['label']))
else: else:
print "Enabling read-write mounting of partition %s (%s)" % ( print("Enabling read-write mounting of partition %s (%s)" % (
partition['num'], partition['label']) partition['num'], partition['label']))
# offset of ro_compat, highest order byte (le 32 bit field) # offset of ro_compat, highest order byte (le 32 bit field)
flag_offset = 0x464 + 3 flag_offset = 0x464 + 3
@ -767,7 +768,7 @@ def Verity(options):
config, partitions = LoadPartitionConfig(options) config, partitions = LoadPartitionConfig(options)
GetPartitionTableFromImage(options, config, partitions) GetPartitionTableFromImage(options, config, partitions)
for part_num, part in partitions.iteritems(): for part_num, part in partitions.items():
if 'verity' not in part.get('features', []): if 'verity' not in part.get('features', []):
continue continue
@ -785,7 +786,7 @@ def Verity(options):
'--hash-block-size', part['fs_block_size'], '--hash-block-size', part['fs_block_size'],
'--data-blocks', part['fs_blocks'], '--data-blocks', part['fs_blocks'],
'--hash-offset', part['fs_bytes'], '--hash-offset', part['fs_bytes'],
loop_dev, loop_dev]) loop_dev, loop_dev]).decode('utf8')
print(verityout.strip()) print(verityout.strip())
m = re.search("Root hash:\s+([a-f0-9]{64})$", verityout, re.IGNORECASE|re.MULTILINE) m = re.search("Root hash:\s+([a-f0-9]{64})$", verityout, re.IGNORECASE|re.MULTILINE)
if not m: if not m:
@ -848,7 +849,7 @@ def GetPartitionByLabel(partitions, label):
Returns: Returns:
An object for the selected partition An object for the selected partition
""" """
for partition in partitions.itervalues(): for partition in partitions.values():
if partition['type'] == 'blank': if partition['type'] == 'blank':
continue continue
elif partition['label'] == label: elif partition['label'] == label:
@ -882,7 +883,7 @@ def GetBlockSize(options):
""" """
config, partitions = LoadPartitionConfig(options) config, partitions = LoadPartitionConfig(options)
print config['metadata']['block_size'] print(config['metadata']['block_size'])
def GetFilesystemBlockSize(options): def GetFilesystemBlockSize(options):
@ -897,7 +898,7 @@ def GetFilesystemBlockSize(options):
""" """
config, partitions = LoadPartitionConfig(options) config, partitions = LoadPartitionConfig(options)
print config['metadata']['fs_block_size'] print(config['metadata']['fs_block_size'])
def GetPartitionSize(options): def GetPartitionSize(options):
@ -911,7 +912,7 @@ def GetPartitionSize(options):
partitions = GetPartitionTableFromConfig(options) partitions = GetPartitionTableFromConfig(options)
partition = GetPartitionByNumber(partitions, options.partition_num) partition = GetPartitionByNumber(partitions, options.partition_num)
print partition['bytes'] print(partition['bytes'])
def GetFilesystemSize(options): def GetFilesystemSize(options):
@ -927,7 +928,7 @@ def GetFilesystemSize(options):
partitions = GetPartitionTableFromConfig(options) partitions = GetPartitionTableFromConfig(options)
partition = GetPartitionByNumber(partitions, options.partition_num) partition = GetPartitionByNumber(partitions, options.partition_num)
print partition.get('fs_bytes', partition['bytes']) print(partition.get('fs_bytes', partition['bytes']))
def GetLabel(options): def GetLabel(options):
@ -941,7 +942,7 @@ def GetLabel(options):
partitions = GetPartitionTableFromConfig(options) partitions = GetPartitionTableFromConfig(options)
partition = GetPartitionByNumber(partitions, options.partition_num) partition = GetPartitionByNumber(partitions, options.partition_num)
print partition.get('label', 'UNTITLED') print(partition.get('label', 'UNTITLED'))
def GetNum(options): def GetNum(options):
@ -955,7 +956,7 @@ def GetNum(options):
partitions = GetPartitionTableFromConfig(options) partitions = GetPartitionTableFromConfig(options)
partition = GetPartitionByLabel(partitions, options.label) partition = GetPartitionByLabel(partitions, options.label)
print partition.get('num', '-1') print(partition.get('num', '-1'))
def GetUuid(options): def GetUuid(options):
@ -969,7 +970,7 @@ def GetUuid(options):
partitions = GetPartitionTableFromConfig(options) partitions = GetPartitionTableFromConfig(options)
partition = GetPartitionByLabel(partitions, options.label) partition = GetPartitionByLabel(partitions, options.label)
print partition.get('uuid', '') print(partition.get('uuid', ''))
def DoDebugOutput(options): def DoDebugOutput(options):
@ -983,7 +984,7 @@ def DoDebugOutput(options):
""" """
partitions = GetPartitionTableFromConfig(options) partitions = GetPartitionTableFromConfig(options)
for num, partition in sorted(partitions.iteritems()): for num, partition in sorted(partitions.items()):
if partition['type'] != 'blank': if partition['type'] != 'blank':
if partition['bytes'] < 1024 * 1024: if partition['bytes'] < 1024 * 1024:
size = '%d bytes' % partition['bytes'] size = '%d bytes' % partition['bytes']
@ -994,11 +995,11 @@ def DoDebugOutput(options):
fs_size = '%d bytes' % partition['fs_bytes'] fs_size = '%d bytes' % partition['fs_bytes']
else: else:
fs_size = '%d MB' % (partition['fs_bytes'] / 1024 / 1024) fs_size = '%d MB' % (partition['fs_bytes'] / 1024 / 1024)
print '%s: %s - %s/%s' % (num, partition['label'], fs_size, size) print('%s: %s - %s/%s' % (num, partition['label'], fs_size, size))
else: else:
print '%s: %s - %s' % (num, partition['label'], size) print('%s: %s - %s' % (num, partition['label'], size))
else: else:
print '%s: blank' % num print('%s: blank' % num)
def DoParseOnly(options): def DoParseOnly(options):

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/python3
'''Scan an existing directory tree and record installed directories. '''Scan an existing directory tree and record installed directories.
During build a number of directories under /var are created in the state During build a number of directories under /var are created in the state
@ -82,7 +82,7 @@ def main():
fd.write('\n'.join(config)+'\n') fd.write('\n'.join(config)+'\n')
fd.close() fd.close()
else: else:
print '\n'.join(config) print('\n'.join(config))
if __name__ == '__main__': if __name__ == '__main__':
main() main()

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/python3
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be # Use of this source code is governed by a BSD-style license that can be
@ -82,7 +82,7 @@ def _SplitAndStrip(data):
Args: Args:
data: list of libraries from ldd output data: list of libraries from ldd output
Returns: Returns:
list of libararies that we should copy list of libraries that we should copy
""" """
return_list = [] return_list = []
@ -120,7 +120,7 @@ def DepsToCopy(ldd_files, allow_list):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE)
(stdout_data, stderr_data) = proc.communicate(input=None) (stdout_data, stderr_data) = proc.communicate(input=None)
except subprocess.CalledProcessError, e: except subprocess.CalledProcessError as e:
logging.error('Command %s failed', cmd) logging.error('Command %s failed', cmd)
logging.error('error code %s', e.returncode) logging.error('error code %s', e.returncode)
logging.error('ouput %s', e.output) logging.error('ouput %s', e.output)
@ -128,6 +128,8 @@ def DepsToCopy(ldd_files, allow_list):
if not stdout_data: continue if not stdout_data: continue
stdout_data = stdout_data.decode('utf8')
stderr_data = stderr_data.decode('utf8')
logging.debug('ldd for %s = stdout = %s stderr =%s', file_name, logging.debug('ldd for %s = stdout = %s stderr =%s', file_name,
stdout_data, stderr_data) stdout_data, stderr_data)
@ -153,7 +155,7 @@ def CopyRequiredFiles(dest_files_root, allow_list):
sys.exit(1) sys.exit(1)
all_files = DYNAMIC_EXECUTABLES + STATIC_FILES all_files = DYNAMIC_EXECUTABLES + STATIC_FILES
all_files = map(os.path.expanduser, all_files) all_files = list(map(os.path.expanduser, all_files))
for file_name in all_files: for file_name in all_files:
if not os.path.isfile(file_name): if not os.path.isfile(file_name):
@ -161,7 +163,6 @@ def CopyRequiredFiles(dest_files_root, allow_list):
sys.exit(1) sys.exit(1)
logging.debug('Given files that need to be copied = %s' % '' .join(all_files)) logging.debug('Given files that need to be copied = %s' % '' .join(all_files))
all_files
for file_name in all_files: for file_name in all_files:
logging.debug('Copying file %s to %s', file_name, dest_files_root) logging.debug('Copying file %s to %s', file_name, dest_files_root)
try: try:
@ -181,7 +182,7 @@ def CopyRequiredFiles(dest_files_root, allow_list):
logging.exception("Copying '%s' to %s failed", file_name, lib_dir) logging.exception("Copying '%s' to %s failed", file_name, lib_dir)
sys.exit(1) sys.exit(1)
for source_dir, target_dir in RECURSE_DIRS.iteritems(): for source_dir, target_dir in RECURSE_DIRS.items():
logging.debug('Processing directory %s', source_dir) logging.debug('Processing directory %s', source_dir)
full_path = os.path.expanduser(source_dir) full_path = os.path.expanduser(source_dir)
if not os.path.isdir(full_path): if not os.path.isdir(full_path):
@ -212,7 +213,7 @@ def WrapExecutableFiles(dest_files_root, ld_linux):
local_exec_wrapped = local_exec + ".bin" local_exec_wrapped = local_exec + ".bin"
shutil.move(local_exec, local_exec_wrapped) shutil.move(local_exec, local_exec_wrapped)
fd = os.open(local_exec, os.O_WRONLY | os.O_CREAT, 0733) fd = os.open(local_exec, os.O_WRONLY | os.O_CREAT, 0o733)
with os.fdopen(fd, 'w') as script: with os.fdopen(fd, 'w') as script:
script.write('#!/bin/sh\n') script.write('#!/bin/sh\n')
script.write('# Auto-generated wrapper script\n') script.write('# Auto-generated wrapper script\n')
@ -249,7 +250,7 @@ def GenerateZipFile(base_name, root_dir):
try: try:
subprocess.Popen(['zip', '-r', '-9', base_name, '.'], subprocess.Popen(['zip', '-r', '-9', base_name, '.'],
stdout=subprocess.PIPE).communicate()[0] stdout=subprocess.PIPE).communicate()[0]
except OSError, e: except OSError as e:
logging.error('Execution failed:%s', e.strerror) logging.error('Execution failed:%s', e.strerror)
return False return False
finally: finally:

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/python3
import hashlib import hashlib
import json import json
@ -11,7 +11,7 @@ filename = sys.argv[1]
grubdir = sys.argv[2] grubdir = sys.argv[2]
outputdir = sys.argv[3] outputdir = sys.argv[3]
version = sys.argv[4] version = sys.argv[4]
bootoffset = string.atoi(subprocess.check_output(['cgpt', 'show', '-i', '2', '-b', filename])) * 512 bootoffset = int(subprocess.check_output(['cgpt', 'show', '-i', '2', '-b', filename])) * 512
with open(filename, "rb") as f: with open(filename, "rb") as f:
boot = f.read(440) boot = f.read(440)
f.seek(bootoffset) f.seek(bootoffset)

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/python3
import hashlib import hashlib
import json import json
@ -10,4 +10,4 @@ version=sys.argv[2]
with open(path, "rb") as f: with open(path, "rb") as f:
kernel = f.read() kernel = f.read()
print json.dumps({"9": {"binaryvalues": [{"prefix": "grub_linux", "values": [{"value": hashlib.sha1(kernel).hexdigest(), "description": "flatcar-%s" % version}]}]}}) print(json.dumps({"9": {"binaryvalues": [{"prefix": "grub_linux", "values": [{"value": hashlib.sha1(kernel).hexdigest(), "description": "flatcar-%s" % version}]}]}}))

View File

@ -1,188 +0,0 @@
#!/usr/bin/python2
# TODO: This script needs porting to python3, since portage is also in python3 now.
# Prints out a list of all packages in portage-stable and how they stand relative to gentoo upstream
import argparse
import json
import os
import subprocess
import sys
import portage.versions
def split_package(p):
# split into cat/package,ver-rev
split = portage.versions.catpkgsplit(p.strip())
return (split[0] + "/" + split[1], split[2] + "-" + split[3])
def build_pkg_map(pkgs):
pkgs = map(split_package, pkgs)
package_map = dict()
for pkg, ver in pkgs:
if pkg not in package_map:
package_map[pkg] = [ver]
else:
package_map[pkg].append(ver)
return package_map
def exec_command_strict(cmd):
""" Wraps check_output splitting the input and string'ing the output"""
return bytes.decode(subprocess.check_output(cmd.split()))
def exec_command(cmd):
""" Like exec_command_strict but returns the output even if the command exited unsuccessfully"""
try:
return exec_command_strict(cmd)
except subprocess.CalledProcessError as e:
return bytes.decode(e.output)
def get_portage_tree_packages(tree_path):
""" returns a list of all packages in a portage tree/overlay in the form of cat/pkg-ver"""
pkgs = exec_command_strict("find -L {} -maxdepth 3 -type f -name *.ebuild -not -name skel.ebuild -printf %P\\n".format(tree_path))
def process_line(line):
# cat/pkg/pkg-ver.ebuild -> cat/pkg-ver
chunks = line.split("/")
end = chunks[2].replace(".ebuild", "")
return chunks[0] + "/" + end
return build_pkg_map(map(process_line, pkgs.splitlines()))
def process_emerge_output(eout):
""" transform from emerge --unordered-dispaly to cat/pkg-ver"""
def process_line(line):
return line.strip().split("] ")[1].split(":")[0]
def is_package(line):
# none of the header line have a /
return "/" in line
return map(process_line, filter(is_package, eout.splitlines()))
def get_board_packages(board):
""" gets a list of packages used by a board. valid boards are {arm,amd}64-usr, sdk, and bootstrap"""
emerge_args = "--emptytree --pretend --verbose --unordered-display"
if board == "sdk":
cmd = "emerge {} @system sdk-depends sdk-extras".format(emerge_args)
elif board == "amd64-usr" or board == "arm64-usr":
cmd = "emerge-{} {} @system board-packages".format(board, emerge_args)
elif board == "bootstrap":
pkgs = exec_command_strict("/usr/lib64/catalyst/targets/stage1/build.py")
cmd = "emerge {} {}".format(emerge_args, pkgs)
elif board == "image":
cmd = "emerge-amd64-usr {} --usepkgonly board-packages".format(emerge_args)
else:
raise "invalid board"
return build_pkg_map(process_emerge_output(exec_command(cmd)))
def print_table(report, head, line_head, line_tail, tail, joiner, pkg_joiner):
print(head)
# metapackage that acts as the header
report.insert(0, {"name": "Package",
"common": ["Common"],
"ours": ["Ours"],
"upstream": ["Upstream"],
"tag": "Tag",
"sdk": ["sdk"],
"arm64-usr": ["arm64-usr"],
"amd64-usr": ["amd64-usr"],
"bootstrap": ["bootstrap"],
"modified": "Modified"})
for entry in report:
print(line_head + joiner.join([entry.get("name",""),
pkg_joiner.join(entry.get("common",[])),
pkg_joiner.join(entry.get("ours",[])),
pkg_joiner.join(entry.get("upstream",[])),
entry.get("tag",""),
pkg_joiner.join(entry.get("sdk", [])),
pkg_joiner.join(entry.get("arm64-usr", [])),
pkg_joiner.join(entry.get("amd64-usr", [])),
pkg_joiner.join(entry.get("bootstrap", [])),
entry.get("modified","")]) + line_tail)
print(tail)
def print_table_human(report):
print_table(report, "", "", "", "", "\t", " ")
def print_html_table(report):
print_table(report, "<html><body><table border=1>", "<tr><td>", "</td></tr>", "</table></body></html>", "</td><td>", "<br>")
def get_date(pkg, repo_root, fmt):
return exec_command_strict("git -C {} --no-pager log -1 --pretty=%ad --date={} {}".format(repo_root, fmt, pkg)).strip()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--update-upstream", help="run git-pull in the gentoo mirror repo first", action="store_true")
parser.add_argument("--upstream-git", help="git uri to clone for upstream", default="https://github.com/gentoo/gentoo.git")
parser.add_argument("--upstream-path", help="path to gentoo tree", default="/mnt/host/source/src/gentoo-portage")
parser.add_argument("--portage-stable-path", help="path to portage-stable", default="/mnt/host/source/src/third_party/portage-stable")
parser.add_argument("--date-fmt", help="format for git-date to use", default="relative")
parser.add_argument("--output", help="output format, json, table, and html are accepted", default="json")
args = parser.parse_args()
if not os.path.exists(args.upstream_path):
os.makedirs(args.upstream_path)
subprocess.check_call(["git", "clone", args.upstream_git, args.upstream_path])
elif args.update_upstream:
# elif to not pull if we just cloned
subprocess.check_call(["git", "-C", args.upstream_path, "pull"])
pkg_lists = {}
sources = ["sdk", "bootstrap", "amd64-usr", "arm64-usr", "image"]
for i in sources:
pkg_lists[i] = get_board_packages(i)
gentoo_packages = get_portage_tree_packages(args.upstream_path)
packages = get_portage_tree_packages(args.portage_stable_path)
# time to make the report
report = []
for pkg, vers in packages.iteritems():
upstream = gentoo_packages.get(pkg, [])
entry = {
"name": pkg,
"common": list(set(vers).intersection(upstream)),
"ours": list(set(vers).difference(upstream)),
"upstream": list(set(upstream).difference(vers)),
"modified": get_date(pkg, args.portage_stable_path, args.date_fmt)
}
if not entry["upstream"]:
entry["tag"] = "updated"
elif entry["common"]:
entry["tag"] = "has_update"
elif pkg in gentoo_packages:
entry["tag"] = "no_ebuild_upstream"
else:
entry["tag"] = "deleted_upstream"
for src in sources:
if pkg in pkg_lists[src]:
entry[src] = pkg_lists[src][pkg]
report.append(entry)
if args.output == "json":
print(json.dumps(report))
elif args.output == "table":
print_table_human(report)
elif args.output == "html":
print_html_table(report)
else:
print("Unknown output type. Dying.")
sys.exit(2)
if __name__ == "__main__":
main()

View File

@ -843,8 +843,8 @@ reinterpret_path_for_chroot() {
# Get the relative path between two locations. Handy for printing paths to # Get the relative path between two locations. Handy for printing paths to
# the user that will usually make sense both inside and outside the chroot. # the user that will usually make sense both inside and outside the chroot.
relpath() { relpath() {
local py='import sys, os; print os.path.relpath(sys.argv[1], sys.argv[2])' local py='import sys, os; print(os.path.relpath(sys.argv[1], sys.argv[2]))'
python2 -c "${py}" "${1}" "${2:-.}" python3 -c "${py}" "${1}" "${2:-.}"
} }
enable_strict_sudo() { enable_strict_sudo() {