From c887cfdce6ac94d33f2ed6ce3750db1ebce37f26 Mon Sep 17 00:00:00 2001 From: David Michael Date: Fri, 28 Sep 2018 13:53:13 +0000 Subject: [PATCH 01/21] bootstrap_sdk: Explicitly set the Python version to 2.7 It has recently started picking Python 3 for /usr/bin/python. --- build_library/catalyst_default_stage4.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_library/catalyst_default_stage4.sh b/build_library/catalyst_default_stage4.sh index e2b540a6d2..a1a3be9f13 100644 --- a/build_library/catalyst_default_stage4.sh +++ b/build_library/catalyst_default_stage4.sh @@ -4,3 +4,6 @@ source /tmp/chroot-functions.sh echo "Double checking everything is fresh and happy." run_merge -uDN --with-bdeps=y world + +echo "Setting the default Python interpreter to Python 2." +eselect python set python2.7 From 729575c22ee828f38eea7586933e6e4187dcd8a2 Mon Sep 17 00:00:00 2001 From: David Michael Date: Fri, 12 Oct 2018 17:58:09 +0000 Subject: [PATCH 02/21] update_chroot: Allow upgrading glibc and binutils --- build_library/toolchain_util.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 53f829a4b4..26cf2a6bd5 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -190,8 +190,8 @@ get_cross_pkgs() { # Get portage arguments restricting toolchains to binary packages only. get_binonly_args() { local pkgs=( "${TOOLCHAIN_PKGS[@]}" $(get_cross_pkgs "$@") ) - # XXX: Drop this exception after stable > 1729. - local tmppkgs=( ${pkgs[*]##*/glibc} ) + # XXX: Drop this exception after stable > 1925. + local tmppkgs=( ${pkgs[*]##*/glibc} ) ; tmppkgs=( ${tmppkgs[*]##*/binutils} ) echo "${tmppkgs[@]/#/--useoldpkg-atoms=}" "${pkgs[@]/#/--rebuild-exclude=}" } From 1d19e63d4373e37d2e5fe0d7c64319a6215e4d18 Mon Sep 17 00:00:00 2001 From: David Michael Date: Fri, 12 Oct 2018 20:09:24 +0000 Subject: [PATCH 03/21] jenkins/sdk: Drop redundant update_chroot This happens from "cork update --create" already. --- jenkins/sdk.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh index 8437776a4e..ca482dd2b6 100644 --- a/jenkins/sdk.sh +++ b/jenkins/sdk.sh @@ -14,7 +14,6 @@ gpg --import "${GPG_SECRET_KEY_FILE}" sudo rm -rf src/build S=/mnt/host/source/src/scripts -enter ${S}/update_chroot enter sudo emerge -uv --jobs=2 catalyst enter sudo ${S}/bootstrap_sdk \ --sign="${SIGNING_USER}" \ From 9adb659106babd8b3abec9899f62499a4a6e584d Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 18 Oct 2018 15:58:27 +0000 Subject: [PATCH 04/21] prod_image_util: Drop the portage cache database --- build_library/prod_image_util.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 89d31e6746..33ef711e56 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -94,6 +94,7 @@ create_prod_image() { # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env sudo rm -rf ${root_fs_dir}/etc/env.d + sudo rm -rf ${root_fs_dir}/var/cache/edb sudo rm -rf ${root_fs_dir}/var/db/pkg sudo mv ${root_fs_dir}/etc/profile.env \ From 23c5b018c68dcb59426f8dba4dfa0d0cbb1bfb33 Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 18 Oct 2018 16:32:32 +0000 Subject: [PATCH 05/21] prod_image_util: Drop /usr/include This should be covered by INSTALL_MASK, but it apparently gets confused by a symlink loop installed by json-c. Remove it here. --- build_library/prod_image_util.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 33ef711e56..b3b707381d 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -94,6 +94,7 @@ create_prod_image() { # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env sudo rm -rf ${root_fs_dir}/etc/env.d + sudo rm -rf ${root_fs_dir}/usr/include sudo rm -rf ${root_fs_dir}/var/cache/edb sudo rm -rf ${root_fs_dir}/var/db/pkg From a59886441395a7b60820faf64a0e315cef9f6df0 Mon Sep 17 00:00:00 2001 From: David Michael Date: Fri, 19 Oct 2018 12:12:46 -0400 Subject: [PATCH 06/21] catalyst: Update sharedir for catalyst 3 --- build_library/catalyst.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index ea1fccf774..6b3f571966 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -59,7 +59,7 @@ contents="auto" digests="md5 sha1 sha512 whirlpool" hash_function="crc32" options="pkgcache" -sharedir="/usr/lib/catalyst" +sharedir="/usr/share/catalyst" storedir="$CATALYST_ROOT" distdir="$DISTDIR" envscript="$TEMPDIR/catalystrc" From e9f17c814306aa0ac476e51cf1c69f3a159d6e05 Mon Sep 17 00:00:00 2001 From: David Michael Date: Tue, 23 Oct 2018 17:36:30 +0000 Subject: [PATCH 07/21] jenkins: Drop the unused arm64 image format list --- jenkins/formats-arm64-usr.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 jenkins/formats-arm64-usr.txt diff --git a/jenkins/formats-arm64-usr.txt b/jenkins/formats-arm64-usr.txt deleted file mode 100644 index fe6f2e2598..0000000000 --- a/jenkins/formats-arm64-usr.txt +++ /dev/null @@ -1,5 +0,0 @@ -openstack -openstack_mini -packet -pxe -qemu_uefi From 8f761a7fdb5ba29694ec7401bca8c2d7528305de Mon Sep 17 00:00:00 2001 From: David Michael Date: Tue, 23 Oct 2018 17:38:55 +0000 Subject: [PATCH 08/21] core_dev_sign_update: Default to the amd64-usr board There are no other supported boards at this point. --- core_dev_sign_update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core_dev_sign_update b/core_dev_sign_update index 46704653a4..d08b9a5c71 100755 --- a/core_dev_sign_update +++ b/core_dev_sign_update @@ -6,7 +6,7 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) assert_inside_chroot DEFINE_string data_dir "" "Directory containing downloaded release artifacts" -DEFINE_string board "" "Board to sign artifacts for" +DEFINE_string board "amd64-usr" "Board to sign artifacts for" DEFINE_string version "" "Version to sign artifacts for" DEFINE_integer n_signatures "2" "Number of signatures this release will be signed with" DEFINE_string output_dir "" "Output directory" From 8e8d07f9a03308c960d3ab406b119379d678597f Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 25 Oct 2018 14:11:42 +0000 Subject: [PATCH 09/21] common.sh: Use amd64-usr as the default default board We only support amd64-usr at this point, so this removes a required step when setting up a new SDK. If the default board is specified normally or through the environment, it will override this value. --- common.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common.sh b/common.sh index 7a47ed4363..453cadd5ba 100644 --- a/common.sh +++ b/common.sh @@ -383,6 +383,8 @@ if [[ -f ${GCLIENT_ROOT}/src/scripts/.default_board ]]; then die ".default_board: invalid name detected; please fix:" \ "'${DEFAULT_BOARD}'" fi +elif [ -z "${DEFAULT_BOARD-}" ]; then + DEFAULT_BOARD=amd64-usr fi # Directory to store built images. Should be set by sourcing script when used. From 2467923d56134683a0fd4c58808cd361ea1f855c Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 25 Oct 2018 16:00:09 +0000 Subject: [PATCH 10/21] Remove arm64 from supported board operations --- build_library/build_image_util.sh | 9 -------- build_library/check_root | 35 +++---------------------------- build_library/ebuild_aci_util.sh | 1 - build_library/grub.cfg | 19 ++++------------- build_library/grub_install.sh | 26 ----------------------- build_library/oem_aci_util.sh | 1 - build_library/prod_image_util.sh | 7 ++----- build_library/qemu_template.sh | 17 --------------- build_library/set_lsb_release | 5 ----- build_library/vm_image_util.sh | 17 --------------- check_out_of_date.py | 8 +++---- common.sh | 16 -------------- image_inject_bootchain | 2 -- oem/openstack/glance_load.sh | 3 --- setup_board | 2 +- 15 files changed, 13 insertions(+), 155 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index ccf24bd749..b64fc36ac5 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -103,8 +103,6 @@ generate_update() { run_ldconfig() { local root_fs_dir=$1 case ${ARCH} in - arm64) - sudo qemu-aarch64 "${root_fs_dir}"/usr/sbin/ldconfig -r "${root_fs_dir}";; x86|amd64) sudo ldconfig -r "${root_fs_dir}";; *) @@ -115,8 +113,6 @@ run_ldconfig() { run_localedef() { local root_fs_dir="$1" loader=() case ${ARCH} in - arm64) - loader=( qemu-aarch64 -L "${root_fs_dir}" );; amd64) loader=( "${root_fs_dir}/usr/lib64/ld-linux-x86-64.so.2" \ --library-path "${root_fs_dir}/usr/lib64" );; @@ -460,7 +456,6 @@ finish_image() { # Only enable rootfs verification on supported boards. case "${FLAGS_board}" in amd64-usr) verity_offset=64 ;; - arm64-usr) verity_offset=512 ;; *) disable_read_write=${FLAGS_FALSE} ;; esac @@ -532,7 +527,6 @@ EOF # Magic alert! Root hash injection works by writing the hash value to a # known unused SHA256-sized location in the kernel image. # For amd64 the rdev error message is used. - # For arm64 an area between the EFI headers and the kernel text is used. # Our modified GRUB extracts the hash and adds it to the cmdline. printf %s "$(cat ${BUILD_DIR}/${image_name%.bin}_verity.txt)" | \ sudo dd of="${root_fs_dir}/boot/coreos/vmlinuz-a" conv=notrunc \ @@ -570,9 +564,6 @@ EOF if [[ "${install_grub}" -eq 1 ]]; then local target local target_list="i386-pc x86_64-efi x86_64-xen" - if [[ ${BOARD} == "arm64-usr" ]]; then - target_list="arm64-efi" - fi local grub_args=() if [[ ${disable_read_write} -eq ${FLAGS_TRUE} ]]; then grub_args+=(--verity) diff --git a/build_library/check_root b/build_library/check_root index 4a1b41af55..b4d7615706 100755 --- a/build_library/check_root +++ b/build_library/check_root @@ -29,32 +29,10 @@ IGNORE_MISSING = { SonameAtom("x86_64", "libc.so.6")], # RPATHs and symlinks apparently confuse the perl-5.24 package - "dev-lang/perl": [SonameAtom("arm_64", "libperl.so.5.24.1"), - SonameAtom("x86_64", "libperl.so.5.24.1")], + "dev-lang/perl": [SonameAtom("x86_64", "libperl.so.5.24.1")], # https://bugs.gentoo.org/show_bug.cgi?id=554582 - "net-firewall/ebtables": [SonameAtom("arm_64", "libebt_802_3.so"), - SonameAtom("arm_64", "libebt_among.so"), - SonameAtom("arm_64", "libebt_arp.so"), - SonameAtom("arm_64", "libebt_arpreply.so"), - SonameAtom("arm_64", "libebt_ip.so"), - SonameAtom("arm_64", "libebt_ip6.so"), - SonameAtom("arm_64", "libebt_limit.so"), - SonameAtom("arm_64", "libebt_log.so"), - SonameAtom("arm_64", "libebt_mark.so"), - SonameAtom("arm_64", "libebt_mark_m.so"), - SonameAtom("arm_64", "libebt_nat.so"), - SonameAtom("arm_64", "libebt_nflog.so"), - SonameAtom("arm_64", "libebt_pkttype.so"), - SonameAtom("arm_64", "libebt_redirect.so"), - SonameAtom("arm_64", "libebt_standard.so"), - SonameAtom("arm_64", "libebt_stp.so"), - SonameAtom("arm_64", "libebt_ulog.so"), - SonameAtom("arm_64", "libebt_vlan.so"), - SonameAtom("arm_64", "libebtable_broute.so"), - SonameAtom("arm_64", "libebtable_filter.so"), - SonameAtom("arm_64", "libebtable_nat.so"), - SonameAtom("x86_64", "libebt_802_3.so"), + "net-firewall/ebtables": [SonameAtom("x86_64", "libebt_802_3.so"), SonameAtom("x86_64", "libebt_among.so"), SonameAtom("x86_64", "libebt_arp.so"), SonameAtom("x86_64", "libebt_arpreply.so"), @@ -77,14 +55,7 @@ IGNORE_MISSING = { SonameAtom("x86_64", "libebtable_nat.so")], # Ignore the Rust libraries in their own libdir. - "dev-libs/rustlib": [SonameAtom("arm_64", "librustc_data_structures.so"), - SonameAtom("arm_64", "librustc_errors.so"), - SonameAtom("arm_64", "libserialize.so"), - SonameAtom("arm_64", "libstd.so"), - SonameAtom("arm_64", "libsyntax.so"), - SonameAtom("arm_64", "libsyntax_pos.so"), - SonameAtom("arm_64", "libterm.so"), - SonameAtom("x86_64", "librustc_data_structures.so"), + "dev-libs/rustlib": [SonameAtom("x86_64", "librustc_data_structures.so"), SonameAtom("x86_64", "librustc_errors.so"), SonameAtom("x86_64", "libserialize.so"), SonameAtom("x86_64", "libstd.so"), diff --git a/build_library/ebuild_aci_util.sh b/build_library/ebuild_aci_util.sh index 73aedfba74..7ae6d1824b 100644 --- a/build_library/ebuild_aci_util.sh +++ b/build_library/ebuild_aci_util.sh @@ -40,7 +40,6 @@ ebuild_aci_write_manifest() { case "${BOARD}" in amd64-usr) appc_arch=amd64 ;; - arm64-usr) appc_arch=aarch64 ;; *) die_notrace "Cannot map \"${BOARD}\" to an appc arch" ;; esac diff --git a/build_library/grub.cfg b/build_library/grub.cfg index 13c8fc7b72..1d347d5389 100644 --- a/build_library/grub.cfg +++ b/build_library/grub.cfg @@ -91,32 +91,21 @@ if [ -z "$linux_console" ]; then terminal_input console serial_com0 terminal_output console serial_com0 elif [ "$grub_platform" = efi ]; then - if [ "$grub_cpu" = arm64 ]; then - set linux_console="console=ttyAMA0,115200n8" - else - set linux_console="console=ttyS0,115200n8 console=tty0" - fi + set linux_console="console=ttyS0,115200n8 console=tty0" elif [ "$grub_platform" = xen ]; then set linux_console="console=hvc0" fi fi -set extra_options="" -if [ "$grub_cpu" = arm64 ]; then - set extra_options="acpi=force" -fi - set suf="" -# UEFI uses linuxefi/initrdefi instead of linux/initrd except for arm64 +# UEFI uses linuxefi/initrdefi instead of linux/initrd if [ "$grub_platform" = efi ]; then - if [ "$grub_cpu" != arm64 ]; then - set suf="efi" - fi + set suf="efi" fi # Assemble the options applicable to all the kernels below -set linux_cmdline="rootflags=rw mount.usrflags=ro consoleblank=0 $linux_root $linux_console $first_boot $randomize_disk_guid $extra_options $oem $linux_append" +set linux_cmdline="rootflags=rw mount.usrflags=ro consoleblank=0 $linux_root $linux_console $first_boot $randomize_disk_guid $oem $linux_append" # Re-implement grub_abort() since no command exposes it. function abort { diff --git a/build_library/grub_install.sh b/build_library/grub_install.sh index adf411596a..c2a12190a2 100755 --- a/build_library/grub_install.sh +++ b/build_library/grub_install.sh @@ -48,10 +48,6 @@ CORE_MODULES=( normal search test fat part_gpt search_fs_uuid gzio search_part_l # Name of the core image, depends on target CORE_NAME= -# Whether the SDK's grub or the board root's grub is used. Once amd64 is -# fixed up the board root's grub will always be used. -BOARD_GRUB=0 - case "${FLAGS_target}" in i386-pc) CORE_MODULES+=( biosdisk serial ) @@ -64,21 +60,11 @@ case "${FLAGS_target}" in x86_64-xen) CORE_NAME="core.elf" ;; - arm64-efi) - CORE_MODULES+=( serial linux efi_gop getenv smbios efinet verify http tftp ) - CORE_NAME="core.efi" - BOARD_GRUB=1 - ;; *) die_notrace "Unknown GRUB target ${FLAGS_target}" ;; esac -if [[ $BOARD_GRUB -eq 1 ]]; then - info "Updating GRUB in ${BOARD_ROOT}" - emerge-${BOARD} --nodeps --select -qugKN sys-boot/grub - GRUB_SRC="${BOARD_ROOT}/usr/lib/grub/${FLAGS_target}" -fi [[ -d "${GRUB_SRC}" ]] || die "GRUB not installed at ${GRUB_SRC}" # In order for grub-setup-bios to properly detect the layout of the disk @@ -230,18 +216,6 @@ case "${FLAGS_target}" in sudo cp "${BUILD_LIBRARY_DIR}/menu.lst" \ "${ESP_DIR}/boot/grub/menu.lst" ;; - arm64-efi) - info "Installing default arm64 UEFI bootloader." - sudo mkdir -p "${ESP_DIR}/EFI/boot" - #FIXME(andrejro): shim not ported to aarch64 - sudo cp "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \ - "${ESP_DIR}/EFI/boot/bootaa64.efi" - if [[ -n "${FLAGS_copy_efi_grub}" ]]; then - # copying from vfat so ignore permissions - cp --no-preserve=mode "${ESP_DIR}/EFI/boot/bootaa64.efi" \ - "${FLAGS_copy_efi_grub}" - fi - ;; esac cleanup diff --git a/build_library/oem_aci_util.sh b/build_library/oem_aci_util.sh index f6a1bbf176..f7a19987ae 100644 --- a/build_library/oem_aci_util.sh +++ b/build_library/oem_aci_util.sh @@ -74,7 +74,6 @@ oem_aci_write_manifest() { case "${BOARD}" in amd64-usr) appc_arch=amd64 ;; - arm64-usr) appc_arch=aarch64 ;; *) die_notrace "Cannot map \"${BOARD}\" to an appc arch" ;; esac diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index b3b707381d..83400892c6 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -85,7 +85,7 @@ create_prod_image() { # Assert that if this is supposed to be an official build that the # official update keys have been used. - if [[ ${COREOS_OFFICIAL:-0} -eq 1 && "${BOARD}" != arm64-usr ]]; then + if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then grep -q official \ "${root_fs_dir}"/var/db/pkg/coreos-base/coreos-au-key-*/USE \ || die_notrace "coreos-au-key is missing the 'official' use flag" @@ -141,11 +141,8 @@ EOF "${BUILD_DIR}/${image_kernel}" "${BUILD_DIR}/${image_pcr_policy}" "${BUILD_DIR}/${image_grub}" + "${BUILD_DIR}/${image_shim}" "${BUILD_DIR}/${image_kconfig}" ) - # FIXME(bgilbert): no shim on arm64 - if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then - to_upload+=("${BUILD_DIR}/${image_shim}") - fi upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" "${to_upload[@]}" } diff --git a/build_library/qemu_template.sh b/build_library/qemu_template.sh index 06cfd00faf..e287f1b958 100755 --- a/build_library/qemu_template.sh +++ b/build_library/qemu_template.sh @@ -153,10 +153,6 @@ else set -- -machine accel=kvm -cpu host -smp "${VM_NCPUS}" "$@" ;; amd64-usr+*) set -- -machine pc-q35-2.8 -cpu kvm64 -smp 1 -nographic "$@" ;; - arm64-usr+aarch64) - set -- -machine virt,accel=kvm,gic-version=3 -cpu host -smp "${VM_NCPUS}" -nographic "$@" ;; - arm64-usr+*) - set -- -machine virt -cpu cortex-a57 -smp 1 -nographic "$@" ;; *) die "Unsupported arch" ;; esac @@ -177,10 +173,6 @@ if [ -n "${VM_IMAGE}" ]; then case "${VM_BOARD}" in amd64-usr) set -- -drive if=virtio,file="${SCRIPT_DIR}/${VM_IMAGE}" "$@" ;; - arm64-usr) - set -- -drive if=none,id=blk,file="${SCRIPT_DIR}/${VM_IMAGE}" \ - -device virtio-blk-device,drive=blk "$@" - ;; *) die "Unsupported arch" ;; esac fi @@ -223,15 +215,6 @@ case "${VM_BOARD}" in -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ "$@" ;; - arm64-usr) - qemu-system-aarch64 \ - -name "$VM_NAME" \ - -m ${VM_MEMORY} \ - -netdev user,id=eth0,hostfwd=tcp::"${SSH_PORT}"-:22,hostname="${VM_NAME}" \ - -device virtio-net-device,netdev=eth0 \ - -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ - "$@" - ;; *) die "Unsupported arch" ;; esac diff --git a/build_library/set_lsb_release b/build_library/set_lsb_release index 3c7a88cdc2..cfc9967297 100755 --- a/build_library/set_lsb_release +++ b/build_library/set_lsb_release @@ -30,12 +30,7 @@ OS_CODENAME="Rhyolite" OS_ID="coreos" OS_PRETTY_NAME="$OS_NAME $COREOS_VERSION (${OS_CODENAME})" -# temporarily point arm at a different appid until update support -# is completed in update_engine and the CoreUpdate service. COREOS_APPID="{e96281a6-d1af-4bde-9a0a-97b76e56dc57}" -if [[ "${FLAGS_board}" == arm64-* ]]; then - COREOS_APPID="{103867da-e3a2-4c92-b0b3-7fbd7f7d8b71}" -fi # DISTRIB_* are the standard lsb-release names sudo mkdir -p "${ROOT_FS_DIR}/usr/share/coreos" "${ROOT_FS_DIR}/etc/coreos" diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 2732ac2391..4922a9f9a4 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -303,9 +303,6 @@ get_default_vm_type() { amd64-usr) echo "qemu" ;; - arm64-usr) - echo "qemu_uefi" - ;; *) return 1 ;; @@ -622,7 +619,6 @@ _write_cpio_disk() { local grub_arch case $BOARD in amd64-usr) grub_arch="x86_64-efi" ;; - arm64-usr) grub_arch="arm64-efi" ;; esac cp "${base_dir}/boot/coreos/grub/${grub_arch}/core.efi" "${dst_dir}/${grub_name}" @@ -719,19 +715,6 @@ _write_qemu_uefi_conf() { cp "/usr/share/edk2/OVMF_CODE.fd" "$(_dst_dir)/${flash_ro}" cp "/usr/share/edk2/OVMF_VARS.fd" "$(_dst_dir)/${flash_rw}" ;; - arm64-usr) - # Get edk2 files into local build workspace. - info "Updating edk2 in /build/${BOARD}" - emerge-${BOARD} --nodeps --select -qugKN sys-firmware/edk2 - # Create 64MiB flash device image files. - dd if=/dev/zero bs=1M count=64 of="$(_dst_dir)/${flash_rw}" \ - status=none - cp "/build/${BOARD}/usr/share/edk2/QEMU_EFI.fd" \ - "$(_dst_dir)/${flash_ro}.work" - truncate --reference="$(_dst_dir)/${flash_rw}" \ - "$(_dst_dir)/${flash_ro}.work" - mv "$(_dst_dir)/${flash_ro}.work" "$(_dst_dir)/${flash_ro}" - ;; esac sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO='${flash_ro}'%" \ diff --git a/check_out_of_date.py b/check_out_of_date.py index 0f32991cab..f0b99f072c 100755 --- a/check_out_of_date.py +++ b/check_out_of_date.py @@ -67,11 +67,11 @@ def process_emerge_output(eout): def get_board_packages(board): - """ gets a list of packages used by a board. valid boards are {arm,amd}64-usr, sdk, and bootstrap""" + """ gets a list of packages used by a board. valid boards are amd64-usr, sdk, and bootstrap""" emerge_args = "--emptytree --pretend --verbose --unordered-display" if board == "sdk": cmd = "emerge {} @system sdk-depends sdk-extras".format(emerge_args) - elif board == "amd64-usr" or board == "arm64-usr": + elif board == "amd64-usr": cmd = "emerge-{} {} @system board-packages".format(board, emerge_args) elif board == "bootstrap": pkgs = exec_command_strict("/usr/lib64/catalyst/targets/stage1/build.py") @@ -92,7 +92,6 @@ def print_table(report, head, line_head, line_tail, tail, joiner, pkg_joiner): "upstream": ["Upstream"], "tag": "Tag", "sdk": ["sdk"], - "arm64-usr": ["arm64-usr"], "amd64-usr": ["amd64-usr"], "bootstrap": ["bootstrap"], "modified": "Modified"}) @@ -103,7 +102,6 @@ def print_table(report, head, line_head, line_tail, tail, joiner, pkg_joiner): pkg_joiner.join(entry.get("upstream",[])), entry.get("tag",""), pkg_joiner.join(entry.get("sdk", [])), - pkg_joiner.join(entry.get("arm64-usr", [])), pkg_joiner.join(entry.get("amd64-usr", [])), pkg_joiner.join(entry.get("bootstrap", [])), entry.get("modified","")]) + line_tail) @@ -140,7 +138,7 @@ def main(): subprocess.check_call(["git", "-C", args.upstream_path, "pull"]) pkg_lists = {} - sources = ["sdk", "bootstrap", "amd64-usr", "arm64-usr", "image"] + sources = ["sdk", "bootstrap", "amd64-usr", "image"] for i in sources: pkg_lists[i] = get_board_packages(i) diff --git a/common.sh b/common.sh index 453cadd5ba..aa31ab4718 100644 --- a/common.sh +++ b/common.sh @@ -928,19 +928,10 @@ BOAT } # The binfmt_misc support in the kernel is required. -# The aarch64 binaries should be executed through -# "/usr/bin/qemu-aarch64-static" setup_qemu_static() { local root_fs_dir="$1" case "${BOARD}" in amd64-usr) return 0;; - arm64-usr) - if [[ -f "${root_fs_dir}/sbin/ldconfig" ]]; then - sudo cp /usr/bin/qemu-aarch64 "${root_fs_dir}"/usr/bin/qemu-aarch64-static - else - die "Missing basic layout in target rootfs" - fi - ;; *) die "Unsupported arch" ;; esac } @@ -949,13 +940,6 @@ clean_qemu_static() { local root_fs_dir="$1" case "${BOARD}" in amd64-usr) return 0;; - arm64-usr) - if [[ -f "${root_fs_dir}/usr/bin/qemu-aarch64-static" ]]; then - sudo rm "${root_fs_dir}"/usr/bin/qemu-aarch64-static - else - die "File not found" - fi - ;; *) die "Unsupported arch" ;; esac } diff --git a/image_inject_bootchain b/image_inject_bootchain index a537242ba6..2430ee21b6 100755 --- a/image_inject_bootchain +++ b/image_inject_bootchain @@ -77,11 +77,9 @@ if [[ -n "${FLAGS_kernel_path}" ]]; then do_copy "${FLAGS_kernel_path}" "/boot/coreos/vmlinuz-a" fi -# FIXME(bgilbert): no shim on arm64 if [[ -n "${FLAGS_efi_grub_path}" ]]; then case "${BOARD}" in amd64-usr) image_name="grub.efi" ;; - arm64-usr) image_name="bootaa64.efi" ;; *) die "GRUB filename not known for this board" ;; esac diff --git a/oem/openstack/glance_load.sh b/oem/openstack/glance_load.sh index e79b06fb69..3bef9377d6 100755 --- a/oem/openstack/glance_load.sh +++ b/oem/openstack/glance_load.sh @@ -34,9 +34,6 @@ while [[ $# -gt 0 ]]; do amd64|amd64-usr) board="amd64-usr" ;; - arm64|arm64-usr) - board="arm64-usr" - ;; *) echo "Error: Unknown arg: $1" exit 1 diff --git a/setup_board b/setup_board index 6e16283f8a..73a8b8cb73 100755 --- a/setup_board +++ b/setup_board @@ -59,7 +59,7 @@ DEFINE_string variant "" \ "Board variant." -# builds wrappers like equery-arm-generic. +# builds wrappers like equery-amd64-usr. # args: # $1: command to wrap # rest: extra arguments to pass to the command From f9432f6702ff2b2355c47688c3a03f8eba01a29b Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 25 Oct 2018 17:15:03 +0000 Subject: [PATCH 11/21] vm_image_util: Switch back to edk2-ovmf paths The custom sys-firmware/edk2 package has been replaced by Gentoo's sys-firmware/edk2-ovmf package now that only amd64 is supported. This partially reverts 1761d9d071794f50267da782d135fb44b8344e8b . --- build_library/vm_image_util.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 4922a9f9a4..29ef4200f9 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -712,8 +712,8 @@ _write_qemu_uefi_conf() { case $BOARD in amd64-usr) - cp "/usr/share/edk2/OVMF_CODE.fd" "$(_dst_dir)/${flash_ro}" - cp "/usr/share/edk2/OVMF_VARS.fd" "$(_dst_dir)/${flash_rw}" + cp "/usr/share/edk2-ovmf/OVMF_CODE.fd" "$(_dst_dir)/${flash_ro}" + cp "/usr/share/edk2-ovmf/OVMF_VARS.fd" "$(_dst_dir)/${flash_rw}" ;; esac From ba7d7f14104c6ccf3f44c8e77df3b04fd19f3211 Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 25 Oct 2018 20:22:35 +0000 Subject: [PATCH 12/21] update_chroot: Add a workaround for a package rename This avoids SDK update failures due to conflicts. It can be reverted once SDKs have the new version built into them. --- update_chroot | 3 +++ 1 file changed, 3 insertions(+) diff --git a/update_chroot b/update_chroot index 870d599ef6..b9eff5847a 100755 --- a/update_chroot +++ b/update_chroot @@ -233,6 +233,9 @@ if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then done fi +# XXX: Work around a rename; delete this after SDKs contain the new package. +sudo -E ${EMERGE_CMD} --rage-clean app-crypt/sbsigntool 2>/dev/null || : + # Second pass, update everything else. EMERGE_FLAGS+=( --deep ) info "Updating all SDK packages" From 373d5a814ba7b4ecba181ad37b8b4c8b98bd0402 Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 25 Oct 2018 17:08:02 -0400 Subject: [PATCH 13/21] catalyst: Temporarily disable update_seed again This works around the edk2 rename breaking SDK bootstrapping. It can be reverted when an SDK has the new version built into it. --- build_library/catalyst.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 6b3f571966..7592237483 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -120,7 +120,7 @@ cat < Date: Thu, 25 Oct 2018 12:14:18 -0700 Subject: [PATCH 14/21] signing/sign.sh: fix armoring We use gpg --enarmor to convert the torcx manifest binary sigs to ascii armored ones. Unfortunately --enarmor just wraps the binary blob without realizing its a signature, which breaks torcx. Pipe the output through sed to fix the header. --- signing/sign.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/signing/sign.sh b/signing/sign.sh index fdc61dd458..3eabd2dbd9 100755 --- a/signing/sign.sh +++ b/signing/sign.sh @@ -56,7 +56,10 @@ fero-client \ --secret-key coreos-app-signing-key \ ${torcx_signature_arg} gpg2 --enarmor \ - --output "${DATA_DIR}/torcx_manifest.json.asc" \ - "${DATA_DIR}/torcx_manifest.json.sig-fero" + --output - \ + "${DATA_DIR}/torcx_manifest.json.sig-fero" \ + | sed 's/ARMORED FILE/SIGNATURE/' \ + > "${DATA_DIR}/torcx_manifest.json.asc" + echo "=== Torcx manifest signed successfully. ===" rm -f "${DATA_DIR}/torcx_manifest.json.sig-fero" From 103c7425fe64110740a92daac21bcd83f293d3ec Mon Sep 17 00:00:00 2001 From: David Michael Date: Thu, 1 Nov 2018 14:45:21 +0000 Subject: [PATCH 15/21] build_library: Whitelist the systemd GLSA until we have 239 --- build_library/test_image_content.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_library/test_image_content.sh b/build_library/test_image_content.sh index a79b5703b2..40fcd69244 100644 --- a/build_library/test_image_content.sh +++ b/build_library/test_image_content.sh @@ -7,6 +7,7 @@ GLSA_WHITELIST=( 201710-23 # we handle Go differently; drop when 1.9 builds everything 201803-03 # same as above, drop when all Go < 1.9 packages are gone 201804-12 # same as above, except this requires only Go 1.10 or later + 201810-10 # we fixed the systemd CVEs in 238, but this wants 239 ) glsa_image() { From fceffdb6601d146fd20adb8528a954c614822ccf Mon Sep 17 00:00:00 2001 From: David Michael Date: Wed, 14 Nov 2018 21:09:52 +0000 Subject: [PATCH 16/21] update_chroot: Work around dead cargo file conflicts --- update_chroot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_chroot b/update_chroot index b9eff5847a..293d6bfef7 100755 --- a/update_chroot +++ b/update_chroot @@ -234,7 +234,7 @@ if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then fi # XXX: Work around a rename; delete this after SDKs contain the new package. -sudo -E ${EMERGE_CMD} --rage-clean app-crypt/sbsigntool 2>/dev/null || : +sudo -E ${EMERGE_CMD} --rage-clean app-crypt/sbsigntool dev-util/cargo 2>/dev/null || : # Second pass, update everything else. EMERGE_FLAGS+=( --deep ) From e1761bee547e39eb89a516b9829983ce90c5f266 Mon Sep 17 00:00:00 2001 From: David Michael Date: Fri, 16 Nov 2018 14:11:00 +0000 Subject: [PATCH 17/21] Revert "catalyst: Temporarily disable update_seed again" This reverts commit 373d5a814ba7b4ecba181ad37b8b4c8b98bd0402. --- build_library/catalyst.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 7592237483..6b3f571966 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -120,7 +120,7 @@ cat < Date: Fri, 16 Nov 2018 11:19:39 -0800 Subject: [PATCH 18/21] signing/sign.sh: update fero ip --- signing/sign.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signing/sign.sh b/signing/sign.sh index fdc61dd458..79d2050537 100755 --- a/signing/sign.sh +++ b/signing/sign.sh @@ -10,7 +10,7 @@ fi DATA_DIR="$(readlink -f "$1")" KEYS_DIR="$(readlink -f "$(dirname "$0")")" SIGS_DIR="$(readlink -f "$2")" -SERVER_ADDR="${3:-10.7.68.100}" +SERVER_ADDR="${3:-10.7.68.102}" SERVER_PORT="${4:-50051}" echo "=== Verifying update payload... ===" From e443b9afdb7605e6ebad5c5d147127334309bd91 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Tue, 20 Nov 2018 13:15:51 -0500 Subject: [PATCH 19/21] transfer.sh: drop recursive chown This shouldn't be needed now that we're not copying files via a USB stick. --- signing/transfer.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/signing/transfer.sh b/signing/transfer.sh index 6ee1fb0142..22f45b220d 100755 --- a/signing/transfer.sh +++ b/signing/transfer.sh @@ -175,11 +175,6 @@ case "${CMD}" in if [[ -z "${BASEDIR}" ]]; then usage fi - - if [[ -d "${BASEDIR}" && ! -O "${BASEDIR}" ]]; then - echo "Fixing ownership of ${BASEDIR}..." - sudo chown -R "${USER}" "${BASEDIR}" - fi ;; esac From 6d1cfa6a75434a1dd534b660f91d11b1e71841e8 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Tue, 20 Nov 2018 13:10:36 -0500 Subject: [PATCH 20/21] transfer.sh: add subcommands to devsign and sign releases Both subcommands take a SIGDIR argument which is assumed to be within a git repo we can safely sync. devsign creates commits but does not (currently) push them, leaving that to the user. --- signing/transfer.sh | 46 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/signing/transfer.sh b/signing/transfer.sh index 22f45b220d..4a152f82c7 100755 --- a/signing/transfer.sh +++ b/signing/transfer.sh @@ -46,6 +46,28 @@ download() { popd >/dev/null } +devsign() { + local channel="$1" + local version="$2" + + "$(dirname $0)/../core_dev_sign_update" \ + --data_dir "${BASEDIR}" \ + --version "${version}" \ + --output_dir "${SIGDIR}" + + git -C "${SIGDIR}" add . + git -C "${SIGDIR}" commit -m "Add sigs from ${USER} for ${channel} ${version}" +} + +sign() { + local channel="$1" + local version="$2" + + "$(dirname $0)/sign.sh" \ + "${BASEDIR}/${BOARD}/${version}" \ + "${SIGDIR}/${BOARD}/${version}" +} + upload() { local channel="$1" local version="$2" @@ -141,6 +163,7 @@ roll() { usage() { echo "Usage: $0 {download|upload} [{-a|-b|-s} ]..." >&2 + echo "Usage: $0 {devsign|sign} [{-a|-b|-s} ]..." >&2 echo "Usage: $0 ready [{-a|-b|-s} ]..." >&2 echo "Usage: $0 roll [{-a|-b|-s} ]..." >&2 exit 1 @@ -150,7 +173,7 @@ usage() { CMD="${1:-}" shift ||: case "${CMD}" in - download) + download|devsign|sign) ;; upload|ready|roll) if [[ -e "${HOME}/.config/roller.conf" ]]; then @@ -167,9 +190,9 @@ case "${CMD}" in ;; esac -# Parse basedir if necessary. +# Parse fixed args if necessary. case "${CMD}" in - download|upload) + download|devsign|sign|upload) BASEDIR="${1:-}" shift ||: if [[ -z "${BASEDIR}" ]]; then @@ -177,6 +200,23 @@ case "${CMD}" in fi ;; esac +case "${CMD}" in + devsign|sign) + SIGDIR="${1:-}" + shift ||: + if [[ -z "${SIGDIR}" ]]; then + usage + fi + ;; +esac + +# Sync SIGDIR exactly once. +case "${CMD}" in + devsign|sign) + echo "Updating ${SIGDIR}..." + git -C "${SIGDIR}" pull -r + ;; +esac # Walk argument pairs. while [[ $# > 0 ]]; do From 45c9ef7867c9cec2b2f0f71eed4feb6393975e23 Mon Sep 17 00:00:00 2001 From: David Michael Date: Fri, 21 Dec 2018 16:27:55 +0000 Subject: [PATCH 21/21] build_library: Drop whitelisted Go GLSAs Since Go is no longer cross-compiled, it won't hit the boards' GLSA tests anymore. --- build_library/test_image_content.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/build_library/test_image_content.sh b/build_library/test_image_content.sh index 40fcd69244..d22ab677c6 100644 --- a/build_library/test_image_content.sh +++ b/build_library/test_image_content.sh @@ -4,9 +4,6 @@ GLSA_WHITELIST=( 201412-09 # incompatible CA certificate version numbers - 201710-23 # we handle Go differently; drop when 1.9 builds everything - 201803-03 # same as above, drop when all Go < 1.9 packages are gone - 201804-12 # same as above, except this requires only Go 1.10 or later 201810-10 # we fixed the systemd CVEs in 238, but this wants 239 )