From c611fcd139262f3c95450408e8d53cd281f624e7 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Mon, 15 Jul 2024 12:03:01 +0100 Subject: [PATCH] Support the SDK on arm64 Catalyst 4 adds support for building with QEMU, so I initially leveraged this, but it turned out to be very much slower than emulating the amd64 SDK on arm64, where an arm64 build could then be mostly run without emulation. I have kept the code for the slower approach anyway since it is small and may be useful to somebody. There were several places where we assumed that amd64 was native and arm64 required emulation via QEMU. The scripts are now more architecture-agnostic, paving the way for riscv support later. We no longer set QEMU_LD_PREFIX because it prevents the SDK itself from being emulated. It also assumes there is only one non-native target, which may not always be the case. bubblewrap does a better job of running binaries under QEMU. Signed-off-by: James Le Cuirot --- build_library/build_image_util.sh | 29 +++----------- build_library/catalyst.sh | 15 ++++++++ build_library/portage/env/releng/qemu | 1 + build_library/portage/package.env/qemu | 1 + common.sh | 38 ------------------- .../coreos/config/env/dev-lang/rust | 3 +- .../profiles/coreos/amd64/sdk/package.use | 3 ++ .../profiles/coreos/arm64/sdk/package.use | 3 ++ .../profiles/coreos/targets/sdk/make.defaults | 6 --- .../profiles/coreos/targets/sdk/package.use | 2 +- setup_board | 26 ++++++++++--- 11 files changed, 51 insertions(+), 76 deletions(-) create mode 100644 build_library/portage/env/releng/qemu create mode 100644 build_library/portage/package.env/qemu create mode 100644 sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 221095d45c..4ff46dcb03 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -102,32 +102,13 @@ zip_update_tools() { --arch "$(get_sdk_arch)" --output-dir "${BUILD_DIR}" --zip-name "${update_zip}" } -# ldconfig cannot generate caches for non-native arches. -# Use qemu & the native ldconfig to work around that. -# http://code.google.com/p/chromium/issues/detail?id=378377 run_ldconfig() { - local root_fs_dir=$1 - case ${ARCH} in - arm64) - sudo qemu-aarch64 "${root_fs_dir}"/usr/sbin/ldconfig -r "${root_fs_dir}";; - x86|amd64) - sudo ldconfig -r "${root_fs_dir}";; - *) - die "Unable to run ldconfig for ARCH ${ARCH}" - esac + # This wrapper is created by setup_board. + sudo "ldconfig-${BOARD}" -r "$1" } run_localedef() { - local root_fs_dir="$1" loader=() - case ${ARCH} in - arm64) - loader=( qemu-aarch64 -L "${root_fs_dir}" );; - amd64) - loader=( "${root_fs_dir}/usr/lib64/ld-linux-x86-64.so.2" \ - --library-path "${root_fs_dir}/usr/lib64" );; - *) - die "Unable to run localedef for ARCH ${ARCH}";; - esac + local root_fs_dir="$1" info "Generating C.UTF-8 locale..." local i18n="${root_fs_dir}/usr/share/i18n" # localedef will silently fall back to /usr/share/i18n if missing so @@ -135,8 +116,8 @@ run_localedef() { [[ -f "${i18n}/charmaps/UTF-8.gz" ]] || die [[ -f "${i18n}/locales/C" ]] || die sudo mkdir -p "${root_fs_dir}/usr/lib/locale" - sudo I18NPATH="${i18n}" "${loader[@]}" "${root_fs_dir}/usr/bin/localedef" \ - --prefix="${root_fs_dir}" --charmap=UTF-8 --inputfile=C C.UTF-8 + sudo I18NPATH="${i18n}" "bwrap-${BOARD}" "${root_fs_dir}" /usr/bin/localedef \ + --charmap=UTF-8 --inputfile=C C.UTF-8 } # Basic command to emerge binary packages into the target image. diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 3e792417be..c4055cee63 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -25,6 +25,7 @@ BINPKGS= DISTDIR= TEMPDIR= STAGES= +unset QEMU DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \ "Path to directory for all catalyst images and other files." @@ -97,6 +98,7 @@ cflags: -O2 -pipe cxxflags: -O2 -pipe ldflags: -Wl,-O2 -Wl,--as-needed source_subpath: ${SEED} +${QEMU+interpreter: $(type -P "${QEMU}")} EOF } @@ -207,6 +209,16 @@ catalyst_init() { SEED="seed/${FLAGS_seed_tarball##*/}" SEED="${SEED%.tar.*}" fi + + # Emulate the build, if needed. Note the SDK itself may already be emulated, + # so check the requested arch against the kernel's real arch, not uname -m. + if [[ ${ARCH} != $(get_portage_arch "$(< /proc/sys/kernel/arch)") ]]; then + case "${ARCH}" in + amd64) QEMU=qemu-x86_64 ;; + arm64) QEMU=qemu-aarch64 ;; + riscv) QEMU=qemu-riscv64 ;; + esac + fi } write_configs() { @@ -226,6 +238,9 @@ write_configs() { ln -sfT '/mnt/host/source/src/third_party/coreos-overlay/coreos/user-patches' \ "${TEMPDIR}"/portage/patches + + [[ -n ${QEMU} ]] || + rm "${TEMPDIR}"/portage/package.env/qemu } build_stage() { diff --git a/build_library/portage/env/releng/qemu b/build_library/portage/env/releng/qemu new file mode 100644 index 0000000000..de86517db4 --- /dev/null +++ b/build_library/portage/env/releng/qemu @@ -0,0 +1 @@ +FEATURES="-pid-sandbox -network-sandbox -ipc-sandbox" diff --git a/build_library/portage/package.env/qemu b/build_library/portage/package.env/qemu new file mode 100644 index 0000000000..60c290a8ba --- /dev/null +++ b/build_library/portage/package.env/qemu @@ -0,0 +1 @@ +*/* releng/qemu diff --git a/common.sh b/common.sh index 3dbb8040dc..0cff99ed56 100644 --- a/common.sh +++ b/common.sh @@ -51,9 +51,6 @@ fi # Turn on bash debug support if available for backtraces. shopt -s extdebug 2>/dev/null -# Source qemu library path -. /etc/profile.d/qemu-aarch64.sh 2> /dev/null || true - # Output a backtrace all the way back to the raw invocation, suppressing # only the _dump_trace frame itself. _dump_trace() { @@ -992,38 +989,3 @@ BOAT echo -e "${V_VIDOFF}" die "$* failed" } - -# The binfmt_misc support in the kernel is required. -# The aarch64 binaries should be executed through -# "/usr/bin/qemu-aarch64-static" -setup_qemu_static() { - local root_fs_dir="$1" - case "${BOARD}" in - amd64-usr) return 0;; - arm64-usr) - if [[ -f "${root_fs_dir}/sbin/ldconfig" ]]; then - sudo cp /usr/bin/qemu-aarch64 "${root_fs_dir}"/usr/bin/qemu-aarch64-static - echo export QEMU_LD_PREFIX=\"/build/arm64-usr/\" | sudo tee /etc/profile.d/qemu-aarch64.sh - . /etc/profile.d/qemu-aarch64.sh - else - die "Missing basic layout in target rootfs" - fi - ;; - *) die "Unsupported arch" ;; - esac -} - -clean_qemu_static() { - local root_fs_dir="$1" - case "${BOARD}" in - amd64-usr) return 0;; - arm64-usr) - if [[ -f "${root_fs_dir}/usr/bin/qemu-aarch64-static" ]]; then - sudo rm "${root_fs_dir}"/usr/bin/qemu-aarch64-static - else - die "File not found" - fi - ;; - *) die "Unsupported arch" ;; - esac -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust index c467aedead..d72efdb3d5 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust @@ -2,5 +2,6 @@ INSTALL_MASK+=" *rustdoc*" I_KNOW_WHAT_I_AM_DOING_CROSS=1 RUST_CROSS_TARGETS=( - $(aarch64-cros-linux-gnu-gcc --version >/dev/null && echo "AArch64:aarch64-unknown-linux-gnu:aarch64-cros-linux-gnu") + $(use arm64 || { aarch64-cros-linux-gnu-gcc --version &>/dev/null && echo "AArch64:aarch64-unknown-linux-gnu:aarch64-cros-linux-gnu"; }) + $(use amd64 || { x86_64-cros-linux-gnu-gcc --version &>/dev/null && echo "X86:x86_64-unknown-linux-gnu:x86_64-cros-linux-gnu" ; }) ) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use index e69de29bb2..21b5b3ee04 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use @@ -0,0 +1,3 @@ +# Don't build the user space emulator for this arch. It's not needed and gets in +# the way when using Catalyst with QEMU. +app-emulation/qemu -qemu_user_targets_x86_64 diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use new file mode 100644 index 0000000000..c3e3f1eaed --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use @@ -0,0 +1,3 @@ +# Don't build the user space emulator for this arch. It's not needed and gets in +# the way when using Catalyst with QEMU. +app-emulation/qemu -qemu_user_targets_aarch64 diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults index 3fb2f4e9a7..5a6fa0111d 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults @@ -5,12 +5,6 @@ USE="cros_host expat man -pam" # Enable CPU architectures needed by Rust builds LLVM_TARGETS="X86 AArch64" -# Both x86_64 and i386 targets are required for grub testing -QEMU_SOFTMMU_TARGETS="x86_64 i386 aarch64" - -# For cross build support. -QEMU_USER_TARGETS="aarch64" - # add cros_host to bootstrapping USE flags so SDK / toolchains bootstrapping # will use vim's vimrc instead of baselayouts', BOOTSTRAP_USE="$BOOTSTRAP_USE cros_host" diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use index 641b433bda..2108e23b8b 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use @@ -12,7 +12,7 @@ app-crypt/gnupg smartcard usb # for qemu app-arch/bzip2 static-libs -app-emulation/qemu -doc -jpeg ncurses python static-user virtfs qemu_softmmu_targets_x86_64 qemu_softmmu_targets_aarch64 +app-emulation/qemu -doc -jpeg ncurses python static-user virtfs qemu_softmmu_targets_aarch64 qemu_softmmu_targets_x86_64 qemu_user_targets_aarch64 qemu_user_targets_x86_64 dev-libs/glib static-libs dev-libs/libaio static-libs dev-libs/libpcre2 static-libs diff --git a/setup_board b/setup_board index c1acdc1f25..0b97cc7c5b 100755 --- a/setup_board +++ b/setup_board @@ -92,6 +92,13 @@ generate_all_wrappers() { # the board arch matches the SDK arch and therefore emulation is unnecessary. qemu=$(type -P "qemu-${BOARD_CHOST%%-*}") || unset qemu + # If emulation is necessary, then we need to create a placeholder to bind + # mount QEMU onto now. This avoids needing root to do it later. + if [[ -n ${qemu-} ]]; then + sudo mkdir -p "${BOARD_ROOT}${qemu%/*}" + sudo touch "${BOARD_ROOT}${qemu}" + fi + info "Generating wrapper scripts" for wrapper in emerge ebuild eclean equery portageq \ @@ -113,8 +120,20 @@ exec ${BOARD_CHOST}-gdb -iex 'set sysroot ${BOARD_ROOT}' "\$@" EOF wrappers+=( "${wrapper}" ) + # A general purpose wrapper for effectively chrooting using bubblewrap, + # together with emulation by QEMU if necessary. + wrapper="/usr/local/bin/bwrap-${BOARD_VARIANT}" + sudo_clobber "${wrapper}" < "$GCLIENT_ROOT/src/scripts/.default_board" fi