Support the SDK on arm64

Catalyst 4 adds support for building with QEMU, so I initially leveraged
this, but it turned out to be very much slower than emulating the amd64
SDK on arm64, where an arm64 build could then be mostly run without
emulation. I have kept the code for the slower approach anyway since it
is small and may be useful to somebody.

There were several places where we assumed that amd64 was native and
arm64 required emulation via QEMU. The scripts are now more
architecture-agnostic, paving the way for riscv support later.

We no longer set QEMU_LD_PREFIX because it prevents the SDK itself from
being emulated. It also assumes there is only one non-native target,
which may not always be the case. bubblewrap does a better job of
running binaries under QEMU.

Signed-off-by: James Le Cuirot <jlecuirot@microsoft.com>

build_library/build_image_util.sh

@@ -102,32 +102,13 @@ zip_update_tools() {
     --arch "$(get_sdk_arch)" --output-dir "${BUILD_DIR}" --zip-name "${update_zip}"
 }
 
-# ldconfig cannot generate caches for non-native arches.
-# Use qemu & the native ldconfig to work around that.
-# http://code.google.com/p/chromium/issues/detail?id=378377
 run_ldconfig() {
-  local root_fs_dir=$1
-  case ${ARCH} in
-  arm64)
-    sudo qemu-aarch64 "${root_fs_dir}"/usr/sbin/ldconfig -r "${root_fs_dir}";;
-  x86|amd64)
-    sudo ldconfig -r "${root_fs_dir}";;
-  *)
-    die "Unable to run ldconfig for ARCH ${ARCH}"
-  esac
+  # This wrapper is created by setup_board.
+  sudo "ldconfig-${BOARD}" -r "$1"
 }
 
 run_localedef() {
-  local root_fs_dir="$1" loader=()
-  case ${ARCH} in
-  arm64)
-    loader=( qemu-aarch64 -L "${root_fs_dir}" );;
-  amd64)
-    loader=( "${root_fs_dir}/usr/lib64/ld-linux-x86-64.so.2" \
-               --library-path "${root_fs_dir}/usr/lib64" );;
-  *)
-    die "Unable to run localedef for ARCH ${ARCH}";;
-  esac
+  local root_fs_dir="$1"
   info "Generating C.UTF-8 locale..."
   local i18n="${root_fs_dir}/usr/share/i18n"
   # localedef will silently fall back to /usr/share/i18n if missing so
@@ -135,8 +116,8 @@ run_localedef() {
   [[ -f "${i18n}/charmaps/UTF-8.gz" ]] || die
   [[ -f "${i18n}/locales/C" ]] || die
   sudo mkdir -p "${root_fs_dir}/usr/lib/locale"
-  sudo I18NPATH="${i18n}" "${loader[@]}" "${root_fs_dir}/usr/bin/localedef" \
-      --prefix="${root_fs_dir}" --charmap=UTF-8 --inputfile=C C.UTF-8
+  sudo I18NPATH="${i18n}" "bwrap-${BOARD}" "${root_fs_dir}" /usr/bin/localedef \
+      --charmap=UTF-8 --inputfile=C C.UTF-8
 }
 
 # Basic command to emerge binary packages into the target image.

build_library/catalyst.sh

@@ -25,6 +25,7 @@ BINPKGS=
 DISTDIR=
 TEMPDIR=
 STAGES=
+unset QEMU
 
 DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \
     "Path to directory for all catalyst images and other files."
@@ -97,6 +98,7 @@ cflags: -O2 -pipe
 cxxflags: -O2 -pipe
 ldflags: -Wl,-O2 -Wl,--as-needed
 source_subpath: ${SEED}
+${QEMU+interpreter: $(type -P "${QEMU}")}
 EOF
 }
 
@@ -207,6 +209,16 @@ catalyst_init() {
         SEED="seed/${FLAGS_seed_tarball##*/}"
         SEED="${SEED%.tar.*}"
     fi
+
+    # Emulate the build, if needed. Note the SDK itself may already be emulated,
+    # so check the requested arch against the kernel's real arch, not uname -m.
+    if [[ ${ARCH} != $(get_portage_arch "$(< /proc/sys/kernel/arch)") ]]; then
+        case "${ARCH}" in
+            amd64) QEMU=qemu-x86_64 ;;
+            arm64) QEMU=qemu-aarch64 ;;
+            riscv) QEMU=qemu-riscv64 ;;
+        esac
+    fi
 }
 
 write_configs() {
@@ -226,6 +238,9 @@ write_configs() {
 
     ln -sfT '/mnt/host/source/src/third_party/coreos-overlay/coreos/user-patches' \
         "${TEMPDIR}"/portage/patches
+
+    [[ -n ${QEMU} ]] ||
+        rm "${TEMPDIR}"/portage/package.env/qemu
 }
 
 build_stage() {

build_library/portage/env/releng/qemu

@@ -0,0 +1 @@
+FEATURES="-pid-sandbox -network-sandbox -ipc-sandbox"

build_library/portage/package.env/qemu

@@ -0,0 +1 @@
+*/* releng/qemu

common.sh

@@ -51,9 +51,6 @@ fi
 # Turn on bash debug support if available for backtraces.
 shopt -s extdebug 2>/dev/null
 
-# Source qemu library path
-. /etc/profile.d/qemu-aarch64.sh 2> /dev/null || true
-
 # Output a backtrace all the way back to the raw invocation, suppressing
 # only the _dump_trace frame itself.
 _dump_trace() {
@@ -992,38 +989,3 @@ BOAT
   echo -e "${V_VIDOFF}"
   die "$* failed"
 }
-
-# The binfmt_misc support in the kernel is required.
-# The aarch64 binaries should be executed through
-# "/usr/bin/qemu-aarch64-static"
-setup_qemu_static() {
-  local root_fs_dir="$1"
-  case "${BOARD}" in
-    amd64-usr) return 0;;
-    arm64-usr)
-      if [[ -f "${root_fs_dir}/sbin/ldconfig" ]]; then
-        sudo cp /usr/bin/qemu-aarch64 "${root_fs_dir}"/usr/bin/qemu-aarch64-static
-        echo export QEMU_LD_PREFIX=\"/build/arm64-usr/\" | sudo tee /etc/profile.d/qemu-aarch64.sh
-        . /etc/profile.d/qemu-aarch64.sh
-      else
-        die "Missing basic layout in target rootfs"
-      fi
-    ;;
-    *) die "Unsupported arch" ;;
-  esac
-}
-
-clean_qemu_static() {
-  local root_fs_dir="$1"
-  case "${BOARD}" in
-    amd64-usr) return 0;;
-    arm64-usr)
-      if [[ -f "${root_fs_dir}/usr/bin/qemu-aarch64-static" ]]; then
-        sudo rm "${root_fs_dir}"/usr/bin/qemu-aarch64-static
-      else
-        die "File not found"
-      fi
-    ;;
-    *) die "Unsupported arch" ;;
-  esac
-}

sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-lang/rust

@@ -2,5 +2,6 @@ INSTALL_MASK+=" *rustdoc*"
 I_KNOW_WHAT_I_AM_DOING_CROSS=1
 
 RUST_CROSS_TARGETS=(
-	$(aarch64-cros-linux-gnu-gcc --version >/dev/null && echo "AArch64:aarch64-unknown-linux-gnu:aarch64-cros-linux-gnu")
+	$(use arm64 || { aarch64-cros-linux-gnu-gcc --version &>/dev/null && echo "AArch64:aarch64-unknown-linux-gnu:aarch64-cros-linux-gnu"; })
+	$(use amd64 || { x86_64-cros-linux-gnu-gcc  --version &>/dev/null && echo "X86:x86_64-unknown-linux-gnu:x86_64-cros-linux-gnu"      ; })
 )

sdk_container/src/third_party/coreos-overlay/profiles/coreos/amd64/sdk/package.use

@@ -0,0 +1,3 @@
+# Don't build the user space emulator for this arch. It's not needed and gets in
+# the way when using Catalyst with QEMU.
+app-emulation/qemu -qemu_user_targets_x86_64

sdk_container/src/third_party/coreos-overlay/profiles/coreos/arm64/sdk/package.use

@@ -0,0 +1,3 @@
+# Don't build the user space emulator for this arch. It's not needed and gets in
+# the way when using Catalyst with QEMU.
+app-emulation/qemu -qemu_user_targets_aarch64

sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/make.defaults

@@ -5,12 +5,6 @@ USE="cros_host expat man -pam"
 # Enable CPU architectures needed by Rust builds
 LLVM_TARGETS="X86 AArch64"
 
-# Both x86_64 and i386 targets are required for grub testing
-QEMU_SOFTMMU_TARGETS="x86_64 i386 aarch64"
-
-# For cross build support.
-QEMU_USER_TARGETS="aarch64"
-
 # add cros_host to bootstrapping USE flags so SDK / toolchains bootstrapping
 # will use vim's vimrc instead of baselayouts',
 BOOTSTRAP_USE="$BOOTSTRAP_USE cros_host"

sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use

@@ -12,7 +12,7 @@ app-crypt/gnupg smartcard usb
 
 # for qemu
 app-arch/bzip2 static-libs
-app-emulation/qemu -doc -jpeg ncurses python static-user virtfs qemu_softmmu_targets_x86_64 qemu_softmmu_targets_aarch64
+app-emulation/qemu -doc -jpeg ncurses python static-user virtfs qemu_softmmu_targets_aarch64 qemu_softmmu_targets_x86_64 qemu_user_targets_aarch64 qemu_user_targets_x86_64
 dev-libs/glib static-libs
 dev-libs/libaio static-libs
 dev-libs/libpcre2 static-libs

setup_board

@@ -92,6 +92,13 @@ generate_all_wrappers() {
   # the board arch matches the SDK arch and therefore emulation is unnecessary.
   qemu=$(type -P "qemu-${BOARD_CHOST%%-*}") || unset qemu
 
+  # If emulation is necessary, then we need to create a placeholder to bind
+  # mount QEMU onto now. This avoids needing root to do it later.
+  if [[ -n ${qemu-} ]]; then
+    sudo mkdir -p "${BOARD_ROOT}${qemu%/*}"
+    sudo touch "${BOARD_ROOT}${qemu}"
+  fi
+
   info "Generating wrapper scripts"
 
   for wrapper in emerge ebuild eclean equery portageq \
@@ -113,8 +120,20 @@ exec ${BOARD_CHOST}-gdb -iex 'set sysroot ${BOARD_ROOT}' "\$@"
 EOF
   wrappers+=( "${wrapper}" )
 
+  # A general purpose wrapper for effectively chrooting using bubblewrap,
+  # together with emulation by QEMU if necessary.
+  wrapper="/usr/local/bin/bwrap-${BOARD_VARIANT}"
+  sudo_clobber "${wrapper}" <<EOF
+#!/bin/sh -e
+_ROOT=\${1?Pass a root directory as the first argument}
+shift
+exec bwrap --bind "\${_ROOT}" / --tmpfs /tmp --dev-bind /dev /dev --proc /proc --bind /sys /sys ${qemu+--ro-bind ${qemu} ${qemu}} -- "\$@"
+EOF
+  wrappers+=( "${wrapper}" )
+
   # ldconfig cannot generate caches for non-native arches. Use QEMU and the
-  # native ldconfig to work around that.
+  # native ldconfig to work around that. Don't use the bwrap wrapper above
+  # because it's unnecessarily complex for this use case.
   wrapper="/usr/local/sbin/ldconfig-${BOARD_VARIANT}"
   sudo_clobber "${wrapper}" <<EOF
 #!/bin/sh
@@ -354,11 +373,6 @@ if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
       "${EMERGE_TOOLCHAIN_FLAGS[@]}" "${TOOLCHAIN_PKGS[@]}"
 fi
 
-if [[ ${FLAGS_regen_configs_only} -eq ${FLAGS_FALSE} ]]; then
-  # Setup BOARD_ROOT for QEMU user emulation.
-  setup_qemu_static "${BOARD_ROOT}"
-fi
-
 if [ $FLAGS_default -eq $FLAGS_TRUE ] ; then
   echo $BOARD_VARIANT > "$GCLIENT_ROOT/src/scripts/.default_board"
 fi