diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh
index 286735d2cc..278e7664b1 100644
--- a/build_library/toolchain_util.sh
+++ b/build_library/toolchain_util.sh
@@ -399,6 +399,7 @@ install_cross_libs() {
 install_cross_rust() {
     local cross_chost="$1"; shift
     local emerge_flags=( "$@" --binpkg-respect-use=y --update )
+    local cbuild="$(portageq envvar CBUILD)"
 
     # may be called from either catalyst (root) or upgrade_chroot (user)
     local sudo="env"
@@ -406,10 +407,10 @@ install_cross_rust() {
         sudo="sudo -E"
     fi
 
-    if [ "${cross_chost}" = "aarch64-cros-linux-gnu" ]; then
+    if [ "${cbuild}" = "x86_64-pc-linux-gnu" ] && [ "${cross_chost}" = "aarch64-cros-linux-gnu" ]; then
         echo "Building Rust for arm64"
         # If no aarch64 folder exists, try to remove any existing Rust packages.
-        [ ! -d /usr/lib/rust-*/rustlib/aarch64-unknown-linux-gnu ] && ($sudo emerge -C dev-lang/rust || true)
+        [ ! -d /usr/lib/rustlib/aarch64-unknown-linux-gnu ] && ($sudo emerge -C dev-lang/rust || true)
         $sudo emerge "${emerge_flags[@]}" dev-lang/rust
     fi
 }