Merge pull request #76 from flatcar-linux/kai/updates-for-2466.99.1

Updates for Edge 2466.99.1

build_library/build_image_util.sh

@@ -273,11 +273,80 @@ write_packages() {
     image_packages "$1" | sort > "$2"
 }
 
+# Get metadata $key for package $pkg installed under $prefix
+# The metadata is either read from the portage db folder or
+# via a portageq-BOARD invocation. In cases where SRC_URI is
+# not used for the package, fallback mechanisms are used to find
+# the source URL. Mirror names are replaced with the mirror URLs.
+get_metadata() {
+    local prefix="$1"
+    local pkg="$2"
+    local key="$3"
+    local path="${prefix}/var/db/pkg/${pkg%%:*}/${key}"
+    local val
+    if [[ -f "$path" ]]; then
+        val="$(< $path)"
+    else
+        # The package is not installed in $prefix or $key not exposed as file,
+        # so get the value from its ebuild
+        val=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \
+                    "${pkg%%:*}" "${key}" 2>/dev/null ||:)
+    fi
+    # The value that portageq reports is not a valid URL because it uses a special mirror format.
+    # Also the value can be empty and fallback methods have to be used.
+    if [ "${key}" = "SRC_URI" ]; then
+        local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)"
+        local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild"
+        # SRC_URI is empty for the special github.com/flatcar-linux projects
+        if [ -z "${val}" ]; then
+            # The grep invocation gives errors when the ebuild file is not present.
+            # This can happen if a "scripts" branch does not match the "coreos-overlay" branch
+            # or when the binary packages from ./build_packages are outdated.
+            val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)"
+            if [ -n "${val}" ]; then
+                val="https://github.com/${val}"
+                # All github.com/flatcar-linux projects specify their commit
+                local commit=""
+                commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)"
+                if [ -n "${commit}" ]; then
+                    val="${val}/commit/${commit}"
+                fi
+            fi
+        fi
+        # During development "portageq-BOARD metadata ... ebuild ..." may result in the package not being found, fall back to a parameterized URL
+        if [ -z "${val}" ]; then
+            # Do not attempt to postprocess by resolving ${P} and friends because it does not affect production images
+            val="$(cat "${ebuild_path}" | tr '\n' ' ' | grep -P -o 'SRC_URI=".*?"' | cut -d '"' -f 2)"
+        fi
+        # Some packages use nothing from the above but EGIT_REPO_URI (currently only app-crypt/go-tspi)
+        if [ -z "${val}" ]; then
+            val="$(grep "EGIT_REPO_URI=" "${ebuild_path}" | cut -d '"' -f 2)"
+        fi
+        # Replace all mirror://MIRRORNAME/ parts with the actual URL prefix of the mirror
+        new_val=""
+        for v in ${val}; do
+            local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
+            if [ -n "${mirror}" ]; then
+                # Take only first mirror, those not working should be removed
+                local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
+                v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
+            fi
+            new_val+="${v} "
+        done
+        val="${new_val}"
+    fi
+    echo "${val}"
+}
+
 # Generate a list of packages w/ their licenses in the format:
 #   [
 #     {
 #       "project": "sys-apps/systemd-212-r8::coreos",
-#       "license": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"]
+#       "licenses": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"],
+#       "description": "System and service manager for Linux",
+#       "homepage": "https://www.freedesktop.org/wiki/Software/systemd",
+#       "source": "https://github.com/flatcar-linux/systemd ",
+#       "files": "somefile 63a5736879fa647ac5a8d5317e7cb8b0\nsome -> link\n"
 #     }
 #   ]
 write_licenses() {
@@ -291,19 +360,10 @@ write_licenses() {
             continue
         fi
 
-        local path="$1/var/db/pkg/${pkg%%:*}/LICENSE"
-        local lic_str
-        if [[ -f "$path" ]]; then
-            lic_str="$(< $path)"
-        else
-            # The package is not installed in $1 so get the license from
-            # its ebuild
-            lic_str=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \
-                        "${pkg%%:*}" LICENSE 2>/dev/null ||:)
-            if [[ -z "$lic_str" ]]; then
+        local lic_str="$(get_metadata "$1" "${pkg}" LICENSE)"
+        if [[ -z "$lic_str" ]]; then
                 warn "No license found for ${pkg}"
                 continue
-            fi
         fi
 
         [[ -n $pkg_sep ]] && echo ","
@@ -335,6 +395,16 @@ write_licenses() {
         # Remove duplicate licenses
         local lics=$(tr ' ' '\n' <<< "${req_lics[*]} ${opt_lic}" | sort --unique | tr '\n' ' ')
 
+        local homepage="$(get_metadata "$1" "${pkg}" HOMEPAGE)"
+        local description="$(get_metadata "$1" "${pkg}" DESCRIPTION)"
+        local src_uri="$(get_metadata "$1" "${pkg}" SRC_URI)"
+        # Filter out directories, cut type marker, cut timestamp, quote "\", and convert line breaks to "\n"
+        # Filter any unicode characters "rev" doesn't handle (currently some ca-certificates files) and
+        # replace them with a "?" so that the files can still be opened thanks to shell file name expansion
+        local files="$(get_metadata "$1" "${pkg}" CONTENTS | grep -v '^dir ' | \
+            cut -d ' ' -f 2- | tr -c '[[:print:][:cntrl:]]' '?' | rev | cut -d ' ' -f 2- | rev | \
+            sed 's#\\#\\\\#g' | tr '\n' '*' | sed 's/*/\\n/g')"
+
         echo -n "  {\"project\": \"${pkg}\", \"licenses\": ["
 
         local lic_sep=""
@@ -345,10 +415,45 @@ write_licenses() {
             echo -n "\"${lic}\""
         done
 
-        echo -n "]}"
+        echo -n "], \"description\": \"${description}\", \"homepage\": \"${homepage}\", \
+            \"source\": \"${src_uri}\", \"files\": \"${files}\"}"
     done >> "$2"
 
     echo -e "\n]" >> "$2"
+    # Pretty print the JSON file
+    mv "$2" "$2".tmp
+    jq . "$2".tmp > "$2"
+    rm "$2".tmp
+}
+
+# Include the license JSON and all licenses in the rootfs with a small INFO file about usage and other URLs.
+insert_licenses() {
+    local json_input="$1"
+    local root_fs_dir="$2"
+    sudo mkdir -p "${root_fs_dir}"/usr/share/licenses/common
+    sudo_clobber "${root_fs_dir}"/usr/share/licenses/INFO << "EOF"
+Flatcar Container Linux distributes software from various projects.
+The licenses.json.bz2 file contains the list of projects with their licenses, how to obtain the source code,
+and which binary files in Flatcar Container Linux were created from it.
+You can read it with "less licenses.json.bz2" or convert it to a text format with
+bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"'
+The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz".
+Build system files and patches used to build these projects are located at:
+https://github.com/flatcar-linux/coreos-overlay/
+https://github.com/flatcar-linux/portage-stable/
+https://github.com/flatcar-linux/scripts/
+Information on how to build Flatcar Container Linux can be found under:
+https://docs.flatcar-linux.org/os/sdk-modifying-flatcar/
+EOF
+    sudo cp "${json_input}" "${root_fs_dir}"/usr/share/licenses/licenses.json
+    # Compress the file from 2.1 MB to 0.39 MB
+    sudo lbzip2 -9 "${root_fs_dir}"/usr/share/licenses/licenses.json
+    # Copy all needed licenses to a "common" subdirectory and compress them
+    local license_list # define before assignment because it would mask any error
+    license_list="$(jq -r '.[] | "/var/gentoo/repos/gentoo/licenses/\(.licenses | .[])"' "${json_input}" | sort | uniq)"
+    sudo cp ${license_list} "${root_fs_dir}"/usr/share/licenses/common/
+    # Compress the licenses just with gzip because there is no big difference as they are single files
+    sudo gzip -9 "${root_fs_dir}"/usr/share/licenses/common/*
 }
 
 # Add an entry to the image's package.provided

build_library/dev_container_util.sh

@@ -25,8 +25,8 @@ PKGDIR="/var/lib/portage/pkgs"
 PORT_LOGDIR="/var/log/portage"
 PORTDIR="/var/lib/portage/portage-stable"
 PORTDIR_OVERLAY="/var/lib/portage/coreos-overlay"
-PORTAGE_BINHOST="http://builds.developer.core-os.net/boards/${BOARD}/${FLATCAR_VERSION_ID}/pkgs/
-http://builds.developer.core-os.net/boards/${BOARD}/${FLATCAR_VERSION_ID}/toolchain/"
+PORTAGE_BINHOST="https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/pkgs/
+https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/toolchain/"
 EOF
 
 sudo_clobber "$1/etc/portage/repos.conf/coreos.conf" <<EOF
@@ -79,6 +79,7 @@ create_dev_container() {
   run_localedef "${root_fs_dir}"
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
   write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+  insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
   # Setup portage for emerge and gmerge
   configure_dev_portage "${root_fs_dir}"

build_library/ebuild_aci_util.sh

@@ -27,6 +27,7 @@ create_ebuild_aci_image() {
     run_ldconfig "${root_fs_dir}"
     write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
     write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+    insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
     cleanup_mounts "${root_fs_dir}"
     trap - EXIT

build_library/oem_aci_util.sh

@@ -34,6 +34,7 @@ create_oem_aci_image() {
     run_ldconfig "${root_fs_dir}"
     write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
     write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+    insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
     # clean-ups of things we do not need
     sudo rm ${root_fs_dir}/etc/csh.env

build_library/prod_image_util.sh

@@ -82,6 +82,7 @@ create_prod_image() {
   run_localedef "${root_fs_dir}"
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
   write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+  insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
   # Assert that if this is supposed to be an official build that the
   # official update keys have been used.

build_library/toolchain_util.sh

@@ -143,7 +143,7 @@ get_board_binhost() {
     shift
 
     if [[ $# -eq 0 ]]; then
-        set -- "${FLATCAR_SDK_VERSION}" "${FLATCAR_VERSION_ID}"
+        set -- "${FLATCAR_VERSION_ID}"
     fi
 
     for ver in "$@"; do
@@ -172,12 +172,18 @@ get_sdk_libdir() {
 get_sdk_binhost() {
     local arch=$(get_sdk_arch) ver
     if [[ $# -eq 0 ]]; then
-        set -- "${FLATCAR_SDK_VERSION}" "${FLATCAR_VERSION_ID}"
+        set -- "${FLATCAR_SDK_VERSION}"
     fi
 
+    FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-$FLATCAR_DEV_BUILDS/sdk}"
     for ver in "$@"; do
-        echo "${FLATCAR_DEV_BUILDS}/sdk/${arch}/${ver}/pkgs/"
-        echo "${FLATCAR_DEV_BUILDS}/sdk/${arch}/${ver}/toolchain/"
+        # Usually only crossdev needs to be fetched from /toolchain/ in the setup_board step.
+        # The entry for /pkgs/ is there if something needs to be reinstalled in the SDK
+        # but normally it is not needed because everything is already part of the tarball.
+        # To install the crossdev Rust package, /toolchain-arm64/ is derived from /toolchain/
+        # when necessary in install_cross_toolchain().
+        echo "${FLATCAR_DEV_BUILDS_SDK}/${arch}/${ver}/toolchain/"
+        echo "${FLATCAR_DEV_BUILDS_SDK}/${arch}/${ver}/pkgs/"
     done
 }
 
@@ -325,12 +331,12 @@ install_cross_toolchain() {
         $sudo emerge "${emerge_flags[@]}" \
             "cross-${cross_chost}/gdb" "${cross_pkgs[@]}"
         if [ "${cross_chost}" = aarch64-cros-linux-gnu ]; then
-          # Here we need to take only the binary packages from the toolchain builds
+          # Here we need to take only the binary packages from the toolchain-arm64 builds
           # because the standard Rust packages don't include the arm64 cross target.
-          # Building from source is ok because the cross-compiler got installed.
-          FILTERED="$(echo $PORTAGE_BINHOST | tr ' ' '\n' | grep toolchain | xargs echo)"
+          FILTERED="$(echo $PORTAGE_BINHOST | tr ' ' '\n' | grep toolchain | sed 's#toolchain/#toolchain-arm64/#g' | xargs echo)"
           # If no aarch64 folder exists, try to remove any existing Rust packages.
           [ ! -d /usr/lib/rust-*/rustlib/aarch64-unknown-linux-gnu ] && ($sudo emerge -C dev-lang/rust || true)
+          # Building from source is also ok because the cross-compiler got installed.
           $sudo PORTAGE_BINHOST="$FILTERED" emerge "${emerge_flags[@]}" dev-lang/rust
         fi
     fi

build_packages

@@ -18,6 +18,8 @@ DEFINE_boolean usepkg "${FLAGS_TRUE}" \
   "Use binary packages when possible."
 DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \
   "Only use/download binary packages. Implies --noworkon"
+DEFINE_string usepkg_exclude "" \
+  "Exclude these binary packages."
 DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \
   "Download binary packages from remote repository."
 DEFINE_string getbinpkgver "" \
@@ -154,6 +156,9 @@ if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then
   if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
       EMERGE_FLAGS+=( --getbinpkg )
   fi
+  if [[ -n "${FLAGS_usepkg_exclude}"  ]]; then
+    EMERGE_FLAGS+=("--usepkg-exclude=${FLAGS_usepkg_exclude}")
+  fi
 fi
 
 if [[ "${FLAGS_jobs}" -ne -1 ]]; then

build_toolchains

@@ -42,6 +42,14 @@ def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}"
 sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
     "toolchain/" "${BINPKGS}/crossdev"/*
 
+for board in $(get_board_list); do
+    if [ "$board" = arm64-usr ]; then
+        sign_and_upload_files "Rust aarch64 crossdev toolchain packages" "${def_upload_path}" \
+            "toolchain-arm64/" "${BINPKGS}/target/${board}"/Packages "${BINPKGS}/target/${board}"/dev-lang
+    fi
+done
+
+# TODO: Actually just TOOLCHAIN_PKGS and the exact dependencies should be uploaded
 for board in $(get_board_list); do
     board_packages="${BINPKGS}/target/${board}"
     def_upload_path="${UPLOAD_ROOT}/boards/${board}/${FLAGS_version}"

jenkins/images.sh

@@ -12,6 +12,7 @@ enter() {
         sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
         bin/cork enter --bind-gpg-agent=false -- env \
             FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
             {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
 --json-key=/etc/portage/gangue.json $verify_key \
 "'"${URI}" "${DISTDIR}/${FILE}"' \

jenkins/packages.sh

@@ -16,6 +16,7 @@ enter() {
             CCACHE_DIR=/mnt/host/source/ccache \
             CCACHE_MAXSIZE=5G \
             FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+            FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
             {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
 --json-key=/etc/portage/gangue.json $verify_key \
 "'"${URI}" "${DISTDIR}/${FILE}"' \
@@ -44,6 +45,7 @@ script setup_board \
 script build_packages \
     --board="${BOARD}" \
     --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \
+    --usepkg_exclude="${BINARY_PACKAGES_TO_EXCLUDE}" \
     --skip_chroot_upgrade \
     --skip_torcx_store \
     --sign="${SIGNING_USER}" \

jenkins/sdk.sh

@@ -13,7 +13,7 @@ gpg --import "${GPG_SECRET_KEY_FILE}"
 # Wipe all of catalyst.
 sudo rm -rf src/build
 
-enter sudo /mnt/host/source/src/scripts/bootstrap_sdk \
+enter sudo FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" /mnt/host/source/src/scripts/bootstrap_sdk \
     --sign="${SIGNING_USER}" \
     --sign_digests="${SIGNING_USER}" \
     --upload_root="${UPLOAD_ROOT}" \

jenkins/toolchains.sh

@@ -13,7 +13,7 @@ gpg --import "${GPG_SECRET_KEY_FILE}"
 # Wipe all of catalyst.
 sudo rm -rf src/build
 
-enter sudo /mnt/host/source/src/scripts/build_toolchains \
+enter sudo FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" /mnt/host/source/src/scripts/build_toolchains \
     --sign="${SIGNING_USER}" \
     --sign_digests="${SIGNING_USER}" \
     --upload_root="${UPLOAD_ROOT}" \

jenkins/vm.sh

@@ -12,6 +12,7 @@ enter() {
         sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
         bin/cork enter --bind-gpg-agent=false -- env \
             FLATCAR_DEV_BUILDS="${GS_DEVEL_ROOT}" \
+            FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
             {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
 --json-key=/etc/portage/gangue.json $verify_key \
 "'"${URI}" "${DISTDIR}/${FILE}"' \

sdk_lib/sdk_util.sh

@@ -11,7 +11,8 @@ FLATCAR_SDK_ARCH="amd64" # We are unlikely to support anything else.
 FLATCAR_SDK_TARBALL="flatcar-sdk-${FLATCAR_SDK_ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2"
 FLATCAR_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks"
 FLATCAR_SDK_TARBALL_PATH="${FLATCAR_SDK_TARBALL_CACHE}/${FLATCAR_SDK_TARBALL}"
-FLATCAR_SDK_URL="${FLATCAR_DEV_BUILDS}/sdk/${FLATCAR_SDK_ARCH}/${FLATCAR_SDK_VERSION}/${FLATCAR_SDK_TARBALL}"
+FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-$FLATCAR_DEV_BUILDS/sdk}"
+FLATCAR_SDK_URL="${FLATCAR_DEV_BUILDS_SDK}/${FLATCAR_SDK_ARCH}/${FLATCAR_SDK_VERSION}/${FLATCAR_SDK_TARBALL}"
 
 # Download the current SDK tarball (if required) and verify digests/sig
 sdk_download_tarball() {

update_chroot

@@ -27,6 +27,8 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
   "Don't update the toolchains."
 DEFINE_string toolchain_boards "" \
   "Extra toolchains to setup for the specified boards."
+DEFINE_string dev_builds_sdk "" \
+  "Set FLATCAR_DEV_BUILDS_SDK which defaults to FLATCAR_DEV_BUILDS/sdk"
 DEFINE_string binhost "" \
   "Use binary packages from a specific location (e.g. https://storage.googleapis.com/flatcar-jenkins/sdk/amd64/2000.0.0/pkgs)"
 
@@ -53,6 +55,10 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
   FLAGS_workon="${FLAGS_FALSE}"
 fi
 
+if [[ -n "${FLAGS_dev_builds_sdk}" ]]; then
+  FLATCAR_DEV_BUILDS_SDK="${FLAGS_dev_builds_sdk}"
+fi
+
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh"
 
 PORTAGE_STABLE_OVERLAY="${REPO_ROOT}/src/third_party/portage-stable"