From c7ad59e28ce387b9e32654f0a3229f7f4c6c77f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Tue, 2 Jun 2020 16:09:21 +0200 Subject: [PATCH] build_library: Enhance license info and store it on the image The license JSON file did only include the package names but not any other metadata. Also since the file was not on the image itself, it had to be downloaded. Add more metadata to the license JSON and store it on the image. --- build_library/build_image_util.sh | 129 +++++++++++++++++++++++++--- build_library/dev_container_util.sh | 1 + build_library/ebuild_aci_util.sh | 1 + build_library/oem_aci_util.sh | 1 + build_library/prod_image_util.sh | 1 + 5 files changed, 120 insertions(+), 13 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 3dde0d6ec1..2c4ac38d45 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -273,11 +273,80 @@ write_packages() { image_packages "$1" | sort > "$2" } +# Get metadata $key for package $pkg installed under $prefix +# The metadata is either read from the portage db folder or +# via a portageq-BOARD invocation. In cases where SRC_URI is +# not used for the package, fallback mechanisms are used to find +# the source URL. Mirror names are replaced with the mirror URLs. +get_metadata() { + local prefix="$1" + local pkg="$2" + local key="$3" + local path="${prefix}/var/db/pkg/${pkg%%:*}/${key}" + local val + if [[ -f "$path" ]]; then + val="$(< $path)" + else + # The package is not installed in $prefix or $key not exposed as file, + # so get the value from its ebuild + val=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \ + "${pkg%%:*}" "${key}" 2>/dev/null ||:) + fi + # The value that portageq reports is not a valid URL because it uses a special mirror format. + # Also the value can be empty and fallback methods have to be used. + if [ "${key}" = "SRC_URI" ]; then + local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)" + local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild" + # SRC_URI is empty for the special github.com/flatcar-linux projects + if [ -z "${val}" ]; then + # The grep invocation gives errors when the ebuild file is not present. + # This can happen if a "scripts" branch does not match the "coreos-overlay" branch + # or when the binary packages from ./build_packages are outdated. + val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)" + if [ -n "${val}" ]; then + val="https://github.com/${val}" + # All github.com/flatcar-linux projects specify their commit + local commit="" + commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)" + if [ -n "${commit}" ]; then + val="${val}/commit/${commit}" + fi + fi + fi + # During development "portageq-BOARD metadata ... ebuild ..." may result in the package not being found, fall back to a parameterized URL + if [ -z "${val}" ]; then + # Do not attempt to postprocess by resolving ${P} and friends because it does not affect production images + val="$(cat "${ebuild_path}" | tr '\n' ' ' | grep -P -o 'SRC_URI=".*?"' | cut -d '"' -f 2)" + fi + # Some packages use nothing from the above but EGIT_REPO_URI (currently only app-crypt/go-tspi) + if [ -z "${val}" ]; then + val="$(grep "EGIT_REPO_URI=" "${ebuild_path}" | cut -d '"' -f 2)" + fi + # Replace all mirror://MIRRORNAME/ parts with the actual URL prefix of the mirror + new_val="" + for v in ${val}; do + local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)" + if [ -n "${mirror}" ]; then + # Take only first mirror, those not working should be removed + local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')" + v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")" + fi + new_val+="${v} " + done + val="${new_val}" + fi + echo "${val}" +} + # Generate a list of packages w/ their licenses in the format: # [ # { # "project": "sys-apps/systemd-212-r8::coreos", -# "license": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"] +# "licenses": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"], +# "description": "System and service manager for Linux", +# "homepage": "https://www.freedesktop.org/wiki/Software/systemd", +# "source": "https://github.com/flatcar-linux/systemd ", +# "files": "somefile 63a5736879fa647ac5a8d5317e7cb8b0\nsome -> link\n" # } # ] write_licenses() { @@ -291,19 +360,10 @@ write_licenses() { continue fi - local path="$1/var/db/pkg/${pkg%%:*}/LICENSE" - local lic_str - if [[ -f "$path" ]]; then - lic_str="$(< $path)" - else - # The package is not installed in $1 so get the license from - # its ebuild - lic_str=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \ - "${pkg%%:*}" LICENSE 2>/dev/null ||:) - if [[ -z "$lic_str" ]]; then + local lic_str="$(get_metadata "$1" "${pkg}" LICENSE)" + if [[ -z "$lic_str" ]]; then warn "No license found for ${pkg}" continue - fi fi [[ -n $pkg_sep ]] && echo "," @@ -335,6 +395,14 @@ write_licenses() { # Remove duplicate licenses local lics=$(tr ' ' '\n' <<< "${req_lics[*]} ${opt_lic}" | sort --unique | tr '\n' ' ') + local homepage="$(get_metadata "$1" "${pkg}" HOMEPAGE)" + local description="$(get_metadata "$1" "${pkg}" DESCRIPTION)" + local src_uri="$(get_metadata "$1" "${pkg}" SRC_URI)" + # Filter out directories, cut type marker, cut timestamp, quote "\", and convert line breaks to "\n" + local files="$(get_metadata "$1" "${pkg}" CONTENTS | grep -v '^dir ' | \ + cut -d ' ' -f 2- | rev | cut -d ' ' -f 2- | rev | \ + sed 's#\\#\\\\#g' | tr '\n' '*' | sed 's/*/\\n/g')" + echo -n " {\"project\": \"${pkg}\", \"licenses\": [" local lic_sep="" @@ -345,10 +413,45 @@ write_licenses() { echo -n "\"${lic}\"" done - echo -n "]}" + echo -n "], \"description\": \"${description}\", \"homepage\": \"${homepage}\", \ + \"source\": \"${src_uri}\", \"files\": \"${files}\"}" done >> "$2" echo -e "\n]" >> "$2" + # Pretty print the JSON file + mv "$2" "$2".tmp + jq . "$2".tmp > "$2" + rm "$2".tmp +} + +# Include the license JSON and all licenses in the rootfs with a small INFO file about usage and other URLs. +insert_licenses() { + local json_input="$1" + local root_fs_dir="$2" + sudo mkdir -p "${root_fs_dir}"/usr/share/licenses/common + sudo_clobber "${root_fs_dir}"/usr/share/licenses/INFO << "EOF" +Flatcar Container Linux distributes software from various projects. +The licenses.json.bz2 file contains the list of projects with their licenses, how to obtain the source code, +and which binary files in Flatcar Container Linux were created from it. +You can read it with "less licenses.json.bz2" or convert it to a text format with +bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"' +The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz". +Build system files and patches used to build these projects are located at: +https://github.com/flatcar-linux/coreos-overlay/ +https://github.com/flatcar-linux/portage-stable/ +https://github.com/flatcar-linux/scripts/ +Information on how to build Flatcar Container Linux can be found under: +https://docs.flatcar-linux.org/os/sdk-modifying-flatcar/ +EOF + sudo cp "${json_input}" "${root_fs_dir}"/usr/share/licenses/licenses.json + # Compress the file from 2.1 MB to 0.39 MB + sudo lbzip2 -9 "${root_fs_dir}"/usr/share/licenses/licenses.json + # Copy all needed licenses to a "common" subdirectory and compress them + local license_list # define before assignment because it would mask any error + license_list="$(jq -r '.[] | "/var/gentoo/repos/gentoo/licenses/\(.licenses | .[])"' "${json_input}" | sort | uniq)" + sudo cp ${license_list} "${root_fs_dir}"/usr/share/licenses/common/ + # Compress the licenses just with gzip because there is no big difference as they are single files + sudo gzip -9 "${root_fs_dir}"/usr/share/licenses/common/* } # Add an entry to the image's package.provided diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index b6a1908a1d..201cf7d0a7 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -79,6 +79,7 @@ create_dev_container() { run_localedef "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" # Setup portage for emerge and gmerge configure_dev_portage "${root_fs_dir}" diff --git a/build_library/ebuild_aci_util.sh b/build_library/ebuild_aci_util.sh index 4fc2a5eeff..b25449bc19 100644 --- a/build_library/ebuild_aci_util.sh +++ b/build_library/ebuild_aci_util.sh @@ -27,6 +27,7 @@ create_ebuild_aci_image() { run_ldconfig "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" cleanup_mounts "${root_fs_dir}" trap - EXIT diff --git a/build_library/oem_aci_util.sh b/build_library/oem_aci_util.sh index 90fcd4d700..4f9fd3b343 100644 --- a/build_library/oem_aci_util.sh +++ b/build_library/oem_aci_util.sh @@ -34,6 +34,7 @@ create_oem_aci_image() { run_ldconfig "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 8016c31aaf..509e350aa4 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -82,6 +82,7 @@ create_prod_image() { run_localedef "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" # Assert that if this is supposed to be an official build that the # official update keys have been used.