diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 3dde0d6ec1..2c4ac38d45 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -273,11 +273,80 @@ write_packages() { image_packages "$1" | sort > "$2" } +# Get metadata $key for package $pkg installed under $prefix +# The metadata is either read from the portage db folder or +# via a portageq-BOARD invocation. In cases where SRC_URI is +# not used for the package, fallback mechanisms are used to find +# the source URL. Mirror names are replaced with the mirror URLs. +get_metadata() { + local prefix="$1" + local pkg="$2" + local key="$3" + local path="${prefix}/var/db/pkg/${pkg%%:*}/${key}" + local val + if [[ -f "$path" ]]; then + val="$(< $path)" + else + # The package is not installed in $prefix or $key not exposed as file, + # so get the value from its ebuild + val=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \ + "${pkg%%:*}" "${key}" 2>/dev/null ||:) + fi + # The value that portageq reports is not a valid URL because it uses a special mirror format. + # Also the value can be empty and fallback methods have to be used. + if [ "${key}" = "SRC_URI" ]; then + local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)" + local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild" + # SRC_URI is empty for the special github.com/flatcar-linux projects + if [ -z "${val}" ]; then + # The grep invocation gives errors when the ebuild file is not present. + # This can happen if a "scripts" branch does not match the "coreos-overlay" branch + # or when the binary packages from ./build_packages are outdated. + val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)" + if [ -n "${val}" ]; then + val="https://github.com/${val}" + # All github.com/flatcar-linux projects specify their commit + local commit="" + commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)" + if [ -n "${commit}" ]; then + val="${val}/commit/${commit}" + fi + fi + fi + # During development "portageq-BOARD metadata ... ebuild ..." may result in the package not being found, fall back to a parameterized URL + if [ -z "${val}" ]; then + # Do not attempt to postprocess by resolving ${P} and friends because it does not affect production images + val="$(cat "${ebuild_path}" | tr '\n' ' ' | grep -P -o 'SRC_URI=".*?"' | cut -d '"' -f 2)" + fi + # Some packages use nothing from the above but EGIT_REPO_URI (currently only app-crypt/go-tspi) + if [ -z "${val}" ]; then + val="$(grep "EGIT_REPO_URI=" "${ebuild_path}" | cut -d '"' -f 2)" + fi + # Replace all mirror://MIRRORNAME/ parts with the actual URL prefix of the mirror + new_val="" + for v in ${val}; do + local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)" + if [ -n "${mirror}" ]; then + # Take only first mirror, those not working should be removed + local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')" + v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")" + fi + new_val+="${v} " + done + val="${new_val}" + fi + echo "${val}" +} + # Generate a list of packages w/ their licenses in the format: # [ # { # "project": "sys-apps/systemd-212-r8::coreos", -# "license": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"] +# "licenses": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"], +# "description": "System and service manager for Linux", +# "homepage": "https://www.freedesktop.org/wiki/Software/systemd", +# "source": "https://github.com/flatcar-linux/systemd ", +# "files": "somefile 63a5736879fa647ac5a8d5317e7cb8b0\nsome -> link\n" # } # ] write_licenses() { @@ -291,19 +360,10 @@ write_licenses() { continue fi - local path="$1/var/db/pkg/${pkg%%:*}/LICENSE" - local lic_str - if [[ -f "$path" ]]; then - lic_str="$(< $path)" - else - # The package is not installed in $1 so get the license from - # its ebuild - lic_str=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \ - "${pkg%%:*}" LICENSE 2>/dev/null ||:) - if [[ -z "$lic_str" ]]; then + local lic_str="$(get_metadata "$1" "${pkg}" LICENSE)" + if [[ -z "$lic_str" ]]; then warn "No license found for ${pkg}" continue - fi fi [[ -n $pkg_sep ]] && echo "," @@ -335,6 +395,14 @@ write_licenses() { # Remove duplicate licenses local lics=$(tr ' ' '\n' <<< "${req_lics[*]} ${opt_lic}" | sort --unique | tr '\n' ' ') + local homepage="$(get_metadata "$1" "${pkg}" HOMEPAGE)" + local description="$(get_metadata "$1" "${pkg}" DESCRIPTION)" + local src_uri="$(get_metadata "$1" "${pkg}" SRC_URI)" + # Filter out directories, cut type marker, cut timestamp, quote "\", and convert line breaks to "\n" + local files="$(get_metadata "$1" "${pkg}" CONTENTS | grep -v '^dir ' | \ + cut -d ' ' -f 2- | rev | cut -d ' ' -f 2- | rev | \ + sed 's#\\#\\\\#g' | tr '\n' '*' | sed 's/*/\\n/g')" + echo -n " {\"project\": \"${pkg}\", \"licenses\": [" local lic_sep="" @@ -345,10 +413,45 @@ write_licenses() { echo -n "\"${lic}\"" done - echo -n "]}" + echo -n "], \"description\": \"${description}\", \"homepage\": \"${homepage}\", \ + \"source\": \"${src_uri}\", \"files\": \"${files}\"}" done >> "$2" echo -e "\n]" >> "$2" + # Pretty print the JSON file + mv "$2" "$2".tmp + jq . "$2".tmp > "$2" + rm "$2".tmp +} + +# Include the license JSON and all licenses in the rootfs with a small INFO file about usage and other URLs. +insert_licenses() { + local json_input="$1" + local root_fs_dir="$2" + sudo mkdir -p "${root_fs_dir}"/usr/share/licenses/common + sudo_clobber "${root_fs_dir}"/usr/share/licenses/INFO << "EOF" +Flatcar Container Linux distributes software from various projects. +The licenses.json.bz2 file contains the list of projects with their licenses, how to obtain the source code, +and which binary files in Flatcar Container Linux were created from it. +You can read it with "less licenses.json.bz2" or convert it to a text format with +bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"' +The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz". +Build system files and patches used to build these projects are located at: +https://github.com/flatcar-linux/coreos-overlay/ +https://github.com/flatcar-linux/portage-stable/ +https://github.com/flatcar-linux/scripts/ +Information on how to build Flatcar Container Linux can be found under: +https://docs.flatcar-linux.org/os/sdk-modifying-flatcar/ +EOF + sudo cp "${json_input}" "${root_fs_dir}"/usr/share/licenses/licenses.json + # Compress the file from 2.1 MB to 0.39 MB + sudo lbzip2 -9 "${root_fs_dir}"/usr/share/licenses/licenses.json + # Copy all needed licenses to a "common" subdirectory and compress them + local license_list # define before assignment because it would mask any error + license_list="$(jq -r '.[] | "/var/gentoo/repos/gentoo/licenses/\(.licenses | .[])"' "${json_input}" | sort | uniq)" + sudo cp ${license_list} "${root_fs_dir}"/usr/share/licenses/common/ + # Compress the licenses just with gzip because there is no big difference as they are single files + sudo gzip -9 "${root_fs_dir}"/usr/share/licenses/common/* } # Add an entry to the image's package.provided diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index b6a1908a1d..201cf7d0a7 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -79,6 +79,7 @@ create_dev_container() { run_localedef "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" # Setup portage for emerge and gmerge configure_dev_portage "${root_fs_dir}" diff --git a/build_library/ebuild_aci_util.sh b/build_library/ebuild_aci_util.sh index 4fc2a5eeff..b25449bc19 100644 --- a/build_library/ebuild_aci_util.sh +++ b/build_library/ebuild_aci_util.sh @@ -27,6 +27,7 @@ create_ebuild_aci_image() { run_ldconfig "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" cleanup_mounts "${root_fs_dir}" trap - EXIT diff --git a/build_library/oem_aci_util.sh b/build_library/oem_aci_util.sh index 90fcd4d700..4f9fd3b343 100644 --- a/build_library/oem_aci_util.sh +++ b/build_library/oem_aci_util.sh @@ -34,6 +34,7 @@ create_oem_aci_image() { run_ldconfig "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 8016c31aaf..509e350aa4 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -82,6 +82,7 @@ create_prod_image() { run_localedef "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" # Assert that if this is supposed to be an official build that the # official update keys have been used.