Merge pull request #70 from flatcar-linux/kai/write-license

build_library: Enhance license info and store it on the image

build_library/build_image_util.sh

@@ -273,11 +273,80 @@ write_packages() {
     image_packages "$1" | sort > "$2"
 }
 
+# Get metadata $key for package $pkg installed under $prefix
+# The metadata is either read from the portage db folder or
+# via a portageq-BOARD invocation. In cases where SRC_URI is
+# not used for the package, fallback mechanisms are used to find
+# the source URL. Mirror names are replaced with the mirror URLs.
+get_metadata() {
+    local prefix="$1"
+    local pkg="$2"
+    local key="$3"
+    local path="${prefix}/var/db/pkg/${pkg%%:*}/${key}"
+    local val
+    if [[ -f "$path" ]]; then
+        val="$(< $path)"
+    else
+        # The package is not installed in $prefix or $key not exposed as file,
+        # so get the value from its ebuild
+        val=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \
+                    "${pkg%%:*}" "${key}" 2>/dev/null ||:)
+    fi
+    # The value that portageq reports is not a valid URL because it uses a special mirror format.
+    # Also the value can be empty and fallback methods have to be used.
+    if [ "${key}" = "SRC_URI" ]; then
+        local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)"
+        local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild"
+        # SRC_URI is empty for the special github.com/flatcar-linux projects
+        if [ -z "${val}" ]; then
+            # The grep invocation gives errors when the ebuild file is not present.
+            # This can happen if a "scripts" branch does not match the "coreos-overlay" branch
+            # or when the binary packages from ./build_packages are outdated.
+            val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)"
+            if [ -n "${val}" ]; then
+                val="https://github.com/${val}"
+                # All github.com/flatcar-linux projects specify their commit
+                local commit=""
+                commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)"
+                if [ -n "${commit}" ]; then
+                    val="${val}/commit/${commit}"
+                fi
+            fi
+        fi
+        # During development "portageq-BOARD metadata ... ebuild ..." may result in the package not being found, fall back to a parameterized URL
+        if [ -z "${val}" ]; then
+            # Do not attempt to postprocess by resolving ${P} and friends because it does not affect production images
+            val="$(cat "${ebuild_path}" | tr '\n' ' ' | grep -P -o 'SRC_URI=".*?"' | cut -d '"' -f 2)"
+        fi
+        # Some packages use nothing from the above but EGIT_REPO_URI (currently only app-crypt/go-tspi)
+        if [ -z "${val}" ]; then
+            val="$(grep "EGIT_REPO_URI=" "${ebuild_path}" | cut -d '"' -f 2)"
+        fi
+        # Replace all mirror://MIRRORNAME/ parts with the actual URL prefix of the mirror
+        new_val=""
+        for v in ${val}; do
+            local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
+            if [ -n "${mirror}" ]; then
+                # Take only first mirror, those not working should be removed
+                local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
+                v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
+            fi
+            new_val+="${v} "
+        done
+        val="${new_val}"
+    fi
+    echo "${val}"
+}
+
 # Generate a list of packages w/ their licenses in the format:
 #   [
 #     {
 #       "project": "sys-apps/systemd-212-r8::coreos",
-#       "license": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"]
+#       "licenses": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"],
+#       "description": "System and service manager for Linux",
+#       "homepage": "https://www.freedesktop.org/wiki/Software/systemd",
+#       "source": "https://github.com/flatcar-linux/systemd ",
+#       "files": "somefile 63a5736879fa647ac5a8d5317e7cb8b0\nsome -> link\n"
 #     }
 #   ]
 write_licenses() {
@@ -291,19 +360,10 @@ write_licenses() {
             continue
         fi
 
-        local path="$1/var/db/pkg/${pkg%%:*}/LICENSE"
-        local lic_str
-        if [[ -f "$path" ]]; then
-            lic_str="$(< $path)"
-        else
-            # The package is not installed in $1 so get the license from
-            # its ebuild
-            lic_str=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \
-                        "${pkg%%:*}" LICENSE 2>/dev/null ||:)
-            if [[ -z "$lic_str" ]]; then
+        local lic_str="$(get_metadata "$1" "${pkg}" LICENSE)"
+        if [[ -z "$lic_str" ]]; then
                 warn "No license found for ${pkg}"
                 continue
-            fi
         fi
 
         [[ -n $pkg_sep ]] && echo ","
@@ -335,6 +395,14 @@ write_licenses() {
         # Remove duplicate licenses
         local lics=$(tr ' ' '\n' <<< "${req_lics[*]} ${opt_lic}" | sort --unique | tr '\n' ' ')
 
+        local homepage="$(get_metadata "$1" "${pkg}" HOMEPAGE)"
+        local description="$(get_metadata "$1" "${pkg}" DESCRIPTION)"
+        local src_uri="$(get_metadata "$1" "${pkg}" SRC_URI)"
+        # Filter out directories, cut type marker, cut timestamp, quote "\", and convert line breaks to "\n"
+        local files="$(get_metadata "$1" "${pkg}" CONTENTS | grep -v '^dir ' | \
+            cut -d ' ' -f 2- | rev | cut -d ' ' -f 2- | rev | \
+            sed 's#\\#\\\\#g' | tr '\n' '*' | sed 's/*/\\n/g')"
+
         echo -n "  {\"project\": \"${pkg}\", \"licenses\": ["
 
         local lic_sep=""
@@ -345,10 +413,45 @@ write_licenses() {
             echo -n "\"${lic}\""
         done
 
-        echo -n "]}"
+        echo -n "], \"description\": \"${description}\", \"homepage\": \"${homepage}\", \
+            \"source\": \"${src_uri}\", \"files\": \"${files}\"}"
     done >> "$2"
 
     echo -e "\n]" >> "$2"
+    # Pretty print the JSON file
+    mv "$2" "$2".tmp
+    jq . "$2".tmp > "$2"
+    rm "$2".tmp
+}
+
+# Include the license JSON and all licenses in the rootfs with a small INFO file about usage and other URLs.
+insert_licenses() {
+    local json_input="$1"
+    local root_fs_dir="$2"
+    sudo mkdir -p "${root_fs_dir}"/usr/share/licenses/common
+    sudo_clobber "${root_fs_dir}"/usr/share/licenses/INFO << "EOF"
+Flatcar Container Linux distributes software from various projects.
+The licenses.json.bz2 file contains the list of projects with their licenses, how to obtain the source code,
+and which binary files in Flatcar Container Linux were created from it.
+You can read it with "less licenses.json.bz2" or convert it to a text format with
+bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"'
+The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz".
+Build system files and patches used to build these projects are located at:
+https://github.com/flatcar-linux/coreos-overlay/
+https://github.com/flatcar-linux/portage-stable/
+https://github.com/flatcar-linux/scripts/
+Information on how to build Flatcar Container Linux can be found under:
+https://docs.flatcar-linux.org/os/sdk-modifying-flatcar/
+EOF
+    sudo cp "${json_input}" "${root_fs_dir}"/usr/share/licenses/licenses.json
+    # Compress the file from 2.1 MB to 0.39 MB
+    sudo lbzip2 -9 "${root_fs_dir}"/usr/share/licenses/licenses.json
+    # Copy all needed licenses to a "common" subdirectory and compress them
+    local license_list # define before assignment because it would mask any error
+    license_list="$(jq -r '.[] | "/var/gentoo/repos/gentoo/licenses/\(.licenses | .[])"' "${json_input}" | sort | uniq)"
+    sudo cp ${license_list} "${root_fs_dir}"/usr/share/licenses/common/
+    # Compress the licenses just with gzip because there is no big difference as they are single files
+    sudo gzip -9 "${root_fs_dir}"/usr/share/licenses/common/*
 }
 
 # Add an entry to the image's package.provided

build_library/dev_container_util.sh

@@ -79,6 +79,7 @@ create_dev_container() {
   run_localedef "${root_fs_dir}"
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
   write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+  insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
   # Setup portage for emerge and gmerge
   configure_dev_portage "${root_fs_dir}"

build_library/ebuild_aci_util.sh

@@ -27,6 +27,7 @@ create_ebuild_aci_image() {
     run_ldconfig "${root_fs_dir}"
     write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
     write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+    insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
     cleanup_mounts "${root_fs_dir}"
     trap - EXIT

build_library/oem_aci_util.sh

@@ -34,6 +34,7 @@ create_oem_aci_image() {
     run_ldconfig "${root_fs_dir}"
     write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
     write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+    insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
     # clean-ups of things we do not need
     sudo rm ${root_fs_dir}/etc/csh.env

build_library/prod_image_util.sh

@@ -82,6 +82,7 @@ create_prod_image() {
   run_localedef "${root_fs_dir}"
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
   write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
+  insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
 
   # Assert that if this is supposed to be an official build that the
   # official update keys have been used.