mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-12-15 22:31:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

build_library: Enhance license info and store it on the image

Browse Source 
The license JSON file did only include the package names but not
any other metadata. Also since the file was not on the image itself,
it had to be downloaded.
Add more metadata to the license JSON and store it on the image.
This commit is contained in:
Kai Lüke 2020-06-02 16:09:21 +02:00
parent c666ae5614
commit c7ad59e28c
No known key found for this signature in database
GPG Key ID: E5601DA3A1D902A8
5 changed files with 120 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
build_library/build_image_util.sh
View File

@ -273,11 +273,80 @@ write_packages() {
image_packages "$1" | sort > "$2"
}
# Get metadata $key for package $pkg installed under $prefix
# The metadata is either read from the portage db folder or
# via a portageq-BOARD invocation. In cases where SRC_URI is
# not used for the package, fallback mechanisms are used to find
# the source URL. Mirror names are replaced with the mirror URLs.
get_metadata() {
local prefix="$1"
local pkg="$2"
local key="$3"
local path="${prefix}/var/db/pkg/${pkg%%:*}/${key}"
local val
if [[ -f "$path" ]]; then
val="$(< $path)"
else
# The package is not installed in $prefix or $key not exposed as file,
# so get the value from its ebuild
val=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \
"${pkg%%:*}" "${key}" 2>/dev/null ||:)
fi
# The value that portageq reports is not a valid URL because it uses a special mirror format.
# Also the value can be empty and fallback methods have to be used.
if [ "${key}" = "SRC_URI" ]; then
local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)"
local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild"
# SRC_URI is empty for the special github.com/flatcar-linux projects
if [ -z "${val}" ]; then
# The grep invocation gives errors when the ebuild file is not present.
# This can happen if a "scripts" branch does not match the "coreos-overlay" branch
# or when the binary packages from ./build_packages are outdated.
val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)"
if [ -n "${val}" ]; then
val="https://github.com/${val}"
# All github.com/flatcar-linux projects specify their commit
local commit=""
commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)"
if [ -n "${commit}" ]; then
val="${val}/commit/${commit}"
fi
fi
fi
# During development "portageq-BOARD metadata ... ebuild ..." may result in the package not being found, fall back to a parameterized URL
if [ -z "${val}" ]; then
# Do not attempt to postprocess by resolving ${P} and friends because it does not affect production images
val="$(cat "${ebuild_path}" | tr '\n' ' ' | grep -P -o 'SRC_URI=".*?"' | cut -d '"' -f 2)"
fi
# Some packages use nothing from the above but EGIT_REPO_URI (currently only app-crypt/go-tspi)
if [ -z "${val}" ]; then
val="$(grep "EGIT_REPO_URI=" "${ebuild_path}" | cut -d '"' -f 2)"
fi
# Replace all mirror://MIRRORNAME/ parts with the actual URL prefix of the mirror
new_val=""
for v in ${val}; do
local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
if [ -n "${mirror}" ]; then
# Take only first mirror, those not working should be removed
local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
fi
new_val+="${v} "
done
val="${new_val}"
fi
echo "${val}"
}
# Generate a list of packages w/ their licenses in the format:
# [
# {
# "project": "sys-apps/systemd-212-r8::coreos",
# "license": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"]
# "licenses": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"],
# "description": "System and service manager for Linux",
# "homepage": "https://www.freedesktop.org/wiki/Software/systemd",
# "source": "https://github.com/flatcar-linux/systemd ",
# "files": "somefile 63a5736879fa647ac5a8d5317e7cb8b0\nsome -> link\n"
# }
# ]
write_licenses() {
@ -291,19 +360,10 @@ write_licenses() {
continue
fi
local path="$1/var/db/pkg/${pkg%%:*}/LICENSE"
local lic_str
if [[ -f "$path" ]]; then
lic_str="$(< $path)"
else
# The package is not installed in $1 so get the license from
# its ebuild
lic_str=$(portageq-${BOARD} metadata "${BOARD_ROOT}" ebuild \
"${pkg%%:*}" LICENSE 2>/dev/null ||:)
if [[ -z "$lic_str" ]]; then
local lic_str="$(get_metadata "$1" "${pkg}" LICENSE)"
if [[ -z "$lic_str" ]]; then
warn "No license found for ${pkg}"
continue
fi
fi
[[ -n $pkg_sep ]] && echo ","
@ -335,6 +395,14 @@ write_licenses() {
# Remove duplicate licenses
local lics=$(tr ' ' '\n' <<< "${req_lics[*]} ${opt_lic}" | sort --unique | tr '\n' ' ')
local homepage="$(get_metadata "$1" "${pkg}" HOMEPAGE)"
local description="$(get_metadata "$1" "${pkg}" DESCRIPTION)"
local src_uri="$(get_metadata "$1" "${pkg}" SRC_URI)"
# Filter out directories, cut type marker, cut timestamp, quote "\", and convert line breaks to "\n"
local files="$(get_metadata "$1" "${pkg}" CONTENTS | grep -v '^dir ' | \
cut -d ' ' -f 2- | rev | cut -d ' ' -f 2- | rev | \
sed 's#\\#\\\\#g' | tr '\n' '*' | sed 's/*/\\n/g')"
echo -n " {\"project\": \"${pkg}\", \"licenses\": ["
local lic_sep=""
@ -345,10 +413,45 @@ write_licenses() {
echo -n "\"${lic}\""
done
echo -n "]}"
echo -n "], \"description\": \"${description}\", \"homepage\": \"${homepage}\", \
\"source\": \"${src_uri}\", \"files\": \"${files}\"}"
done >> "$2"
echo -e "\n]" >> "$2"
# Pretty print the JSON file
mv "$2" "$2".tmp
jq . "$2".tmp > "$2"
rm "$2".tmp
}
# Include the license JSON and all licenses in the rootfs with a small INFO file about usage and other URLs.
insert_licenses() {
local json_input="$1"
local root_fs_dir="$2"
sudo mkdir -p "${root_fs_dir}"/usr/share/licenses/common
sudo_clobber "${root_fs_dir}"/usr/share/licenses/INFO << "EOF"
Flatcar Container Linux distributes software from various projects.
The licenses.json.bz2 file contains the list of projects with their licenses, how to obtain the source code,
and which binary files in Flatcar Container Linux were created from it.
You can read it with "less licenses.json.bz2" or convert it to a text format with
bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"'
The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz".
Build system files and patches used to build these projects are located at:
https://github.com/flatcar-linux/coreos-overlay/
https://github.com/flatcar-linux/portage-stable/
https://github.com/flatcar-linux/scripts/
Information on how to build Flatcar Container Linux can be found under:
https://docs.flatcar-linux.org/os/sdk-modifying-flatcar/
EOF
sudo cp "${json_input}" "${root_fs_dir}"/usr/share/licenses/licenses.json
# Compress the file from 2.1 MB to 0.39 MB
sudo lbzip2 -9 "${root_fs_dir}"/usr/share/licenses/licenses.json
# Copy all needed licenses to a "common" subdirectory and compress them
local license_list # define before assignment because it would mask any error
license_list="$(jq -r '.[] | "/var/gentoo/repos/gentoo/licenses/\(.licenses | .[])"' "${json_input}" | sort | uniq)"
sudo cp ${license_list} "${root_fs_dir}"/usr/share/licenses/common/
# Compress the licenses just with gzip because there is no big difference as they are single files
sudo gzip -9 "${root_fs_dir}"/usr/share/licenses/common/*
}
# Add an entry to the image's package.provided

1
build_library/dev_container_util.sh
View File

@ -79,6 +79,7 @@ create_dev_container() {
run_localedef "${root_fs_dir}"
write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
# Setup portage for emerge and gmerge
configure_dev_portage "${root_fs_dir}"

1
build_library/ebuild_aci_util.sh
View File

@ -27,6 +27,7 @@ create_ebuild_aci_image() {
run_ldconfig "${root_fs_dir}"
write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
cleanup_mounts "${root_fs_dir}"
trap - EXIT

1
build_library/oem_aci_util.sh
View File

@ -34,6 +34,7 @@ create_oem_aci_image() {
run_ldconfig "${root_fs_dir}"
write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
# clean-ups of things we do not need
sudo rm ${root_fs_dir}/etc/csh.env

1
build_library/prod_image_util.sh
View File

@ -82,6 +82,7 @@ create_prod_image() {
run_localedef "${root_fs_dir}"
write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
# Assert that if this is supposed to be an official build that the
# official update keys have been used.