From 388220cfad7a3c648c2cacde7ba2450a455b8134 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 8 Jun 2014 14:44:37 -0400 Subject: [PATCH 1/2] fix(build_image): move upload calls to build_library This will let the image building code upload extra data without the wrapper script having to know about it. --- build_image | 3 --- build_library/dev_image_util.sh | 2 ++ build_library/prod_image_util.sh | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build_image b/build_image index c1d4bdf752..1361315b48 100755 --- a/build_image +++ b/build_image @@ -151,17 +151,14 @@ fi if [[ "${DEV_IMAGE}" -eq 1 ]]; then create_dev_image ${COREOS_DEVELOPER_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} - upload_image "${BUILD_DIR}/${COREOS_DEVELOPER_IMAGE_NAME}" fi if [[ "${CONTAINER}" -eq 1 ]]; then create_dev_image "${COREOS_DEVELOPER_CONTAINER_NAME}" "${CONTAINER_LAYOUT}" "${FLAGS_group}" - upload_image "${BUILD_DIR}/${OREOS_DEVELOPER_CONTAINER_NAME}" fi if [[ "${PROD_IMAGE}" -eq 1 ]]; then create_prod_image ${COREOS_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} - upload_image "${BUILD_DIR}/${COREOS_PRODUCTION_IMAGE_NAME}" if [[ ${FLAGS_generate_update} -eq ${FLAGS_TRUE} ]]; then generate_update "${COREOS_PRODUCTION_IMAGE_NAME}" ${DISK_LAYOUT} fi diff --git a/build_library/dev_image_util.sh b/build_library/dev_image_util.sh index 4b036979cb..259f5c3dc8 100755 --- a/build_library/dev_image_util.sh +++ b/build_library/dev_image_util.sh @@ -93,4 +93,6 @@ EOF systemd_enable "${root_fs_dir}" "local-fs.target" "remount-usr.service" finish_image "${disk_layout}" "${root_fs_dir}" + upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \ + "${BUILD_DIR}/${image_name}" } diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 4fd934bb4a..26a6dbe350 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -76,4 +76,7 @@ EOF "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ tune --disable2fs_rw "${BUILD_DIR}/${image_name}" "USR-A" fi + + upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \ + "${BUILD_DIR}/${image_name}" } From bb3d751e6fb328b0ab1af15df5b9c6a9d351d87e Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 8 Jun 2014 16:18:34 -0400 Subject: [PATCH 2/2] feat(build_image): Generate lists of image contents Index contents by file and package. --- build_library/build_image_util.sh | 23 +++++++++++++++++++++++ build_library/dev_image_util.sh | 7 ++++++- build_library/prod_image_util.sh | 7 ++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index a2e3ec8008..154fb266b1 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -117,6 +117,26 @@ systemd_enable() { sudo ln -sf "../${unit_file}" "${wants_dir}/${unit_alias}" } +# Generate a ls-like listing of a directory tree. +# The ugly printf is used to predictable time format and size in bytes. +write_contents() { + info "Writing ${2##*/}" + pushd "$1" >/dev/null + sudo TZ=UTC find -printf \ + '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \ + | sed -e 's/ -> $//' > "$2" + popd >/dev/null +} + +# Generate a list of installed packages in the format: +# sys-apps/systemd-212-r8::coreos +write_packages() { + info "Writing ${2##*/}" + ROOT="$1" equery-$BOARD --no-color \ + list '*' --format '$cpv::$repo' \ + > "$2" +} + start_image() { local image_name="$1" local disk_layout="$2" @@ -152,6 +172,7 @@ start_image() { finish_image() { local disk_layout="$1" local root_fs_dir="$2" + local image_contents="$3" # Record directories installed to the state partition. # Explicitly ignore entries covered by existing configs. @@ -182,6 +203,8 @@ finish_image() { "user-cloudinit@.path" "user-cloudinit@${unit_path}.path" fi + write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}" + # Zero all fs free space to make it more compressible so auto-update # payloads become smaller, not fatal since it won't work on linux < 3.2 sudo fstrim "${root_fs_dir}" || true diff --git a/build_library/dev_image_util.sh b/build_library/dev_image_util.sh index 259f5c3dc8..823fdc6a90 100755 --- a/build_library/dev_image_util.sh +++ b/build_library/dev_image_util.sh @@ -66,10 +66,13 @@ create_dev_image() { info "Building developer image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" + local image_contents="${image_name%.bin}_contents.txt" + local image_packages="${image_name%.bin}_packages.txt" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" emerge_to_image "${root_fs_dir}" coreos-base/coreos-dev + write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" # Setup portage for emerge and gmerge configure_dev_portage "${root_fs_dir}" "${devserver}" @@ -92,7 +95,9 @@ EOF # The remount services are provided by coreos-base/coreos-init systemd_enable "${root_fs_dir}" "local-fs.target" "remount-usr.service" - finish_image "${disk_layout}" "${root_fs_dir}" + finish_image "${disk_layout}" "${root_fs_dir}" "${image_contents}" upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \ + "${BUILD_DIR}/${image_contents}" \ + "${BUILD_DIR}/${image_packages}" \ "${BUILD_DIR}/${image_name}" } diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 26a6dbe350..96b1b5b6da 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -32,12 +32,15 @@ create_prod_image() { info "Building production image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" + local image_contents="${image_name%.bin}_contents.txt" + local image_packages="${image_name%.bin}_packages.txt" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" # Install minimal GCC (libs only) and then everything else emerge_prod_gcc "${root_fs_dir}" emerge_to_image "${root_fs_dir}" coreos-base/coreos + write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env @@ -69,7 +72,7 @@ EOF disable_read_write=${FLAGS_FALSE} fi - finish_image "${disk_layout}" "${root_fs_dir}" + finish_image "${disk_layout}" "${root_fs_dir}" "${image_contents}" # Make the filesystem un-mountable as read-write. if [[ ${disable_read_write} -eq ${FLAGS_TRUE} ]]; then @@ -78,5 +81,7 @@ EOF fi upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \ + "${BUILD_DIR}/${image_contents}" \ + "${BUILD_DIR}/${image_packages}" \ "${BUILD_DIR}/${image_name}" }