From 1ea1e8ba9b52a5d41ca4970ab481d3d96fc9915e Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 26 Jul 2013 22:26:40 -0400 Subject: [PATCH 1/4] cleanup(common): Remove rarely used pv_cat_cmd Its single use is in build_common and even then having a little progress bar for copying images isn't that interesting, they just get lost in the noise of the emerge output. Keep it simple, use cp. --- build_library/build_common.sh | 2 +- common.sh | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/build_library/build_common.sh b/build_library/build_common.sh index 09a31e6df4..12f07afd3f 100644 --- a/build_library/build_common.sh +++ b/build_library/build_common.sh @@ -42,7 +42,7 @@ copy_image() { local dst="${BUILD_DIR}/$2" if should_build_image $1; then echo "Creating $2 from $1..." - $(pv_cat_cmd) "${src}" >"${dst}" || die "Cannot copy $1 to $2" + cp "${src}" "${dst}" || die "Cannot copy $1 to $2" else mv "${src}" "${dst}" || die "Cannot move $1 to $2" fi diff --git a/common.sh b/common.sh index ac7e3d844c..e49316943d 100644 --- a/common.sh +++ b/common.sh @@ -14,23 +14,6 @@ fi # Ensure that any sub scripts we invoke get the max proc count. export NUM_JOBS -# Returns the pv command if it's available, otherwise plain-old cat. Note that -# this function echoes the command, rather than running it, so it can be used -# as an argument to other commands (like sudo). -pv_cat_cmd() { - if type -P pv >&/dev/null; then - # Limit pv's output to 80 columns, for readability. - local term_cols=$(stty size 2>/dev/null | cut -d' ' -f2) - if [[ ${term_cols:-0} -gt 80 ]]; then - echo pv -w 80 - else - echo pv - fi - else - echo cat - fi -} - # Make sure we have the location and name of the calling script, using # the current value if it is already set. : ${SCRIPT_LOCATION:=$(dirname "$(readlink -f "$0")")} From f46ee8c332604d24389d8cc369bcc1044be6d2d4 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 26 Jul 2013 22:28:05 -0400 Subject: [PATCH 2/4] fix(build_image): Fix building dev and prod without base Ever since adding prod it hasn't been possible to build just dev and prod without explicitly building base. Base is always built but usually there is no point to keeping it around. Add some logic to make dev not conflict with prod and make sure base is deleted and not uploaded if it wasn't explicitly requested. --- build_image | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/build_image b/build_image index dd73979b01..58f883a091 100755 --- a/build_image +++ b/build_image @@ -151,7 +151,9 @@ mkdir -p "${BUILD_DIR}" # Create the base image. create_base_image ${PRISTINE_IMAGE_NAME} ${FLAGS_enable_rootfs_verification} \ ${FLAGS_enable_bootcache} -upload_image "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}" +if should_build_image ${PRISTINE_IMAGE_NAME}; then + upload_image "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}" +fi # Running board-specific setup if any exists. if type board_setup &>/dev/null; then @@ -161,7 +163,12 @@ fi # Create a developer image if an image that is based on it is requested. if should_build_image ${CHROMEOS_DEVELOPER_IMAGE_NAME} \ ${CHROMEOS_TEST_IMAGE_NAME}; then - copy_image ${CHROMEOS_BASE_IMAGE_NAME} ${CHROMEOS_DEVELOPER_IMAGE_NAME} + if should_build_image ${COREOS_PRODUCTION_IMAGE_NAME}; then + cp "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}" \ + "${BUILD_DIR}/${CHROMEOS_DEVELOPER_IMAGE_NAME}" + else + copy_image ${PRISTINE_IMAGE_NAME} ${CHROMEOS_DEVELOPER_IMAGE_NAME} + fi install_dev_packages ${CHROMEOS_DEVELOPER_IMAGE_NAME} upload_image "${BUILD_DIR}/${CHROMEOS_DEVELOPER_IMAGE_NAME}" fi @@ -178,6 +185,10 @@ if should_build_image ${COREOS_PRODUCTION_IMAGE_NAME}; then upload_image "${BUILD_DIR}/${COREOS_PRODUCTION_IMAGE_NAME}" fi +if ! should_build_image ${PRISTINE_IMAGE_NAME}; then + rm -f "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}" +fi + # From a developer image create a test|factory_test image. if should_build_image ${CHROMEOS_TEST_IMAGE_NAME}; then copy_image ${CHROMEOS_DEVELOPER_IMAGE_NAME} ${CHROMEOS_TEST_IMAGE_NAME} From a1a1ed830c682ef99be46b0f37a841976b20a915 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 26 Jul 2013 23:45:10 -0400 Subject: [PATCH 3/4] fix(build_library): Use sparse files for disk images, no useless sudo Enable sparse files for all dd and cp commands and replace some dd commands that are really better off being truncate commands. While in the neighborhood there were a number of useless sudo commands for things that just happen to be in sbin. Call them directly instead. --- build_library/base_image_util.sh | 26 +++++++++++--------------- build_library/build_common.sh | 2 +- build_library/disk_layout_util.sh | 14 +++++++------- build_library/vm_image_util.sh | 18 +++++++++--------- 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/build_library/base_image_util.sh b/build_library/base_image_util.sh index 46bbc73a24..b91b920597 100755 --- a/build_library/base_image_util.sh +++ b/build_library/base_image_util.sh @@ -100,11 +100,10 @@ create_base_image() { # Build root FS image. info "Building ${root_fs_img}" - dd if=/dev/zero of="${root_fs_img}" bs=1 count=1 \ - seek=$((root_fs_bytes - 1)) status=none - sudo mkfs.ext2 -F -q -b ${fs_block_size} "${root_fs_img}" \ + truncate --size="${root_fs_bytes}" "${root_fs_img}" + /sbin/mkfs.ext2 -F -q -b ${fs_block_size} "${root_fs_img}" \ "$((root_fs_bytes / fs_block_size))" - sudo tune2fs -L "${root_fs_label}" \ + /sbin/tune2fs -L "${root_fs_label}" \ -U clear \ -T 20091119110000 \ -c 0 \ @@ -120,26 +119,23 @@ create_base_image() { # Build stateful FS disk image. info "Building ${stateful_fs_img}" - dd if=/dev/zero of="${stateful_fs_img}" bs=1 count=1 \ - seek=$((stateful_fs_bytes - 1)) status=none - sudo mkfs.ext4 -F -q "${stateful_fs_img}" - sudo tune2fs -L "${stateful_fs_label}" -U "${stateful_fs_uuid}" \ + truncate --size="${stateful_fs_bytes}" "${stateful_fs_img}" + /sbin/mkfs.ext4 -F -q "${stateful_fs_img}" + /sbin/tune2fs -L "${stateful_fs_label}" -U "${stateful_fs_uuid}" \ -c 0 -i 0 "${stateful_fs_img}" mkdir -p "${stateful_fs_dir}" sudo mount -o loop "${stateful_fs_img}" "${stateful_fs_dir}" # Build ESP disk image. info "Building ${esp_fs_img}" - dd if=/dev/zero of="${esp_fs_img}" bs=1 count=1 \ - seek=$((esp_fs_bytes - 1)) status=none - sudo mkfs.vfat "${esp_fs_img}" + truncate --size="${esp_fs_bytes}" "${esp_fs_img}" + /usr/sbin/mkfs.vfat "${esp_fs_img}" # Build OEM FS disk image. info "Building ${oem_fs_img}" - dd if=/dev/zero of="${oem_fs_img}" bs=1 count=1 \ - seek=$((oem_fs_bytes - 1)) status=none - sudo mkfs.ext4 -F -q "${oem_fs_img}" - sudo tune2fs -L "${oem_fs_label}" -U "${oem_fs_uuid}" \ + truncate --size="${oem_fs_bytes}" "${oem_fs_img}" + /sbin/mkfs.ext4 -F -q "${oem_fs_img}" + /sbin/tune2fs -L "${oem_fs_label}" -U "${oem_fs_uuid}" \ -c 0 -i 0 "${oem_fs_img}" mkdir -p "${oem_fs_dir}" sudo mount -o loop "${oem_fs_img}" "${oem_fs_dir}" diff --git a/build_library/build_common.sh b/build_library/build_common.sh index 12f07afd3f..92e2ee6391 100644 --- a/build_library/build_common.sh +++ b/build_library/build_common.sh @@ -42,7 +42,7 @@ copy_image() { local dst="${BUILD_DIR}/$2" if should_build_image $1; then echo "Creating $2 from $1..." - cp "${src}" "${dst}" || die "Cannot copy $1 to $2" + cp --sparse=always "${src}" "${dst}" || die "Cannot copy $1 to $2" else mv "${src}" "${dst}" || die "Cannot move $1 to $2" fi diff --git a/build_library/disk_layout_util.sh b/build_library/disk_layout_util.sh index ad1b2b6a04..1808603e82 100644 --- a/build_library/disk_layout_util.sh +++ b/build_library/disk_layout_util.sh @@ -168,7 +168,7 @@ EOF local file="part_${part}" local dir="dir_${part}" local target='"${TARGET}"' - local dd_args="bs=512 count=${size}" + local dd_args="bs=512 count=${size} conv=sparse" local start_b=$(( start * 512 )) local size_b=$(( size * 512 )) echo "dd if=${target} of=${file} ${dd_args} skip=${start}" >>"${unpack}" @@ -226,19 +226,19 @@ build_gpt() { # Now populate the partitions. info "Copying stateful partition..." - $sudo dd if="$stateful_img" of="$outdev" conv=notrunc bs=512 \ + $sudo dd if="$stateful_img" of="$outdev" conv=notrunc,sparse bs=512 \ seek=$(partoffset ${outdev} ${stateful_fs_num}) status=none info "Copying rootfs..." - $sudo dd if="$rootfs_img" of="$outdev" conv=notrunc bs=512 \ + $sudo dd if="$rootfs_img" of="$outdev" conv=notrunc,sparse bs=512 \ seek=$(partoffset ${outdev} ${root_fs_num}) status=none info "Copying EFI system partition..." - $sudo dd if="$esp_img" of="$outdev" conv=notrunc bs=512 \ + $sudo dd if="$esp_img" of="$outdev" conv=notrunc,sparse bs=512 \ seek=$(partoffset ${outdev} ${esp_fs_num}) status=none info "Copying OEM partition..." - $sudo dd if="$oem_img" of="$outdev" conv=notrunc bs=512 \ + $sudo dd if="$oem_img" of="$outdev" conv=notrunc,sparse bs=512 \ seek=$(partoffset ${outdev} ${oem_fs_num}) status=none # Pre-set "sucessful" bit in gpt, so we will never mark-for-death @@ -317,11 +317,11 @@ update_partition_table() { -t ${tguid} -u ${uguid} ${dst_img} if [ "${label}" != "STATE" ]; then # Copy source partition as-is. - dd if="${src_img}" of="${dst_img}" conv=notrunc bs=512 \ + dd if="${src_img}" of="${dst_img}" conv=notrunc,sparse bs=512 \ skip=${src_start} seek=${dst_start} count=${size} status=none else # Copy new stateful partition into place. - dd if="${src_state}" of="${dst_img}" conv=notrunc bs=512 \ + dd if="${src_state}" of="${dst_img}" conv=notrunc,sparse bs=512 \ seek=${dst_start} status=none fi done diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index cf0dd573f1..852af5d15a 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -164,7 +164,7 @@ unpack_source_disk() { TEMP_STATE="${VM_TMP_DIR}"/part_${NUM_STATEFUL} # Copy the replacement STATE image if it is set if [[ -n "${alternate_state_image}" ]]; then - cp "${alternate_state_image}" "${TEMP_STATE}" + cp --sparse=always "${alternate_state_image}" "${TEMP_STATE}" fi TEMP_PMBR="${VM_TMP_DIR}"/pmbr @@ -216,14 +216,14 @@ install_oem_package() { # Write the vm disk image to the target directory in the proper format write_vm_disk() { info "Writing partitions to new disk image" - dd if="${TEMP_ROOTFS}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${VM_TMP_IMG} ${NUM_ROOTFS_A}) - dd if="${TEMP_STATE}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${VM_TMP_IMG} ${NUM_STATEFUL}) - dd if="${TEMP_ESP}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${VM_TMP_IMG} ${NUM_ESP}) - dd if="${TEMP_OEM}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${VM_TMP_IMG} ${NUM_OEM}) + dd if="${TEMP_ROOTFS}" of="${VM_TMP_IMG}" conv=notrunc,sparse \ + bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_ROOTFS_A}) + dd if="${TEMP_STATE}" of="${VM_TMP_IMG}" conv=notrunc,sparse \ + bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_STATEFUL}) + dd if="${TEMP_ESP}" of="${VM_TMP_IMG}" conv=notrunc,sparse \ + bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_ESP}) + dd if="${TEMP_OEM}" of="${VM_TMP_IMG}" conv=notrunc,sparse \ + bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_OEM}) if [[ $(_get_vm_opt HYBRID_MBR) -eq 1 ]]; then info "Creating hybrid MBR" From 505107c48da14cf2b83fd4bbdfd24ff9d8acd6a7 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 27 Jul 2013 00:07:09 -0400 Subject: [PATCH 4/4] fix(catalyst): Cleanup temporary files. Catalyst doesn't clean up the temporary root directories even though we don't have seedcache enabled. So lets wipe that mess up. --- lib/catalyst.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/catalyst.sh b/lib/catalyst.sh index 7b514bd745..e94e4c3b24 100644 --- a/lib/catalyst.sh +++ b/lib/catalyst.sh @@ -59,7 +59,7 @@ cat <