Merge pull request #65 from marineam/misc

Misc build updates

build_image

@@ -151,7 +151,9 @@ mkdir -p "${BUILD_DIR}"
 # Create the base image.
 create_base_image ${PRISTINE_IMAGE_NAME} ${FLAGS_enable_rootfs_verification} \
   ${FLAGS_enable_bootcache}
-upload_image "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}"
+if should_build_image ${PRISTINE_IMAGE_NAME}; then
+  upload_image "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}"
+fi
 
 # Running board-specific setup if any exists.
 if type board_setup &>/dev/null; then
@@ -161,7 +163,12 @@ fi
 # Create a developer image if an image that is based on it is requested.
 if should_build_image ${CHROMEOS_DEVELOPER_IMAGE_NAME} \
     ${CHROMEOS_TEST_IMAGE_NAME}; then
-  copy_image ${CHROMEOS_BASE_IMAGE_NAME} ${CHROMEOS_DEVELOPER_IMAGE_NAME}
+  if should_build_image ${COREOS_PRODUCTION_IMAGE_NAME}; then
+    cp "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}" \
+        "${BUILD_DIR}/${CHROMEOS_DEVELOPER_IMAGE_NAME}"
+  else
+    copy_image ${PRISTINE_IMAGE_NAME} ${CHROMEOS_DEVELOPER_IMAGE_NAME}
+  fi
   install_dev_packages ${CHROMEOS_DEVELOPER_IMAGE_NAME}
   upload_image "${BUILD_DIR}/${CHROMEOS_DEVELOPER_IMAGE_NAME}"
 fi
@@ -178,6 +185,10 @@ if should_build_image ${COREOS_PRODUCTION_IMAGE_NAME}; then
   upload_image "${BUILD_DIR}/${COREOS_PRODUCTION_IMAGE_NAME}"
 fi
 
+if ! should_build_image ${PRISTINE_IMAGE_NAME}; then
+  rm -f "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}"
+fi
+
 # From a developer image create a test|factory_test image.
 if should_build_image ${CHROMEOS_TEST_IMAGE_NAME}; then
   copy_image  ${CHROMEOS_DEVELOPER_IMAGE_NAME} ${CHROMEOS_TEST_IMAGE_NAME}

build_library/base_image_util.sh

@@ -100,11 +100,10 @@ create_base_image() {
 
   # Build root FS image.
   info "Building ${root_fs_img}"
-  dd if=/dev/zero of="${root_fs_img}" bs=1 count=1 \
-    seek=$((root_fs_bytes - 1)) status=none
-  sudo mkfs.ext2 -F -q -b ${fs_block_size} "${root_fs_img}" \
+  truncate --size="${root_fs_bytes}" "${root_fs_img}"
+  /sbin/mkfs.ext2 -F -q -b ${fs_block_size} "${root_fs_img}" \
     "$((root_fs_bytes / fs_block_size))"
-  sudo tune2fs -L "${root_fs_label}" \
+  /sbin/tune2fs -L "${root_fs_label}" \
                -U clear \
                -T 20091119110000 \
                -c 0 \
@@ -120,26 +119,23 @@ create_base_image() {
 
   # Build stateful FS disk image.
   info "Building ${stateful_fs_img}"
-  dd if=/dev/zero of="${stateful_fs_img}" bs=1 count=1 \
-    seek=$((stateful_fs_bytes - 1)) status=none
-  sudo mkfs.ext4 -F -q "${stateful_fs_img}"
-  sudo tune2fs -L "${stateful_fs_label}" -U "${stateful_fs_uuid}" \
+  truncate --size="${stateful_fs_bytes}" "${stateful_fs_img}"
+  /sbin/mkfs.ext4 -F -q "${stateful_fs_img}"
+  /sbin/tune2fs -L "${stateful_fs_label}" -U "${stateful_fs_uuid}" \
                -c 0 -i 0 "${stateful_fs_img}"
   mkdir -p "${stateful_fs_dir}"
   sudo mount -o loop "${stateful_fs_img}" "${stateful_fs_dir}"
 
   # Build ESP disk image.
   info "Building ${esp_fs_img}"
-  dd if=/dev/zero of="${esp_fs_img}" bs=1 count=1 \
-    seek=$((esp_fs_bytes - 1)) status=none
-  sudo mkfs.vfat "${esp_fs_img}"
+  truncate --size="${esp_fs_bytes}" "${esp_fs_img}"
+  /usr/sbin/mkfs.vfat "${esp_fs_img}"
 
   # Build OEM FS disk image.
   info "Building ${oem_fs_img}"
-  dd if=/dev/zero of="${oem_fs_img}" bs=1 count=1 \
-    seek=$((oem_fs_bytes - 1)) status=none
-  sudo mkfs.ext4 -F -q "${oem_fs_img}"
-  sudo tune2fs -L "${oem_fs_label}" -U "${oem_fs_uuid}" \
+  truncate --size="${oem_fs_bytes}" "${oem_fs_img}"
+  /sbin/mkfs.ext4 -F -q "${oem_fs_img}"
+  /sbin/tune2fs -L "${oem_fs_label}" -U "${oem_fs_uuid}" \
                -c 0 -i 0 "${oem_fs_img}"
   mkdir -p "${oem_fs_dir}"
   sudo mount -o loop "${oem_fs_img}" "${oem_fs_dir}"

build_library/build_common.sh

@@ -42,7 +42,7 @@ copy_image() {
   local dst="${BUILD_DIR}/$2"
   if should_build_image $1; then
     echo "Creating $2 from $1..."
-    $(pv_cat_cmd) "${src}" >"${dst}" || die "Cannot copy $1 to $2"
+    cp --sparse=always "${src}" "${dst}" || die "Cannot copy $1 to $2"
   else
     mv "${src}" "${dst}" || die "Cannot move $1 to $2"
   fi

build_library/disk_layout_util.sh

@@ -168,7 +168,7 @@ EOF
     local file="part_${part}"
     local dir="dir_${part}"
     local target='"${TARGET}"'
-    local dd_args="bs=512 count=${size}"
+    local dd_args="bs=512 count=${size} conv=sparse"
     local start_b=$(( start * 512 ))
     local size_b=$(( size * 512 ))
     echo "dd if=${target} of=${file} ${dd_args} skip=${start}" >>"${unpack}"
@@ -226,19 +226,19 @@ build_gpt() {
 
   # Now populate the partitions.
   info "Copying stateful partition..."
-  $sudo dd if="$stateful_img" of="$outdev" conv=notrunc bs=512 \
+  $sudo dd if="$stateful_img" of="$outdev" conv=notrunc,sparse bs=512 \
       seek=$(partoffset ${outdev} ${stateful_fs_num}) status=none
 
   info "Copying rootfs..."
-  $sudo dd if="$rootfs_img" of="$outdev" conv=notrunc bs=512 \
+  $sudo dd if="$rootfs_img" of="$outdev" conv=notrunc,sparse bs=512 \
       seek=$(partoffset ${outdev} ${root_fs_num}) status=none
 
   info "Copying EFI system partition..."
-  $sudo dd if="$esp_img" of="$outdev" conv=notrunc bs=512 \
+  $sudo dd if="$esp_img" of="$outdev" conv=notrunc,sparse bs=512 \
       seek=$(partoffset ${outdev} ${esp_fs_num}) status=none
 
   info "Copying OEM partition..."
-  $sudo dd if="$oem_img" of="$outdev" conv=notrunc bs=512 \
+  $sudo dd if="$oem_img" of="$outdev" conv=notrunc,sparse bs=512 \
       seek=$(partoffset ${outdev} ${oem_fs_num}) status=none
 
   # Pre-set "sucessful" bit in gpt, so we will never mark-for-death
@@ -317,11 +317,11 @@ update_partition_table() {
              -t ${tguid} -u ${uguid} ${dst_img}
     if [ "${label}" != "STATE" ]; then
       # Copy source partition as-is.
-      dd if="${src_img}" of="${dst_img}" conv=notrunc bs=512 \
+      dd if="${src_img}" of="${dst_img}" conv=notrunc,sparse bs=512 \
         skip=${src_start} seek=${dst_start} count=${size} status=none
     else
       # Copy new stateful partition into place.
-      dd if="${src_state}" of="${dst_img}" conv=notrunc bs=512 \
+      dd if="${src_state}" of="${dst_img}" conv=notrunc,sparse bs=512 \
         seek=${dst_start} status=none
     fi
   done

build_library/vm_image_util.sh

@@ -164,7 +164,7 @@ unpack_source_disk() {
     TEMP_STATE="${VM_TMP_DIR}"/part_${NUM_STATEFUL}
     # Copy the replacement STATE image if it is set
     if [[ -n "${alternate_state_image}" ]]; then
-        cp "${alternate_state_image}" "${TEMP_STATE}"
+        cp --sparse=always "${alternate_state_image}" "${TEMP_STATE}"
     fi
 
     TEMP_PMBR="${VM_TMP_DIR}"/pmbr
@@ -216,14 +216,14 @@ install_oem_package() {
 # Write the vm disk image to the target directory in the proper format
 write_vm_disk() {
     info "Writing partitions to new disk image"
-    dd if="${TEMP_ROOTFS}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \
-      seek=$(partoffset ${VM_TMP_IMG} ${NUM_ROOTFS_A})
-    dd if="${TEMP_STATE}"  of="${VM_TMP_IMG}" conv=notrunc bs=512 \
-      seek=$(partoffset ${VM_TMP_IMG} ${NUM_STATEFUL})
-    dd if="${TEMP_ESP}"    of="${VM_TMP_IMG}" conv=notrunc bs=512 \
-      seek=$(partoffset ${VM_TMP_IMG} ${NUM_ESP})
-    dd if="${TEMP_OEM}"    of="${VM_TMP_IMG}" conv=notrunc bs=512 \
-      seek=$(partoffset ${VM_TMP_IMG} ${NUM_OEM})
+    dd if="${TEMP_ROOTFS}" of="${VM_TMP_IMG}" conv=notrunc,sparse \
+        bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_ROOTFS_A})
+    dd if="${TEMP_STATE}"  of="${VM_TMP_IMG}" conv=notrunc,sparse \
+        bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_STATEFUL})
+    dd if="${TEMP_ESP}"    of="${VM_TMP_IMG}" conv=notrunc,sparse \
+        bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_ESP})
+    dd if="${TEMP_OEM}"    of="${VM_TMP_IMG}" conv=notrunc,sparse \
+        bs=512 seek=$(partoffset ${VM_TMP_IMG} ${NUM_OEM})
 
     if [[ $(_get_vm_opt HYBRID_MBR) -eq 1 ]]; then
         info "Creating hybrid MBR"

common.sh

@@ -14,23 +14,6 @@ fi
 # Ensure that any sub scripts we invoke get the max proc count.
 export NUM_JOBS
 
-# Returns the pv command if it's available, otherwise plain-old cat. Note that
-# this function echoes the command, rather than running it, so it can be used
-# as an argument to other commands (like sudo).
-pv_cat_cmd() {
-  if type -P pv >&/dev/null; then
-    # Limit pv's output to 80 columns, for readability.
-    local term_cols=$(stty size 2>/dev/null | cut -d' ' -f2)
-    if [[ ${term_cols:-0} -gt 80 ]]; then
-      echo pv -w 80
-    else
-      echo pv
-    fi
-  else
-    echo cat
-  fi
-}
-
 # Make sure we have the location and name of the calling script, using
 # the current value if it is already set.
 : ${SCRIPT_LOCATION:=$(dirname "$(readlink -f "$0")")}

lib/catalyst.sh

@@ -59,7 +59,7 @@ cat <<EOF
 contents="auto"
 digests="md5 sha1 sha512 whirlpool"
 hash_function="crc32"
-options="autoresume ccache kerncache pkgcache"
+options="autoresume ccache pkgcache"
 sharedir="/usr/lib/catalyst"
 storedir="$CATALYST_ROOT"
 distdir="$DISTDIR"
@@ -261,6 +261,8 @@ build_stage() {
         -c "$TEMPDIR/catalyst.conf" \
         -f "$TEMPDIR/${stage}.spec" \
         -C "source_subpath=$srcpath"
+    # Catalyst doesn't clean up after itself...
+    rm -rf "$TEMPDIR/$stage-${ARCH}-${FLAGS_version}"
     ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \
         "$BUILDS/$stage-${ARCH}-latest.tar.bz2"
     info "Finished building $target_tarball"
@@ -313,4 +315,7 @@ catalyst_build() {
         build_stage stage4 "$SEED"
         used_seed=1
     fi
+
+    # Cleanup snapshots, we don't use them
+    rm -rf "$CATALYST_ROOT/snapshots/portage-${FLAGS_version}.tar.bz2"*
 }