diff --git a/lib/cros_image_common.sh b/lib/cros_image_common.sh index 0ad9d0bbcd..86bba75504 100644 --- a/lib/cros_image_common.sh +++ b/lib/cros_image_common.sh @@ -8,19 +8,20 @@ # especially for being redistributed into platforms without complete Chromium OS # developing environment. -# Check if given command is available in current system -has_command() { +# Checks if given command is available in current system +image_has_command() { type "$1" >/dev/null 2>&1 } -err_die() { +# Prints error message and exit as 1 (error) +image_die() { echo "ERROR: $@" >&2 exit 1 } -# Finds the best gzip compressor and invoke it. -gzip_compress() { - if has_command pigz; then +# Finds the best gzip compressor and invoke it +image_gzip_compress() { + if image_has_command pigz; then # echo " ** Using parallel gzip **" >&2 # Tested with -b 32, 64, 128(default), 256, 1024, 16384, and -b 32 (max # window size of Deflate) seems to be the best in output size. @@ -30,43 +31,58 @@ gzip_compress() { fi } +# Finds the best bzip2 compressor and invoke it +image_bzip2_compress() { + if image_has_command pbzip2; then + pbzip2 "$@" + else + bzip2 "$@" + fi +} + # Finds if current system has tools for part_* commands -has_part_tools() { - has_command cgpt || has_command parted +image_has_part_tools() { + image_has_command cgpt || image_has_command parted } # Finds the best partition tool and print partition offset -part_offset() { +image_part_offset() { local file="$1" local partno="$2" + local unpack_file="$(dirname "$file")/unpack_partitions.sh" - if has_command cgpt; then + # TODO parted is available on most Linux so we may deprecate other code path + if image_has_command cgpt; then cgpt show -b -i "$partno" "$file" - elif has_command parted; then - parted -m "$file" unit s print | - grep "^$partno:" | cut -d ':' -f 2 | sed 's/s$//' + elif image_has_command parted; then + parted -m "$file" unit s print | awk -F ':' "/^$partno:/ { print int(\$2) }" + elif [ -f "$unpack_file" ]; then + awk "/ $partno *Label:/ { print \$2 }" "$unpack_file" else exit 1 fi } # Finds the best partition tool and print partition size -part_size() { +image_part_size() { local file="$1" local partno="$2" + local unpack_file="$(dirname "$file")/unpack_partitions.sh" - if has_command cgpt; then + # TODO parted is available on most Linux so we may deprecate other code path + if image_has_command cgpt; then cgpt show -s -i "$partno" "$file" - elif has_command parted; then - parted -m "$file" unit s print | - grep "^$partno:" | cut -d ':' -f 4 | sed 's/s$//' + elif image_has_command parted; then + parted -m "$file" unit s print | awk -F ':' "/^$partno:/ { print int(\$4) }" + elif [ -s "$unpack_file" ]; then + awk "/ $partno *Label:/ { print \$3 }" "$unpack_file" else exit 1 fi } # Dumps a file by given offset and size (in sectors) -dump_partial_file() { +image_dump_partial_file() { local file="$1" local offset="$2" local sectors="$3" @@ -82,10 +98,10 @@ dump_partial_file() { bs=$((bs * buffer_ratio)) fi - if has_command pv; then + if image_has_command pv; then dd if="$file" bs=$bs skip="$offset" count="$sectors" \ oflag=sync status=noxfer 2>/dev/null | - pv -ptreb -B 4m -s $((sectors * $bs)) + pv -ptreb -B $bs -s $((sectors * bs)) else dd if="$file" bs=$bs skip="$offset" count="$sectors" \ oflag=sync status=noxfer 2>/dev/null @@ -93,14 +109,62 @@ dump_partial_file() { } # Dumps a specific partition from given image file -dump_partition() { +image_dump_partition() { local file="$1" local part_num="$2" - local offset="$(part_offset "$file" "$part_num")" || - err_die "failed to dump partition #$part_num from: $file" - local size="$(part_size "$file" "$part_num")" || - err_die "failed to dump partition #$part_num from: $file" + local offset="$(image_part_offset "$file" "$part_num")" || + image_die "failed to find partition #$part_num from: $file" + local size="$(image_part_size "$file" "$part_num")" || + image_die "failed to find partition #$part_num from: $file" - dump_partial_file "$file" "$offset" "$size" + image_dump_partial_file "$file" "$offset" "$size" } +# Maps a specific partition from given image file to a loop device +image_map_partition() { + local file="$1" + local part_num="$2" + local offset="$(image_part_offset "$file" "$part_num")" || + image_die "failed to find partition #$part_num from: $file" + local size="$(image_part_size "$file" "$part_num")" || + image_die "failed to find partition #$part_num from: $file" + + losetup --offset $((offset * 512)) --sizelimit=$((size * 512)) \ + -f --show "$file" +} + +# Unmaps a loop device created by image_map_partition +image_unmap_partition() { + local map_point="$1" + + losetup -d "$map_point" +} + +# Mounts a specific partition inside a given image file +image_mount_partition() { + local file="$1" + local part_num="$2" + local mount_point="$3" + local mount_opt="$4" + local offset="$(image_part_offset "$file" "$part_num")" || + image_die "failed to find partition #$part_num from: $file" + local size="$(image_part_size "$file" "$part_num")" || + image_die "failed to find partition #$part_num from: $file" + + if [ -z "$mount_opt" ]; then + # by default, mount as read-only. + mount_opt=",ro" + fi + + mount \ + -o "loop,offset=$((offset * 512)),sizelimit=$((size * 512)),$mount_opt" \ + "$file" \ + "$mount_point" +} + +# Unmounts a partition mount point by mount_partition +image_umount_partition() { + local mount_point="$1" + + umount -d "$mount_point" +} diff --git a/make_factory_package.sh b/make_factory_package.sh index 57c23a159c..0eb200e52a 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -39,32 +39,33 @@ DEFINE_string subfolder "" \ FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" -if [ ! -f "${FLAGS_release}" ] ; then +if [ ! -f "${FLAGS_release}" ]; then echo "Cannot find image file ${FLAGS_release}" exit 1 fi -if [ ! -f "${FLAGS_factory}" ] ; then +if [ ! -f "${FLAGS_factory}" ]; then echo "Cannot find image file ${FLAGS_factory}" exit 1 fi -if [ ! -z "${FLAGS_firmware_updater}" ] && \ - [ ! -f "${FLAGS_firmware_updater}" ] ; then +if [ -n "${FLAGS_firmware_updater}" ] && + [ ! -f "${FLAGS_firmware_updater}" ]; then echo "Cannot find firmware file ${FLAGS_firmware_updater}" exit 1 fi # Convert args to paths. Need eval to un-quote the string so that shell # chars like ~ are processed; just doing FOO=`readlink -f ${FOO}` won't work. -OMAHA_DIR=${SRC_ROOT}/platform/dev -OMAHA_DATA_DIR=${OMAHA_DIR}/static/ +OMAHA_DIR="${SRC_ROOT}/platform/dev" +OMAHA_CONF="${OMAHA_DIR}/miniomaha.conf" +OMAHA_DATA_DIR="${OMAHA_DIR}/static/" # Note: The subfolder flag can only append configs. That means you will need # to have unique board IDs for every time you run. If you delete miniomaha.conf # you can still use this flag and it will start fresh. -if [ -n "${FLAGS_subfolder}" ] ; then - OMAHA_DATA_DIR=${OMAHA_DIR}/static/${FLAGS_subfolder}/ +if [ -n "${FLAGS_subfolder}" ]; then + OMAHA_DATA_DIR="${OMAHA_DIR}/static/${FLAGS_subfolder}/" fi if [ ${INSIDE_CHROOT} -eq 0 ]; then @@ -74,20 +75,20 @@ if [ ${INSIDE_CHROOT} -eq 0 ]; then fi # Use this image as the source image to copy -RELEASE_DIR=`dirname ${FLAGS_release}` -FACTORY_DIR=`dirname ${FLAGS_factory}` -RELEASE_IMAGE=`basename ${FLAGS_release}` -FACTORY_IMAGE=`basename ${FLAGS_factory}` +RELEASE_DIR="$(dirname "${FLAGS_release}")" +FACTORY_DIR="$(dirname "${FLAGS_factory}")" +RELEASE_IMAGE="$(basename "${FLAGS_release}")" +FACTORY_IMAGE="$(basename "${FLAGS_factory}")" prepare_omaha() { - sudo rm -rf ${OMAHA_DATA_DIR}/rootfs-test.gz - sudo rm -rf ${OMAHA_DATA_DIR}/rootfs-release.gz - rm -rf ${OMAHA_DATA_DIR}/efi.gz - rm -rf ${OMAHA_DATA_DIR}/oem.gz - rm -rf ${OMAHA_DATA_DIR}/state.gz - if [ ! -f "${OMAHA_DATA_DIR}" ] ; then - mkdir -p ${OMAHA_DATA_DIR} + sudo rm -rf "${OMAHA_DATA_DIR}/rootfs-test.gz" + sudo rm -rf "${OMAHA_DATA_DIR}/rootfs-release.gz" + rm -rf "${OMAHA_DATA_DIR}/efi.gz" + rm -rf "${OMAHA_DATA_DIR}/oem.gz" + rm -rf "${OMAHA_DATA_DIR}/state.gz" + if [ ! -d "${OMAHA_DATA_DIR}" ]; then + mkdir -p "${OMAHA_DATA_DIR}" fi } @@ -102,12 +103,12 @@ prepare_dir() { compress_and_hash_memento_image() { local input_file="$1" - if has_part_tools; then - sudo "${SCRIPTS_DIR}/mk_memento_images.sh" "$input_file" 2 3 | + if [ -n "${IMAGE_IS_UNPACKED}" ]; then + sudo "${SCRIPTS_DIR}/mk_memento_images.sh" part_2 part_3 | grep hash | awk '{print $4}' else - sudo "${SCRIPTS_DIR}/mk_memento_images.sh" part_2 part_3 | + sudo "${SCRIPTS_DIR}/mk_memento_images.sh" "$input_file" 2 3 | grep hash | awk '{print $4}' fi @@ -119,12 +120,12 @@ compress_and_hash_file() { if [ -z "$input_file" ]; then # Runs as a pipe processor - gzip_compress -c -9 | + image_gzip_compress -c -9 | tee "$output_file" | openssl sha1 -binary | openssl base64 else - gzip_compress -c -9 "$input_file" | + image_gzip_compress -c -9 "$input_file" | tee "$output_file" | openssl sha1 -binary | openssl base64 @@ -136,30 +137,38 @@ compress_and_hash_partition() { local part_num="$2" local output_file="$3" - if has_part_tools; then - dump_partition "$input_file" "$part_num" | - compress_and_hash_file "" "$output_file" - else + if [ -n "${IMAGE_IS_UNPACKED}" ]; then compress_and_hash_file "part_$part_num" "$output_file" + else + image_dump_partition "$input_file" "$part_num" | + compress_and_hash_file "" "$output_file" fi } # Clean up stale config and data files. prepare_omaha +# Decide if we should unpack partition +if image_has_part_tools; then + IMAGE_IS_UNPACKED= +else + #TODO(hungte) Currently we run unpack_partitions.sh if part_tools are not + # found. If the format of unpack_partitions.sh is reliable, we can prevent + # creating temporary files. See image_part_offset for more information. + echo "WARNING: cannot find partition tools. Using unpack_partitions.sh." >&2 + IMAGE_IS_UNPACKED=1 +fi + # Get the release image. -pushd ${RELEASE_DIR} > /dev/null +pushd "${RELEASE_DIR}" >/dev/null echo "Generating omaha release image from ${FLAGS_release}" echo "Generating omaha factory image from ${FLAGS_factory}" echo "Output omaha image to ${OMAHA_DATA_DIR}" -echo "Output omaha config to ${OMAHA_DIR}/miniomaha.conf" +echo "Output omaha config to ${OMAHA_CONF}" prepare_dir -if ! has_part_tools; then - #TODO(hungte) we can still avoid running unpack_partitions.sh - # by $(cat unpack_partitions.sh | grep Label | sed "s/#//" | grep ${name}" | - # awk '{ print $1}') to fetch offset/size. +if [ -n "${IMAGE_IS_UNPACKED}" ]; then echo "Unpacking image ${RELEASE_IMAGE} ..." >&2 sudo ./unpack_partitions.sh "${RELEASE_IMAGE}" 2>/dev/null fi @@ -167,24 +176,24 @@ fi release_hash="$(compress_and_hash_memento_image "${RELEASE_IMAGE}")" sudo chmod a+rw update.gz mv update.gz rootfs-release.gz -mv rootfs-release.gz ${OMAHA_DATA_DIR} +mv rootfs-release.gz "${OMAHA_DATA_DIR}" echo "release: ${release_hash}" oem_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 8 "oem.gz")" -mv oem.gz ${OMAHA_DATA_DIR} +mv oem.gz "${OMAHA_DATA_DIR}" echo "oem: ${oem_hash}" efi_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 12 "efi.gz")" -mv efi.gz ${OMAHA_DATA_DIR} +mv efi.gz "${OMAHA_DATA_DIR}" echo "efi: ${efi_hash}" -popd > /dev/null +popd >/dev/null # Go to retrieve the factory test image. -pushd ${FACTORY_DIR} > /dev/null +pushd "${FACTORY_DIR}" >/dev/null prepare_dir -if ! has_part_tools; then +if [ -n "${IMAGE_IS_UNPACKED}" ]; then echo "Unpacking image ${FACTORY_IMAGE} ..." >&2 sudo ./unpack_partitions.sh "${FACTORY_IMAGE}" 2>/dev/null fi @@ -192,16 +201,16 @@ fi test_hash="$(compress_and_hash_memento_image "${FACTORY_IMAGE}")" sudo chmod a+rw update.gz mv update.gz rootfs-test.gz -mv rootfs-test.gz ${OMAHA_DATA_DIR} +mv rootfs-test.gz "${OMAHA_DATA_DIR}" echo "test: ${test_hash}" state_hash="$(compress_and_hash_partition "${FACTORY_IMAGE}" 1 "state.gz")" -mv state.gz ${OMAHA_DATA_DIR} +mv state.gz "${OMAHA_DATA_DIR}" echo "state: ${state_hash}" -popd > /dev/null +popd >/dev/null -if [ ! -z ${FLAGS_firmware_updater} ] ; then +if [ -n "${FLAGS_firmware_updater}" ]; then SHELLBALL="${FLAGS_firmware_updater}" if [ ! -f "$SHELLBALL" ]; then echo "Failed to find firmware updater: $SHELLBALL." @@ -209,70 +218,70 @@ if [ ! -z ${FLAGS_firmware_updater} ] ; then fi firmware_hash="$(compress_and_hash_file "$SHELLBALL" "firmware.gz")" - mv firmware.gz ${OMAHA_DATA_DIR} + mv firmware.gz "${OMAHA_DATA_DIR}" echo "firmware: ${firmware_hash}" fi # If the file does exist and we are using the subfolder flag we are going to # append another config. -if [ -n "${FLAGS_subfolder}" ] && \ - [ -f "${OMAHA_DIR}"/miniomaha.conf"" ] ; then +if [ -n "${FLAGS_subfolder}" ] && + [ -f "${OMAHA_CONF}" ]; then # Remove the ']' from the last line of the file so we can add another config. - while [ -s "${OMAHA_DIR}/miniomaha.conf" ]; do + while [ -s "${OMAHA_CONF}" ]; do # If the last line is null - if [ -z "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" ]; then - sed -i '$d' "${OMAHA_DIR}/miniomaha.conf" - elif [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" != ']' ]; then - sed -i '$d' "${OMAHA_DIR}/miniomaha.conf" + if [ -z "$(tail -1 "${OMAHA_CONF}")" ]; then + sed -i '$d' "${OMAHA_CONF}" + elif [ "$(tail -1 "${OMAHA_CONF}")" != ']' ]; then + sed -i '$d' "${OMAHA_CONF}" else break fi done # Remove the last ] - if [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" = ']' ]; then - sed -i '$d' "${OMAHA_DIR}/miniomaha.conf" + if [ "$(tail -1 "${OMAHA_CONF}")" = ']' ]; then + sed -i '$d' "${OMAHA_CONF}" fi # If the file is empty, create it from scratch - if [ ! -s "${OMAHA_DIR}/miniomaha.conf" ]; then - echo "config = [" > "${OMAHA_DIR}/miniomaha.conf" + if [ ! -s "${OMAHA_CONF}" ]; then + echo "config = [" >"${OMAHA_CONF}" fi else - echo "config = [" > "${OMAHA_DIR}/miniomaha.conf" + echo "config = [" >"${OMAHA_CONF}" fi -if [ -n "${FLAGS_subfolder}" ] ; then +if [ -n "${FLAGS_subfolder}" ]; then subfolder="${FLAGS_subfolder}/" fi echo -n "{ 'qual_ids': set([\"${FLAGS_board}\"]), - 'factory_image': '"${subfolder}"rootfs-test.gz', + 'factory_image': '${subfolder}rootfs-test.gz', 'factory_checksum': '${test_hash}', - 'release_image': '"${subfolder}"rootfs-release.gz', + 'release_image': '${subfolder}rootfs-release.gz', 'release_checksum': '${release_hash}', - 'oempartitionimg_image': '"${subfolder}"oem.gz', + 'oempartitionimg_image': '${subfolder}oem.gz', 'oempartitionimg_checksum': '${oem_hash}', - 'efipartitionimg_image': '"${subfolder}"efi.gz', + 'efipartitionimg_image': '${subfolder}efi.gz', 'efipartitionimg_checksum': '${efi_hash}', - 'stateimg_image': '"${subfolder}"state.gz', - 'stateimg_checksum': '${state_hash}'," >> ${OMAHA_DIR}/miniomaha.conf + 'stateimg_image': '${subfolder}state.gz', + 'stateimg_checksum': '${state_hash}'," >>"${OMAHA_CONF}" -if [ ! -z "${FLAGS_firmware_updater}" ] ; then +if [ -n "${FLAGS_firmware_updater}" ] ; then echo -n " - 'firmware_image': '"${subfolder}"firmware.gz', - 'firmware_checksum': '${firmware_hash}'," >> ${OMAHA_DIR}/miniomaha.conf + 'firmware_image': '${subfolder}firmware.gz', + 'firmware_checksum': '${firmware_hash}'," >>"${OMAHA_CONF}" fi echo -n " }, ] -" >> ${OMAHA_DIR}/miniomaha.conf +" >>"${OMAHA_CONF}" -echo "The miniomaha server lives in src/platform/dev" -echo "to validate the configutarion, run:" -echo " python2.6 devserver.py --factory_config miniomaha.conf \ ---validate_factory_config" -echo "To run the server:" -echo " python2.6 devserver.py --factory_config miniomaha.conf" +echo "The miniomaha server lives in src/platform/dev. +To validate the configutarion, run: + python2.6 devserver.py --factory_config miniomaha.conf \ + --validate_factory_config +To run the server: + python2.6 devserver.py --factory_config miniomaha.conf" diff --git a/mk_memento_images.sh b/mk_memento_images.sh index 415768fb4b..7a10745427 100755 --- a/mk_memento_images.sh +++ b/mk_memento_images.sh @@ -28,13 +28,13 @@ if [ "$CROS_GENERATE_UPDATE_PAYLOAD_CALLED" != "1" ]; then echo " Please run that script with --help to see how to use it." fi -if ! has_command pigz; then +if ! image_has_command pigz; then (echo "WARNING:" echo " Your system does not have pigz (parallel gzip) installed." echo " COMPRESSING WILL BE VERY SLOW. It is recommended to install pigz" - if has_command apt-get; then + if image_has_command apt-get; then echo " by 'sudo apt-get install pigz'." - elif has_command emerge; then + elif image_has_command emerge; then echo " by 'sudo emerge pigz'." fi) >&2 fi @@ -58,14 +58,14 @@ else # chromiumos_img kern_part_no rootfs_part_no KPART="$1" ROOT_PART="$1" - KPART_OFFSET="$(part_offset "$KPART" "$2")" || - err_die "cannot retieve kernel partition offset" - KPART_SECTORS="$(part_size "$KPART" "$2")" || - err_die "cannot retieve kernel partition size" - ROOT_OFFSET="$(part_offset "$ROOT_PART" "$3")" || - err_die "cannot retieve root partition offset" - ROOT_SECTORS="$(part_size "$ROOT_PART" "$3")" || - err_die "cannot retieve root partition size" + KPART_OFFSET="$(image_part_offset "$KPART" "$2")" || + image_die "cannot retieve kernel partition offset" + KPART_SECTORS="$(image_part_size "$KPART" "$2")" || + image_die "cannot retieve kernel partition size" + ROOT_OFFSET="$(image_part_offset "$ROOT_PART" "$3")" || + image_die "cannot retieve root partition offset" + ROOT_SECTORS="$(image_part_size "$ROOT_PART" "$3")" || + image_die "cannot retieve root partition size" KPART_SIZE=$((KPART_SECTORS * 512)) fi @@ -91,10 +91,10 @@ KPART_SIZE_SIGNATURE="$(printf "%016x" "$KPART_SIZE" | CS_AND_RET_CODES="$( (echo -en "$KPART_SIZE_SIGNATURE" echo "Compressing kernel..." >&2 - dump_partial_file "$KPART" "$KPART_OFFSET" "$KPART_SECTORS" + image_dump_partial_file "$KPART" "$KPART_OFFSET" "$KPART_SECTORS" echo "Compressing rootfs..." >&2 - dump_partial_file "$ROOT_PART" "$ROOT_OFFSET" "$ROOT_SECTORS") | - gzip_compress -9 -c | + image_dump_partial_file "$ROOT_PART" "$ROOT_OFFSET" "$ROOT_SECTORS") | + image_gzip_compress -c -9 | tee "$FINAL_OUT_FILE" | openssl sha1 -binary | openssl base64 | diff --git a/mod_for_factory_scripts/400configAutotest b/mod_for_factory_scripts/400configAutotest index f642e38729..74261a5512 100755 --- a/mod_for_factory_scripts/400configAutotest +++ b/mod_for_factory_scripts/400configAutotest @@ -13,7 +13,7 @@ if [ -f "${GLOBAL_CONFIG}" ]; then exit 1 fi -cat > "${GLOBAL_CONFIG}" <"${GLOBAL_CONFIG}" <