cros_utils: refine factory and imaging related scripts

Major changes are:
 - prefix functions in cros_image_common with 'image' to avoid naming conflicts
 - use awk to replace grep+sed+cut
 - use case to handle board name list
 - factory_setup.sh should return error if applying patch failed
 - refine for shell script coding style guide:
   * replace `` with $()
   * quote variables
   * replace "! -z" by "-n"
   * no space between redirection symbol (>) and target name (eg, >>filename)
   * 1> should be simply >
   * no trailing \ for && and ||

BUG=chrome-os-partner:1583
TEST=manually verified:
 ./image_to_usb.sh --factory # calls mod_image_for_test --factory, and works fine
 ./make_factory_package.sh --factory PATH_TO_IMAGE --release PATH_TO_IMAGE ...
  # factory payloads were created successfully

Change-Id: I6bb10bdfb12cbdb14e9816b3ad72dfe4b7b0472f

Review URL: http://codereview.chromium.org/5168001
This commit is contained in:
Hung-Te Lin 2010-11-18 14:47:38 +08:00
parent 711efd8aec
commit 062947926e
5 changed files with 194 additions and 119 deletions

View File

@ -8,19 +8,20 @@
# especially for being redistributed into platforms without complete Chromium OS
# developing environment.
# Check if given command is available in current system
has_command() {
# Checks if given command is available in current system
image_has_command() {
type "$1" >/dev/null 2>&1
}
err_die() {
# Prints error message and exit as 1 (error)
image_die() {
echo "ERROR: $@" >&2
exit 1
}
# Finds the best gzip compressor and invoke it.
gzip_compress() {
if has_command pigz; then
# Finds the best gzip compressor and invoke it
image_gzip_compress() {
if image_has_command pigz; then
# echo " ** Using parallel gzip **" >&2
# Tested with -b 32, 64, 128(default), 256, 1024, 16384, and -b 32 (max
# window size of Deflate) seems to be the best in output size.
@ -30,43 +31,58 @@ gzip_compress() {
fi
}
# Finds the best bzip2 compressor and invoke it
image_bzip2_compress() {
if image_has_command pbzip2; then
pbzip2 "$@"
else
bzip2 "$@"
fi
}
# Finds if current system has tools for part_* commands
has_part_tools() {
has_command cgpt || has_command parted
image_has_part_tools() {
image_has_command cgpt || image_has_command parted
}
# Finds the best partition tool and print partition offset
part_offset() {
image_part_offset() {
local file="$1"
local partno="$2"
local unpack_file="$(dirname "$file")/unpack_partitions.sh"
if has_command cgpt; then
# TODO parted is available on most Linux so we may deprecate other code path
if image_has_command cgpt; then
cgpt show -b -i "$partno" "$file"
elif has_command parted; then
parted -m "$file" unit s print |
grep "^$partno:" | cut -d ':' -f 2 | sed 's/s$//'
elif image_has_command parted; then
parted -m "$file" unit s print | awk -F ':' "/^$partno:/ { print int(\$2) }"
elif [ -f "$unpack_file" ]; then
awk "/ $partno *Label:/ { print \$2 }" "$unpack_file"
else
exit 1
fi
}
# Finds the best partition tool and print partition size
part_size() {
image_part_size() {
local file="$1"
local partno="$2"
local unpack_file="$(dirname "$file")/unpack_partitions.sh"
if has_command cgpt; then
# TODO parted is available on most Linux so we may deprecate other code path
if image_has_command cgpt; then
cgpt show -s -i "$partno" "$file"
elif has_command parted; then
parted -m "$file" unit s print |
grep "^$partno:" | cut -d ':' -f 4 | sed 's/s$//'
elif image_has_command parted; then
parted -m "$file" unit s print | awk -F ':' "/^$partno:/ { print int(\$4) }"
elif [ -s "$unpack_file" ]; then
awk "/ $partno *Label:/ { print \$3 }" "$unpack_file"
else
exit 1
fi
}
# Dumps a file by given offset and size (in sectors)
dump_partial_file() {
image_dump_partial_file() {
local file="$1"
local offset="$2"
local sectors="$3"
@ -82,10 +98,10 @@ dump_partial_file() {
bs=$((bs * buffer_ratio))
fi
if has_command pv; then
if image_has_command pv; then
dd if="$file" bs=$bs skip="$offset" count="$sectors" \
oflag=sync status=noxfer 2>/dev/null |
pv -ptreb -B 4m -s $((sectors * $bs))
pv -ptreb -B $bs -s $((sectors * bs))
else
dd if="$file" bs=$bs skip="$offset" count="$sectors" \
oflag=sync status=noxfer 2>/dev/null
@ -93,14 +109,62 @@ dump_partial_file() {
}
# Dumps a specific partition from given image file
dump_partition() {
image_dump_partition() {
local file="$1"
local part_num="$2"
local offset="$(part_offset "$file" "$part_num")" ||
err_die "failed to dump partition #$part_num from: $file"
local size="$(part_size "$file" "$part_num")" ||
err_die "failed to dump partition #$part_num from: $file"
local offset="$(image_part_offset "$file" "$part_num")" ||
image_die "failed to find partition #$part_num from: $file"
local size="$(image_part_size "$file" "$part_num")" ||
image_die "failed to find partition #$part_num from: $file"
dump_partial_file "$file" "$offset" "$size"
image_dump_partial_file "$file" "$offset" "$size"
}
# Maps a specific partition from given image file to a loop device
image_map_partition() {
local file="$1"
local part_num="$2"
local offset="$(image_part_offset "$file" "$part_num")" ||
image_die "failed to find partition #$part_num from: $file"
local size="$(image_part_size "$file" "$part_num")" ||
image_die "failed to find partition #$part_num from: $file"
losetup --offset $((offset * 512)) --sizelimit=$((size * 512)) \
-f --show "$file"
}
# Unmaps a loop device created by image_map_partition
image_unmap_partition() {
local map_point="$1"
losetup -d "$map_point"
}
# Mounts a specific partition inside a given image file
image_mount_partition() {
local file="$1"
local part_num="$2"
local mount_point="$3"
local mount_opt="$4"
local offset="$(image_part_offset "$file" "$part_num")" ||
image_die "failed to find partition #$part_num from: $file"
local size="$(image_part_size "$file" "$part_num")" ||
image_die "failed to find partition #$part_num from: $file"
if [ -z "$mount_opt" ]; then
# by default, mount as read-only.
mount_opt=",ro"
fi
mount \
-o "loop,offset=$((offset * 512)),sizelimit=$((size * 512)),$mount_opt" \
"$file" \
"$mount_point"
}
# Unmounts a partition mount point by mount_partition
image_umount_partition() {
local mount_point="$1"
umount -d "$mount_point"
}

View File

@ -39,32 +39,33 @@ DEFINE_string subfolder "" \
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
if [ ! -f "${FLAGS_release}" ] ; then
if [ ! -f "${FLAGS_release}" ]; then
echo "Cannot find image file ${FLAGS_release}"
exit 1
fi
if [ ! -f "${FLAGS_factory}" ] ; then
if [ ! -f "${FLAGS_factory}" ]; then
echo "Cannot find image file ${FLAGS_factory}"
exit 1
fi
if [ ! -z "${FLAGS_firmware_updater}" ] && \
[ ! -f "${FLAGS_firmware_updater}" ] ; then
if [ -n "${FLAGS_firmware_updater}" ] &&
[ ! -f "${FLAGS_firmware_updater}" ]; then
echo "Cannot find firmware file ${FLAGS_firmware_updater}"
exit 1
fi
# Convert args to paths. Need eval to un-quote the string so that shell
# chars like ~ are processed; just doing FOO=`readlink -f ${FOO}` won't work.
OMAHA_DIR=${SRC_ROOT}/platform/dev
OMAHA_DATA_DIR=${OMAHA_DIR}/static/
OMAHA_DIR="${SRC_ROOT}/platform/dev"
OMAHA_CONF="${OMAHA_DIR}/miniomaha.conf"
OMAHA_DATA_DIR="${OMAHA_DIR}/static/"
# Note: The subfolder flag can only append configs. That means you will need
# to have unique board IDs for every time you run. If you delete miniomaha.conf
# you can still use this flag and it will start fresh.
if [ -n "${FLAGS_subfolder}" ] ; then
OMAHA_DATA_DIR=${OMAHA_DIR}/static/${FLAGS_subfolder}/
if [ -n "${FLAGS_subfolder}" ]; then
OMAHA_DATA_DIR="${OMAHA_DIR}/static/${FLAGS_subfolder}/"
fi
if [ ${INSIDE_CHROOT} -eq 0 ]; then
@ -74,20 +75,20 @@ if [ ${INSIDE_CHROOT} -eq 0 ]; then
fi
# Use this image as the source image to copy
RELEASE_DIR=`dirname ${FLAGS_release}`
FACTORY_DIR=`dirname ${FLAGS_factory}`
RELEASE_IMAGE=`basename ${FLAGS_release}`
FACTORY_IMAGE=`basename ${FLAGS_factory}`
RELEASE_DIR="$(dirname "${FLAGS_release}")"
FACTORY_DIR="$(dirname "${FLAGS_factory}")"
RELEASE_IMAGE="$(basename "${FLAGS_release}")"
FACTORY_IMAGE="$(basename "${FLAGS_factory}")"
prepare_omaha() {
sudo rm -rf ${OMAHA_DATA_DIR}/rootfs-test.gz
sudo rm -rf ${OMAHA_DATA_DIR}/rootfs-release.gz
rm -rf ${OMAHA_DATA_DIR}/efi.gz
rm -rf ${OMAHA_DATA_DIR}/oem.gz
rm -rf ${OMAHA_DATA_DIR}/state.gz
if [ ! -f "${OMAHA_DATA_DIR}" ] ; then
mkdir -p ${OMAHA_DATA_DIR}
sudo rm -rf "${OMAHA_DATA_DIR}/rootfs-test.gz"
sudo rm -rf "${OMAHA_DATA_DIR}/rootfs-release.gz"
rm -rf "${OMAHA_DATA_DIR}/efi.gz"
rm -rf "${OMAHA_DATA_DIR}/oem.gz"
rm -rf "${OMAHA_DATA_DIR}/state.gz"
if [ ! -d "${OMAHA_DATA_DIR}" ]; then
mkdir -p "${OMAHA_DATA_DIR}"
fi
}
@ -102,12 +103,12 @@ prepare_dir() {
compress_and_hash_memento_image() {
local input_file="$1"
if has_part_tools; then
sudo "${SCRIPTS_DIR}/mk_memento_images.sh" "$input_file" 2 3 |
if [ -n "${IMAGE_IS_UNPACKED}" ]; then
sudo "${SCRIPTS_DIR}/mk_memento_images.sh" part_2 part_3 |
grep hash |
awk '{print $4}'
else
sudo "${SCRIPTS_DIR}/mk_memento_images.sh" part_2 part_3 |
sudo "${SCRIPTS_DIR}/mk_memento_images.sh" "$input_file" 2 3 |
grep hash |
awk '{print $4}'
fi
@ -119,12 +120,12 @@ compress_and_hash_file() {
if [ -z "$input_file" ]; then
# Runs as a pipe processor
gzip_compress -c -9 |
image_gzip_compress -c -9 |
tee "$output_file" |
openssl sha1 -binary |
openssl base64
else
gzip_compress -c -9 "$input_file" |
image_gzip_compress -c -9 "$input_file" |
tee "$output_file" |
openssl sha1 -binary |
openssl base64
@ -136,30 +137,38 @@ compress_and_hash_partition() {
local part_num="$2"
local output_file="$3"
if has_part_tools; then
dump_partition "$input_file" "$part_num" |
compress_and_hash_file "" "$output_file"
else
if [ -n "${IMAGE_IS_UNPACKED}" ]; then
compress_and_hash_file "part_$part_num" "$output_file"
else
image_dump_partition "$input_file" "$part_num" |
compress_and_hash_file "" "$output_file"
fi
}
# Clean up stale config and data files.
prepare_omaha
# Decide if we should unpack partition
if image_has_part_tools; then
IMAGE_IS_UNPACKED=
else
#TODO(hungte) Currently we run unpack_partitions.sh if part_tools are not
# found. If the format of unpack_partitions.sh is reliable, we can prevent
# creating temporary files. See image_part_offset for more information.
echo "WARNING: cannot find partition tools. Using unpack_partitions.sh." >&2
IMAGE_IS_UNPACKED=1
fi
# Get the release image.
pushd ${RELEASE_DIR} > /dev/null
pushd "${RELEASE_DIR}" >/dev/null
echo "Generating omaha release image from ${FLAGS_release}"
echo "Generating omaha factory image from ${FLAGS_factory}"
echo "Output omaha image to ${OMAHA_DATA_DIR}"
echo "Output omaha config to ${OMAHA_DIR}/miniomaha.conf"
echo "Output omaha config to ${OMAHA_CONF}"
prepare_dir
if ! has_part_tools; then
#TODO(hungte) we can still avoid running unpack_partitions.sh
# by $(cat unpack_partitions.sh | grep Label | sed "s/#//" | grep ${name}" |
# awk '{ print $1}') to fetch offset/size.
if [ -n "${IMAGE_IS_UNPACKED}" ]; then
echo "Unpacking image ${RELEASE_IMAGE} ..." >&2
sudo ./unpack_partitions.sh "${RELEASE_IMAGE}" 2>/dev/null
fi
@ -167,24 +176,24 @@ fi
release_hash="$(compress_and_hash_memento_image "${RELEASE_IMAGE}")"
sudo chmod a+rw update.gz
mv update.gz rootfs-release.gz
mv rootfs-release.gz ${OMAHA_DATA_DIR}
mv rootfs-release.gz "${OMAHA_DATA_DIR}"
echo "release: ${release_hash}"
oem_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 8 "oem.gz")"
mv oem.gz ${OMAHA_DATA_DIR}
mv oem.gz "${OMAHA_DATA_DIR}"
echo "oem: ${oem_hash}"
efi_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 12 "efi.gz")"
mv efi.gz ${OMAHA_DATA_DIR}
mv efi.gz "${OMAHA_DATA_DIR}"
echo "efi: ${efi_hash}"
popd > /dev/null
popd >/dev/null
# Go to retrieve the factory test image.
pushd ${FACTORY_DIR} > /dev/null
pushd "${FACTORY_DIR}" >/dev/null
prepare_dir
if ! has_part_tools; then
if [ -n "${IMAGE_IS_UNPACKED}" ]; then
echo "Unpacking image ${FACTORY_IMAGE} ..." >&2
sudo ./unpack_partitions.sh "${FACTORY_IMAGE}" 2>/dev/null
fi
@ -192,16 +201,16 @@ fi
test_hash="$(compress_and_hash_memento_image "${FACTORY_IMAGE}")"
sudo chmod a+rw update.gz
mv update.gz rootfs-test.gz
mv rootfs-test.gz ${OMAHA_DATA_DIR}
mv rootfs-test.gz "${OMAHA_DATA_DIR}"
echo "test: ${test_hash}"
state_hash="$(compress_and_hash_partition "${FACTORY_IMAGE}" 1 "state.gz")"
mv state.gz ${OMAHA_DATA_DIR}
mv state.gz "${OMAHA_DATA_DIR}"
echo "state: ${state_hash}"
popd > /dev/null
popd >/dev/null
if [ ! -z ${FLAGS_firmware_updater} ] ; then
if [ -n "${FLAGS_firmware_updater}" ]; then
SHELLBALL="${FLAGS_firmware_updater}"
if [ ! -f "$SHELLBALL" ]; then
echo "Failed to find firmware updater: $SHELLBALL."
@ -209,70 +218,70 @@ if [ ! -z ${FLAGS_firmware_updater} ] ; then
fi
firmware_hash="$(compress_and_hash_file "$SHELLBALL" "firmware.gz")"
mv firmware.gz ${OMAHA_DATA_DIR}
mv firmware.gz "${OMAHA_DATA_DIR}"
echo "firmware: ${firmware_hash}"
fi
# If the file does exist and we are using the subfolder flag we are going to
# append another config.
if [ -n "${FLAGS_subfolder}" ] && \
[ -f "${OMAHA_DIR}"/miniomaha.conf"" ] ; then
if [ -n "${FLAGS_subfolder}" ] &&
[ -f "${OMAHA_CONF}" ]; then
# Remove the ']' from the last line of the file so we can add another config.
while [ -s "${OMAHA_DIR}/miniomaha.conf" ]; do
while [ -s "${OMAHA_CONF}" ]; do
# If the last line is null
if [ -z "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" ]; then
sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
elif [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" != ']' ]; then
sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
if [ -z "$(tail -1 "${OMAHA_CONF}")" ]; then
sed -i '$d' "${OMAHA_CONF}"
elif [ "$(tail -1 "${OMAHA_CONF}")" != ']' ]; then
sed -i '$d' "${OMAHA_CONF}"
else
break
fi
done
# Remove the last ]
if [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" = ']' ]; then
sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
if [ "$(tail -1 "${OMAHA_CONF}")" = ']' ]; then
sed -i '$d' "${OMAHA_CONF}"
fi
# If the file is empty, create it from scratch
if [ ! -s "${OMAHA_DIR}/miniomaha.conf" ]; then
echo "config = [" > "${OMAHA_DIR}/miniomaha.conf"
if [ ! -s "${OMAHA_CONF}" ]; then
echo "config = [" >"${OMAHA_CONF}"
fi
else
echo "config = [" > "${OMAHA_DIR}/miniomaha.conf"
echo "config = [" >"${OMAHA_CONF}"
fi
if [ -n "${FLAGS_subfolder}" ] ; then
if [ -n "${FLAGS_subfolder}" ]; then
subfolder="${FLAGS_subfolder}/"
fi
echo -n "{
'qual_ids': set([\"${FLAGS_board}\"]),
'factory_image': '"${subfolder}"rootfs-test.gz',
'factory_image': '${subfolder}rootfs-test.gz',
'factory_checksum': '${test_hash}',
'release_image': '"${subfolder}"rootfs-release.gz',
'release_image': '${subfolder}rootfs-release.gz',
'release_checksum': '${release_hash}',
'oempartitionimg_image': '"${subfolder}"oem.gz',
'oempartitionimg_image': '${subfolder}oem.gz',
'oempartitionimg_checksum': '${oem_hash}',
'efipartitionimg_image': '"${subfolder}"efi.gz',
'efipartitionimg_image': '${subfolder}efi.gz',
'efipartitionimg_checksum': '${efi_hash}',
'stateimg_image': '"${subfolder}"state.gz',
'stateimg_checksum': '${state_hash}'," >> ${OMAHA_DIR}/miniomaha.conf
'stateimg_image': '${subfolder}state.gz',
'stateimg_checksum': '${state_hash}'," >>"${OMAHA_CONF}"
if [ ! -z "${FLAGS_firmware_updater}" ] ; then
if [ -n "${FLAGS_firmware_updater}" ] ; then
echo -n "
'firmware_image': '"${subfolder}"firmware.gz',
'firmware_checksum': '${firmware_hash}'," >> ${OMAHA_DIR}/miniomaha.conf
'firmware_image': '${subfolder}firmware.gz',
'firmware_checksum': '${firmware_hash}'," >>"${OMAHA_CONF}"
fi
echo -n "
},
]
" >> ${OMAHA_DIR}/miniomaha.conf
" >>"${OMAHA_CONF}"
echo "The miniomaha server lives in src/platform/dev"
echo "to validate the configutarion, run:"
echo " python2.6 devserver.py --factory_config miniomaha.conf \
--validate_factory_config"
echo "To run the server:"
echo " python2.6 devserver.py --factory_config miniomaha.conf"
echo "The miniomaha server lives in src/platform/dev.
To validate the configutarion, run:
python2.6 devserver.py --factory_config miniomaha.conf \
--validate_factory_config
To run the server:
python2.6 devserver.py --factory_config miniomaha.conf"

View File

@ -28,13 +28,13 @@ if [ "$CROS_GENERATE_UPDATE_PAYLOAD_CALLED" != "1" ]; then
echo " Please run that script with --help to see how to use it."
fi
if ! has_command pigz; then
if ! image_has_command pigz; then
(echo "WARNING:"
echo " Your system does not have pigz (parallel gzip) installed."
echo " COMPRESSING WILL BE VERY SLOW. It is recommended to install pigz"
if has_command apt-get; then
if image_has_command apt-get; then
echo " by 'sudo apt-get install pigz'."
elif has_command emerge; then
elif image_has_command emerge; then
echo " by 'sudo emerge pigz'."
fi) >&2
fi
@ -58,14 +58,14 @@ else
# chromiumos_img kern_part_no rootfs_part_no
KPART="$1"
ROOT_PART="$1"
KPART_OFFSET="$(part_offset "$KPART" "$2")" ||
err_die "cannot retieve kernel partition offset"
KPART_SECTORS="$(part_size "$KPART" "$2")" ||
err_die "cannot retieve kernel partition size"
ROOT_OFFSET="$(part_offset "$ROOT_PART" "$3")" ||
err_die "cannot retieve root partition offset"
ROOT_SECTORS="$(part_size "$ROOT_PART" "$3")" ||
err_die "cannot retieve root partition size"
KPART_OFFSET="$(image_part_offset "$KPART" "$2")" ||
image_die "cannot retieve kernel partition offset"
KPART_SECTORS="$(image_part_size "$KPART" "$2")" ||
image_die "cannot retieve kernel partition size"
ROOT_OFFSET="$(image_part_offset "$ROOT_PART" "$3")" ||
image_die "cannot retieve root partition offset"
ROOT_SECTORS="$(image_part_size "$ROOT_PART" "$3")" ||
image_die "cannot retieve root partition size"
KPART_SIZE=$((KPART_SECTORS * 512))
fi
@ -91,10 +91,10 @@ KPART_SIZE_SIGNATURE="$(printf "%016x" "$KPART_SIZE" |
CS_AND_RET_CODES="$(
(echo -en "$KPART_SIZE_SIGNATURE"
echo "Compressing kernel..." >&2
dump_partial_file "$KPART" "$KPART_OFFSET" "$KPART_SECTORS"
image_dump_partial_file "$KPART" "$KPART_OFFSET" "$KPART_SECTORS"
echo "Compressing rootfs..." >&2
dump_partial_file "$ROOT_PART" "$ROOT_OFFSET" "$ROOT_SECTORS") |
gzip_compress -9 -c |
image_dump_partial_file "$ROOT_PART" "$ROOT_OFFSET" "$ROOT_SECTORS") |
image_gzip_compress -c -9 |
tee "$FINAL_OUT_FILE" |
openssl sha1 -binary |
openssl base64 |

View File

@ -13,7 +13,7 @@ if [ -f "${GLOBAL_CONFIG}" ]; then
exit 1
fi
cat > "${GLOBAL_CONFIG}" <<EOF
cat >"${GLOBAL_CONFIG}" <<EOF
[CLIENT]
drop_caches: False
drop_caches_between_iterations: False

View File

@ -5,9 +5,11 @@
# found in the LICENSE file.
echo "Modifying image for factory test..."
set -e
for SCRIPT in \
${GCLIENT_ROOT}/src/scripts/mod_for_factory_scripts/[0-9][0-9][0-9]*[!$~]
SCRIPT_BASE="${GCLIENT_ROOT}/src/scripts/mod_for_factory_scripts/"
for SCRIPT in "${SCRIPT_BASE}"[0-9][0-9][0-9]*[!$~]
do
${SCRIPT}
echo "Apply $(basename "${SCRIPT}")..."
"${SCRIPT}"
done