From 033cf224f152936205333ba1448c02e988d7b7cb Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 24 Jul 2013 22:25:33 -0400 Subject: [PATCH] feat(image_to_vm): Refactor to make adding/changing vm formats easier. The old script was heading towards spaghetti code realm. This breaks up all the image variations such as hybrid MBR, OEM packages, etc into configuration options and small functions that actually do the work. All this is in the new vm_image_util.sh library but the command line parsing and overall procedure remains in image_to_vm.sh As part of this we gain support for putting some qemu options in a config file as well as Xen virtual machines using pygrub and pvgrub. Lots of generally unused options have been removed to simplify things and keep output file names consistent. --- build_library/vm_image_util.sh | 407 +++++++++++++++++++++++++++++++++ common.sh | 7 + image_to_vm.sh | 213 ++++------------- 3 files changed, 454 insertions(+), 173 deletions(-) create mode 100644 build_library/vm_image_util.sh diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh new file mode 100644 index 0000000000..3b613a0ac4 --- /dev/null +++ b/build_library/vm_image_util.sh @@ -0,0 +1,407 @@ +# Copyright (c) 2013 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Format options. Each variable uses the form IMG__. +# Default values use the format IMG_DEFAULT_. + +VALID_IMG_TYPES=( + ami + qemu + rackspace + virtualbox + vmware + xen +) + +# Set at runtime to one of the above types +VM_IMG_TYPE=DEFAULT + +# Set at runtime to the source and destination image paths +VM_SRC_IMG= +VM_TMP_IMG= +VM_TMP_DIR= +VM_DST_IMG= +VM_README= +VM_NAME= +VM_UUID= + +## DEFAULT +# If set to 1 use a hybrid GPT/MBR format instead of plain GPT +IMG_DEFAULT_HYBRID_MBR=0 + +# If set install the given package name to the OEM partition +IMG_DEFAULT_OEM_PACKAGE= + +# Name of the target image format. +# May be raw, vdi (virtualbox), or vmdk (vmware) +IMG_DEFAULT_DISK_FORMAT=raw + +# Name of the target config format, default is no config +IMG_DEFAULT_CONF_FORMAT= + +# Memory size to use in any config files +IMG_DEFAULT_MEM=1024 + +## qemu +IMG_qemu_CONF_FORMAT=qemu + +## xen +# Hybrid is required by pvgrub (pygrub supports GPT but we support both) +IMG_xen_HYBRID_MBR=1 +IMG_xen_CONF_FORMAT=xl + +## virtualbox +IMG_virtualbox_DISK_FORMAT=vdi + +## vmware +IMG_vmware_DISK_FORMAT=vmdk +IMG_vmware_CONF_FORMAT=vmx + +## ami +IMG_ami_HYBRID_MBR=1 +IMG_ami_OEM_PACKAGE=oem-ami + +## rackspace +# TODO: package doesn't exist yet +#IMG_rackspace_OEM_PACKAGE=oem-rackspace + +########################################################### + +# Validate and set the vm type to use for the rest of the functions +set_vm_type() { + local vm_type="$1" + local valid_type + for valid_type in "${VALID_IMG_TYPES[@]}"; do + if [[ "${vm_type}" == "${valid_type}" ]]; then + VM_IMG_TYPE="${vm_type}" + return 0 + fi + done + return 1 +} + +# Validate and set source vm image path +set_vm_paths() { + local src_dir="$1" + local dst_dir="$2" + local src_name="$3" + + VM_SRC_IMG="${src_dir}/${src_name}" + if [[ ! -f "${VM_SRC_IMG}" ]]; then + die "Source image does not exist: $VM_SRC_IMG" + fi + + local dst_name="$(_src_to_dst_name "${src_name}" "_image.$(_disk_ext)")" + VM_DST_IMG="${dst_dir}/${dst_name}" + VM_TMP_DIR="${dst_dir}/${dst_name}.vmtmpdir" + VM_TMP_IMG="${VM_TMP_DIR}/disk_image.bin" + + # If src_dir happens to be in the build directory figure out the version + # from the directory name and use it in the vm name for config files. + VM_NAME=$(_src_to_dst_name "${src_name}" "") + if [[ "${src_dir}" =~ /build/images/${BOARD}/\d+\.\d+\.[^/]*$ ]]; then + VM_NAME+="-$(basename ${src_dir})" + fi + + VM_UUID=$(uuidgen) + VM_README="${dst_dir}/$(_src_to_dst_name "${src_name}" ".README")" +} + +_get_vm_opt() { + local opt="$1" + local type_opt="IMG_${VM_IMG_TYPE}_${opt}" + local default_opt="IMG_DEFAULT_${opt}" + echo "${!type_opt:-${!default_opt}}" +} + +# Translate source image names to output names. +# This keeps naming consistent across all vm types. +_src_to_dst_name() { + local src_img="$1" + local suffix="$2" + echo "${1%_image.bin}_${VM_IMG_TYPE}${suffix}" +} + +# Get the proper disk format extension. +_disk_ext() { + local disk_format=$(_get_vm_opt DISK_FORMAT) + case ${disk_format} in + raw) echo bin;; + *) echo "${disk_format}";; + esac +} + +# Unpack the source disk to individual partitions, optionally using an +# alternate filesystem image for the state partition instead of the one +# from VM_SRC_IMG. Start new image using the given disk layout. +unpack_source_disk() { + local disk_layout="$1" + local alternate_state_image="$2" + + if [[ -n "${alternate_state_image}" && ! -f "${alternate_state_image}" ]] + then + die "State image does not exist: $alternate_state_image" + fi + + info "Unpacking source image to $(relpath "${VM_TMP_DIR}")" + + rm -rf "${VM_TMP_DIR}" + mkdir -p "${VM_TMP_DIR}" + + pushd "${VM_TMP_DIR}" >/dev/null + local src_dir=$(dirname "${VM_SRC_IMG}") + "${src_dir}/unpack_partitions.sh" "${VM_SRC_IMG}" + popd >/dev/null + + # Partition paths that have been unpacked from VM_SRC_IMG + TEMP_ESP="${VM_TMP_DIR}"/part_${NUM_ESP} + TEMP_OEM="${VM_TMP_DIR}"/part_${NUM_OEM} + TEMP_ROOTFS="${VM_TMP_DIR}"/part_${NUM_ROOTFS_A} + TEMP_STATE="${VM_TMP_DIR}"/part_${NUM_STATEFUL} + # Copy the replacement STATE image if it is set + if [[ -n "${alternate_state_image}" ]]; then + cp "${alternate_state_image}" "${TEMP_STATE}" + fi + + TEMP_PMBR="${VM_TMP_DIR}"/pmbr + dd if="${VM_SRC_IMG}" of="${TEMP_PMBR}" bs=512 count=1 + + info "Initializing new partition table..." + TEMP_PARTITION_SCRIPT="${VM_TMP_DIR}/partition_script.sh" + write_partition_script "${disk_layout}" "${TEMP_PARTITION_SCRIPT}" + . "${TEMP_PARTITION_SCRIPT}" + write_partition_table "${VM_TMP_IMG}" "${TEMP_PMBR}" +} + +resize_state_partition() { + local size_in_bytes="$1" + local size_in_sectors=$(( size_in_bytes / 512 )) + local size_in_mb=$(( size_in_bytes / 1024 / 1024 )) + local original_size=$(stat -c%s "${TEMP_STATE}") + + if [[ "${original_size}" -gt "${size_in_bytes}" ]]; then + die "Cannot resize stateful image to smaller than original." + fi + + info "Resizing stateful partition to ${size_in_mb}MB" + /sbin/e2fsck -pf "${TEMP_STATE}" + /sbin/resize2fs "${TEMP_STATE}" "${size_in_sectors}s" +} + +# If the current type defines a oem package install it to the given fs image. +install_oem_package() { + local oem_pkg=$(_get_vm_opt OEM_PACKAGE) + local oem_mnt="${VM_TMP_DIR}/oem" + + if [[ -z "${oem_pkg}" ]]; then + return 0 + fi + + info "Installing ${oem_pkg} to OEM partition" + mkdir -p "${oem_mnt}" + sudo mount -o loop "${TEMP_OEM}" "${oem_mnt}" + + # TODO(polvi): figure out how to keep portage from putting these + # portage files on disk, we don't need or want them. + emerge-${BOARD} --root="${oem_mnt}" --root-deps=rdeps "${oem_pkg}" + + sudo umount "${oem_mnt}" + rm -rf "${oem_mnt}" +} + +# Write the vm disk image to the target directory in the proper format +write_vm_disk() { + info "Writing partitions to new disk image" + dd if="${TEMP_ROOTFS}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ + seek=$(partoffset ${VM_TMP_IMG} ${NUM_ROOTFS_A}) + dd if="${TEMP_STATE}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ + seek=$(partoffset ${VM_TMP_IMG} ${NUM_STATEFUL}) + dd if="${TEMP_ESP}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ + seek=$(partoffset ${VM_TMP_IMG} ${NUM_ESP}) + dd if="${TEMP_OEM}" of="${VM_TMP_IMG}" conv=notrunc bs=512 \ + seek=$(partoffset ${VM_TMP_IMG} ${NUM_OEM}) + + if [[ $(_get_vm_opt HYBRID_MBR) -eq 1 ]]; then + info "Creating hybrid MBR" + _write_hybrid_mbr "${VM_TMP_IMG}" + fi + + local disk_format=$(_get_vm_opt DISK_FORMAT) + info "Writing $disk_format image to $(relpath "${VM_DST_IMG}")" + _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}" +} + +_write_hybrid_mbr() { + # TODO(marineam): Switch to sgdisk + /usr/sbin/gdisk "$1" <"${conf_path}" <"${VM_README}" <"${vmx_path}" < "${pygrub}" + echo 'bootloader = "pygrub"' >> "${pygrub}" + + echo '# Xen PV config using pvgrub' > "${pvgrub}" + echo 'kernel = "/usr/lib/xen/boot/pv-grub-x86_64.gz"' >> "${pvgrub}" + echo 'extra = "(hd0,0)/boot/grub/menu.lst"' >> "${pvgrub}" + + # The rest is the same + tee -a "${pygrub}" >> "${pvgrub}" < "${VM_README}" </dev/null -"${FLAGS_from}/unpack_partitions.sh" "${SRC_IMAGE}" -popd >/dev/null - -# Fix the kernel command line -TEMP_ESP="${TEMP_DIR}"/part_${NUM_ESP} -TEMP_OEM="${TEMP_DIR}"/part_${NUM_OEM} -TEMP_ROOTFS="${TEMP_DIR}"/part_${NUM_ROOTFS_A} -TEMP_STATE="${TEMP_DIR}"/part_${NUM_STATEFUL} -if [ -n "${FLAGS_state_image}" ]; then - TEMP_STATE="${FLAGS_state_image}" +if [ ${FLAGS_prod_image} -eq ${FLAGS_TRUE} ]; then + set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${COREOS_PRODUCTION_IMAGE_NAME}" +elif [ ${FLAGS_test_image} -eq ${FLAGS_TRUE} ]; then + set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${CHROMEOS_TEST_IMAGE_NAME}" else - STATEFUL_SIZE_BYTES=$(get_filesystem_size "${FLAGS_disk_layout}" ${NUM_STATEFUL}) - STATEFUL_SIZE_MEGABYTES=$(( STATEFUL_SIZE_BYTES / 1024 / 1024 )) - original_image_size=$(stat -c%s "${TEMP_STATE}") - if [ "${original_image_size}" -gt "${STATEFUL_SIZE_BYTES}" ]; then - die "Cannot resize stateful image to smaller than original. Exiting." - fi - - echo "Resizing stateful partition to ${STATEFUL_SIZE_MEGABYTES}MB" - # Extend the original file size to the new size. - sudo e2fsck -pf "${TEMP_STATE}" - sudo resize2fs "${TEMP_STATE}" ${STATEFUL_SIZE_MEGABYTES}M + # Use the standard image + set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${CHROMEOS_IMAGE_NAME}" fi -# handle OEM stuff if needed -TEMP_OEM_MNT="${TEMP_DIR}"/oem_mnt -mkdir -p $TEMP_OEM_MNT -sudo mount -o loop ${TEMP_OEM} ${TEMP_OEM_MNT} +locate_gpt +legacy_offset_size_export ${VM_SRC_IMG} -# oem hacks -if [ "${FLAGS_format}" == "ami" ]; then - # TODO(polvi): figure out how to keep portage from putting these - # portage files on disk, we don't need or want them. - emerge-${BOARD} --root="${TEMP_OEM_MNT}" --root-deps=rdeps oem-ami -elif [ "${FLAGS_format}" == "rackspace" ]; then - emerge-${BOARD} --root="${TEMP_OEM_MNT}" --root-deps=rdeps oem-rackspace -fi +# Make sure things are cleaned up on failure +trap vm_cleanup EXIT -sudo umount ${TEMP_OEM_MNT} -rm -rf ${TEMP_OEM_MNT} +# Unpack image, using alternate state image if defined +# Resize to use all available space in new disk layout +STATEFUL_SIZE=$(get_filesystem_size "${FLAGS_disk_layout}" ${NUM_STATEFUL}) +unpack_source_disk "${FLAGS_disk_layout}" "${FLAGS_state_image}" +resize_state_partition "${STATEFUL_SIZE}" +# Optionally install any OEM packages +install_oem_package -TEMP_PMBR="${TEMP_DIR}"/pmbr -dd if="${SRC_IMAGE}" of="${TEMP_PMBR}" bs=512 count=1 +# Changes done, glue it together +write_vm_disk +write_vm_conf "${FLAGS_mem}" -# Set up a new partition table -PARTITION_SCRIPT_PATH=$( tempfile ) -write_partition_script "${FLAGS_disk_layout}" "${PARTITION_SCRIPT_PATH}" -. "${PARTITION_SCRIPT_PATH}" -write_partition_table "${TEMP_IMG}" "${TEMP_PMBR}" -rm "${PARTITION_SCRIPT_PATH}" +vm_cleanup +trap - EXIT -# Copy into the partition parts of the file -dd if="${TEMP_ROOTFS}" of="${TEMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${TEMP_IMG} ${NUM_ROOTFS_A}) -dd if="${TEMP_STATE}" of="${TEMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${TEMP_IMG} ${NUM_STATEFUL}) -dd if="${TEMP_ESP}" of="${TEMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${TEMP_IMG} ${NUM_ESP}) -dd if="${TEMP_OEM}" of="${TEMP_IMG}" conv=notrunc bs=512 \ - seek=$(partoffset ${TEMP_IMG} ${NUM_OEM}) - -echo Creating final image -# Convert image to output format -if [ "${FLAGS_format}" = "virtualbox" -o "${FLAGS_format}" = "qemu" \ - -o "${FLAGS_format}" = "xen" ]; then - if [ "${FLAGS_format}" = "virtualbox" ]; then - sudo VBoxManage convertdd "${TEMP_IMG}" "${FLAGS_to}/${FLAGS_vbox_disk}" - else - mv ${TEMP_IMG} ${FLAGS_to}/${DEFAULT_QEMU_IMAGE} - fi -elif [ "${FLAGS_format}" = "vmware" ]; then - qemu-img convert -f raw "${TEMP_IMG}" \ - -O vmdk "${FLAGS_to}/${FLAGS_vmdk}" -elif [ "${FLAGS_format}" = "ami" ]; then - /usr/sbin/gdisk ${TEMP_IMG} < "${FLAGS_to}/${FLAGS_vmx}" - echo "Wrote the following config to: ${FLAGS_to}/${FLAGS_vmx}" - echo "${VMX_CONFIG}" -fi - - -if [ "${FLAGS_format}" == "qemu" ]; then - echo "If you have qemu-kvm installed, you can start the image by:" - echo "qemu-kvm -m ${FLAGS_mem} -curses -pidfile /tmp/kvm.pid -net nic,model=virtio \\" - echo " -net user,hostfwd=tcp::2222-:22 -hda ${FLAGS_to}/${DEFAULT_QEMU_IMAGE}" - echo "SSH into the host with:" - echo "ssh 127.0.0.1 -p 2222" -fi +# Ready to set sail! +okboat +command_completed +print_readme