feat(vm_image_util): Add new build steps, update vagrant builds

This adds two new optional build steps.  The first user of these is the
vagrant images but many of the targets can be simplified now.

- fs_hook: Anything that needs to happen before unmounting the image.
  This happens after the OEM is installed but before disk images are
  made. It can be used to copy any data out of the image.
- bundle_format: Many VM types ship as some sort of archive format
  rather than plain disk images as this script originally assumed.
  Adding this final step lets us stop using the conf step awkwardly.

Vagrant now ships with a Vagrantfile and related code included in the
OEM package. This lets us version our vagrant-side code along with the
images themselves as well making the coreos-vagrant repo optional again.
The coreos-vagrant code will still be useful for handling the fancier
cluster configuration stuff but no longer has to carry the plugin code.

build_library/virtualbox_ovf.sh

@@ -51,18 +51,8 @@ datez() {
 
 if [[ -n "${FLAGS_output_vagrant}" ]]; then
     cat >"${FLAGS_output_vagrant}" <<EOF
-if Vagrant::VERSION < "1.2.3"
-  raise "Need at least vagrant version 1.2.3, please update"
-end
-
 Vagrant.configure("2") do |config|
   config.vm.base_mac = "${PRIMARY_MAC}"
-
-  # SSH in as the default 'core' user, it has the vagrant ssh key.
-  config.ssh.username = "core"
-
-  # Disable the base shared folder, guest additions are unavailable.
-  config.vm.synced_folder ".", "/vagrant", disabled: true
 end
 EOF
 fi

build_library/vm_image_util.sh

@@ -52,6 +52,9 @@ IMG_DEFAULT_OEM_PACKAGE=
 # USE flags for the OEM package
 IMG_DEFAULT_OEM_USE=
 
+# Hook to do any final tweaks or grab data while fs is mounted.
+IMG_DEFAULT_FS_HOOK=
+
 # Name of the target image format.
 # May be raw, qcow2 (qemu), or vmdk (vmware, virtualbox)
 IMG_DEFAULT_DISK_FORMAT=raw
@@ -62,6 +65,9 @@ IMG_DEFAULT_DISK_LAYOUT=base
 # Name of the target config format, default is no config
 IMG_DEFAULT_CONF_FORMAT=
 
+# Bundle configs and disk image into some archive
+IMG_DEFAULT_BUNDLE_FORMAT=
+
 # Memory size to use in any config files
 IMG_DEFAULT_MEM=1024
 
@@ -86,12 +92,16 @@ IMG_virtualbox_DISK_LAYOUT=vm
 IMG_virtualbox_CONF_FORMAT=ovf
 
 ## vagrant
+IMG_vagrant_FS_HOOK=box
+IMG_vagrant_BUNDLE_FORMAT=box
 IMG_vagrant_DISK_FORMAT=vmdk_ide
 IMG_vagrant_DISK_LAYOUT=vagrant
 IMG_vagrant_CONF_FORMAT=vagrant
 IMG_vagrant_OEM_PACKAGE=oem-vagrant
 
 ## vagrant_vmware
+IMG_vagrant_vmware_fusion_FS_HOOK=box
+IMG_vagrant_vmware_fusion_BUNDLE_FORMAT=box
 IMG_vagrant_vmware_fusion_DISK_FORMAT=vmdk_scsi
 IMG_vagrant_vmware_fusion_DISK_LAYOUT=vagrant
 IMG_vagrant_vmware_fusion_CONF_FORMAT=vagrant_vmware_fusion
@@ -209,6 +219,10 @@ _dst_dir() {
     echo $(dirname "$VM_DST_IMG")
 }
 
+# Combine dst name and dir
+_dst_path() {
+    echo "$(_dst_dir)/$(_dst_name "$@")"
+}
 
 # Get the proper disk format extension.
 _disk_ext() {
@@ -277,6 +291,21 @@ install_oem_package() {
     sudo rm -rf "${oem_tmp}"
 }
 
+# Any other tweaks required?
+run_fs_hook() {
+    local fs_hook=$(_get_vm_opt FS_HOOK)
+    if [[ -n "${fs_hook}" ]]; then
+        info "Running ${fs_hook} fs hook"
+        _run_${fs_hook}_fs_hook "$@"
+    fi
+}
+
+_run_box_fs_hook() {
+    # Copy basic Vagrant configs from OEM
+    mkdir -p "${VM_TMP_DIR}/box"
+    cp -R "${VM_TMP_ROOT}/usr/share/oem/box/." "${VM_TMP_DIR}/box"
+}
+
 # Write the vm disk image to the target directory in the proper format
 write_vm_disk() {
     if [[ $(_get_vm_opt PARTITIONED_IMG) -eq 1 ]]; then
@@ -287,7 +316,11 @@ write_vm_disk() {
     local disk_format=$(_get_vm_opt DISK_FORMAT)
     info "Writing $disk_format image $(basename "${VM_DST_IMG}")"
     _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}"
-    VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
+
+    # Add disk image to final file list if it isn't going to be bundled
+    if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then
+        VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
+    fi
 }
 
 _write_raw_disk() {
@@ -635,86 +668,32 @@ EOF
 
 _write_vagrant_conf() {
     local vm_mem="${1:-$(_get_vm_opt MEM)}"
-    local src_name=$(basename "$VM_SRC_IMG")
+    local ovf="${VM_TMP_DIR}/box/box.ovf"
-    local dst_name=$(basename "$VM_DST_IMG")
+    local mac="${VM_TMP_DIR}/box/base_mac.rb"
-    local dst_dir=$(dirname "$VM_DST_IMG")
-    local ovf="${VM_TMP_DIR}/box.ovf"
-    local vfile="${VM_TMP_DIR}/Vagrantfile"
-    local box="${dst_dir}/$(_src_to_dst_name "${src_name}" ".box")"
-
-    # Move the disk image to tmp, it won't be a final output file
-    mv "${VM_DST_IMG}" "${VM_TMP_DIR}/${dst_name}"
 
     "${BUILD_LIBRARY_DIR}/virtualbox_ovf.sh" \
             --vm_name "$VM_NAME" \
-            --disk_vmdk "${VM_TMP_DIR}/${dst_name}" \
+            --disk_vmdk "${VM_DST_IMG}" \
             --memory_size "$vm_mem" \
             --output_ovf "$ovf" \
-            --output_vagrant "$vfile"
+            --output_vagrant "$mac"
 
-    cat > "${VM_TMP_DIR}"/metadata.json <<EOF
+    cat > "${VM_TMP_DIR}"/box/metadata.json <<EOF
 {"provider": "virtualbox"}
 EOF
-
-    tar -czf "${box}" -C "${VM_TMP_DIR}" "box.ovf" "Vagrantfile" "${dst_name}" "metadata.json"
-
-    cat > "${VM_README}" <<EOF
-Vagrant >= 1.2.3 is required. Use something like the following to get started:
-vagrant box add coreos path/to/$(basename "${box}")
-vagrant init coreos
-vagrant up
-vagrant ssh
-
-You will get a warning about "No guest additions were detected...",
-this is expected and should be ignored. SSH should work just dandy.
-EOF
-
-    # Replace list, not append, since we packaged up the disk image.
-    VM_GENERATED_FILES=( "${box}" "${VM_README}" )
 }
 
 _write_vagrant_vmware_fusion_conf() {
     local vm_mem="${1:-$(_get_vm_opt MEM)}"
-    local src_name=$(basename "$VM_SRC_IMG")
+    local vmx=$(_dst_path ".vmx")
-    local dst_name=$(basename "$VM_DST_IMG")
-    local dst_dir=$(dirname "$VM_DST_IMG")
-    local vmx_path="${dst_dir}/$(_src_to_dst_name "${src_name}" ".vmx")"
-    local vmx_file=$(basename "${vmx_path}")
-    local vfile="${VM_TMP_DIR}/Vagrantfile"
-    local box="${dst_dir}/$(_src_to_dst_name "${src_name}" ".box")"
-
-    # Move the disk image to tmp, it won't be a final output file
-    mv "${VM_DST_IMG}" "${VM_TMP_DIR}/${dst_name}"
 
+    mkdir -p "${VM_TMP_DIR}/box"
     _write_vmx_conf ${vm_mem}
-    "${BUILD_LIBRARY_DIR}/virtualbox_ovf.sh" \
+    mv "${vmx}" "${VM_TMP_DIR}/box"
-            --vm_name "$VM_NAME" \
-            --disk_vmdk "${VM_TMP_DIR}/${dst_name}" \
-            --memory_size "$vm_mem" \
-            --output_vagrant "$vfile"
 
-    cat > "${VM_TMP_DIR}"/metadata.json <<EOF
+    cat > "${VM_TMP_DIR}"/box/metadata.json <<EOF
 {"provider": "vmware_fusion"}
 EOF
-
-    mv "${vmx_path}" "${VM_TMP_DIR}/"
-
-    tar -czf "${box}" -C "${VM_TMP_DIR}" "Vagrantfile" "${dst_name}" \
-        "${vmx_file}" "metadata.json"
-
-    cat > "${VM_README}" <<EOF
-Vagrant master (unreleased) currently has full CoreOS support. In the meantime, you may encounter an error about networking that can be ignored
-vagrant box add coreos path/to/$(basename "${box}")
-vagrant init coreos
-vagrant up
-vagrant ssh
-
-You will get a warning about "No guest additions were detected...",
-this is expected and should be ignored. SSH should work just dandy.
-EOF
-
-    # Replace list, not append, since we packaged up the disk image.
-    VM_GENERATED_FILES=( "${box}" "${VM_README}" )
 }
 
 _write_gce_conf() {
@@ -727,6 +706,23 @@ _write_gce_conf() {
     VM_GENERATED_FILES=( "${tar_path}" )
 }
 
+# If this is a bundled format generate it!
+write_vm_bundle() {
+    local bundle_format=$(_get_vm_opt BUNDLE_FORMAT)
+    if [[ -n "${bundle_format}" ]]; then
+        info "Writing ${bundle_format} bundle"
+        _write_${bundle_format}_bundle "$@"
+    fi
+}
+
+_write_box_bundle() {
+    local box=$(_dst_path ".box")
+
+    mv "${VM_DST_IMG}" "${VM_TMP_DIR}/box"
+    tar -czf "${box}" -C "${VM_TMP_DIR}/box" .
+    VM_GENERATED_FILES+=( "${box}" )
+}
+
 vm_cleanup() {
     info "Cleaning up temporary files"
     if mountpoint -q "${VM_TMP_ROOT}"; then

image_to_vm.sh

@@ -102,10 +102,12 @@ setup_disk_image "${FLAGS_disk_layout}"
 
 # Optionally install any OEM packages
 install_oem_package
+run_fs_hook
 
 # Changes done, glue it together
 write_vm_disk
 write_vm_conf "${FLAGS_mem}"
+write_vm_bundle
 
 vm_cleanup
 trap - EXIT