From 39951946ca11a1ac4aa6293a1cc7aa49e3ebca9b Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Mon, 28 Apr 2014 19:52:24 -0400
Subject: [PATCH 1/5] feat(vm_image_util): Add new build steps, update vagrant
 builds

This adds two new optional build steps.  The first user of these is the
vagrant images but many of the targets can be simplified now.

- fs_hook: Anything that needs to happen before unmounting the image.
  This happens after the OEM is installed but before disk images are
  made. It can be used to copy any data out of the image.
- bundle_format: Many VM types ship as some sort of archive format
  rather than plain disk images as this script originally assumed.
  Adding this final step lets us stop using the conf step awkwardly.

Vagrant now ships with a Vagrantfile and related code included in the
OEM package. This lets us version our vagrant-side code along with the
images themselves as well making the coreos-vagrant repo optional again.
The coreos-vagrant code will still be useful for handling the fancier
cluster configuration stuff but no longer has to carry the plugin code.
---
 build_library/virtualbox_ovf.sh |  10 ---
 build_library/vm_image_util.sh  | 124 ++++++++++++++++----------------
 image_to_vm.sh                  |   2 +
 3 files changed, 62 insertions(+), 74 deletions(-)

diff --git a/build_library/virtualbox_ovf.sh b/build_library/virtualbox_ovf.sh
index 6a430a7894..06609927f2 100755
--- a/build_library/virtualbox_ovf.sh
+++ b/build_library/virtualbox_ovf.sh
@@ -51,18 +51,8 @@ datez() {
 
 if [[ -n "${FLAGS_output_vagrant}" ]]; then
     cat >"${FLAGS_output_vagrant}" <<EOF
-if Vagrant::VERSION < "1.2.3"
-  raise "Need at least vagrant version 1.2.3, please update"
-end
-
 Vagrant.configure("2") do |config|
   config.vm.base_mac = "${PRIMARY_MAC}"
-
-  # SSH in as the default 'core' user, it has the vagrant ssh key.
-  config.ssh.username = "core"
-
-  # Disable the base shared folder, guest additions are unavailable.
-  config.vm.synced_folder ".", "/vagrant", disabled: true
 end
 EOF
 fi
diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh
index 16df215c62..5fefafaf61 100644
--- a/build_library/vm_image_util.sh
+++ b/build_library/vm_image_util.sh
@@ -52,6 +52,9 @@ IMG_DEFAULT_OEM_PACKAGE=
 # USE flags for the OEM package
 IMG_DEFAULT_OEM_USE=
 
+# Hook to do any final tweaks or grab data while fs is mounted.
+IMG_DEFAULT_FS_HOOK=
+
 # Name of the target image format.
 # May be raw, qcow2 (qemu), or vmdk (vmware, virtualbox)
 IMG_DEFAULT_DISK_FORMAT=raw
@@ -62,6 +65,9 @@ IMG_DEFAULT_DISK_LAYOUT=base
 # Name of the target config format, default is no config
 IMG_DEFAULT_CONF_FORMAT=
 
+# Bundle configs and disk image into some archive
+IMG_DEFAULT_BUNDLE_FORMAT=
+
 # Memory size to use in any config files
 IMG_DEFAULT_MEM=1024
 
@@ -86,12 +92,16 @@ IMG_virtualbox_DISK_LAYOUT=vm
 IMG_virtualbox_CONF_FORMAT=ovf
 
 ## vagrant
+IMG_vagrant_FS_HOOK=box
+IMG_vagrant_BUNDLE_FORMAT=box
 IMG_vagrant_DISK_FORMAT=vmdk_ide
 IMG_vagrant_DISK_LAYOUT=vagrant
 IMG_vagrant_CONF_FORMAT=vagrant
 IMG_vagrant_OEM_PACKAGE=oem-vagrant
 
 ## vagrant_vmware
+IMG_vagrant_vmware_fusion_FS_HOOK=box
+IMG_vagrant_vmware_fusion_BUNDLE_FORMAT=box
 IMG_vagrant_vmware_fusion_DISK_FORMAT=vmdk_scsi
 IMG_vagrant_vmware_fusion_DISK_LAYOUT=vagrant
 IMG_vagrant_vmware_fusion_CONF_FORMAT=vagrant_vmware_fusion
@@ -209,6 +219,10 @@ _dst_dir() {
     echo $(dirname "$VM_DST_IMG")
 }
 
+# Combine dst name and dir
+_dst_path() {
+    echo "$(_dst_dir)/$(_dst_name "$@")"
+}
 
 # Get the proper disk format extension.
 _disk_ext() {
@@ -277,6 +291,21 @@ install_oem_package() {
     sudo rm -rf "${oem_tmp}"
 }
 
+# Any other tweaks required?
+run_fs_hook() {
+    local fs_hook=$(_get_vm_opt FS_HOOK)
+    if [[ -n "${fs_hook}" ]]; then
+        info "Running ${fs_hook} fs hook"
+        _run_${fs_hook}_fs_hook "$@"
+    fi
+}
+
+_run_box_fs_hook() {
+    # Copy basic Vagrant configs from OEM
+    mkdir -p "${VM_TMP_DIR}/box"
+    cp -R "${VM_TMP_ROOT}/usr/share/oem/box/." "${VM_TMP_DIR}/box"
+}
+
 # Write the vm disk image to the target directory in the proper format
 write_vm_disk() {
     if [[ $(_get_vm_opt PARTITIONED_IMG) -eq 1 ]]; then
@@ -287,7 +316,11 @@ write_vm_disk() {
     local disk_format=$(_get_vm_opt DISK_FORMAT)
     info "Writing $disk_format image $(basename "${VM_DST_IMG}")"
     _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}"
-    VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
+
+    # Add disk image to final file list if it isn't going to be bundled
+    if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then
+        VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
+    fi
 }
 
 _write_raw_disk() {
@@ -635,86 +668,32 @@ EOF
 
 _write_vagrant_conf() {
     local vm_mem="${1:-$(_get_vm_opt MEM)}"
-    local src_name=$(basename "$VM_SRC_IMG")
-    local dst_name=$(basename "$VM_DST_IMG")
-    local dst_dir=$(dirname "$VM_DST_IMG")
-    local ovf="${VM_TMP_DIR}/box.ovf"
-    local vfile="${VM_TMP_DIR}/Vagrantfile"
-    local box="${dst_dir}/$(_src_to_dst_name "${src_name}" ".box")"
-
-    # Move the disk image to tmp, it won't be a final output file
-    mv "${VM_DST_IMG}" "${VM_TMP_DIR}/${dst_name}"
+    local ovf="${VM_TMP_DIR}/box/box.ovf"
+    local mac="${VM_TMP_DIR}/box/base_mac.rb"
 
     "${BUILD_LIBRARY_DIR}/virtualbox_ovf.sh" \
             --vm_name "$VM_NAME" \
-            --disk_vmdk "${VM_TMP_DIR}/${dst_name}" \
+            --disk_vmdk "${VM_DST_IMG}" \
             --memory_size "$vm_mem" \
             --output_ovf "$ovf" \
-            --output_vagrant "$vfile"
+            --output_vagrant "$mac"
 
-    cat > "${VM_TMP_DIR}"/metadata.json <<EOF
+    cat > "${VM_TMP_DIR}"/box/metadata.json <<EOF
 {"provider": "virtualbox"}
 EOF
-
-    tar -czf "${box}" -C "${VM_TMP_DIR}" "box.ovf" "Vagrantfile" "${dst_name}" "metadata.json"
-
-    cat > "${VM_README}" <<EOF
-Vagrant >= 1.2.3 is required. Use something like the following to get started:
-vagrant box add coreos path/to/$(basename "${box}")
-vagrant init coreos
-vagrant up
-vagrant ssh
-
-You will get a warning about "No guest additions were detected...",
-this is expected and should be ignored. SSH should work just dandy.
-EOF
-
-    # Replace list, not append, since we packaged up the disk image.
-    VM_GENERATED_FILES=( "${box}" "${VM_README}" )
 }
 
 _write_vagrant_vmware_fusion_conf() {
     local vm_mem="${1:-$(_get_vm_opt MEM)}"
-    local src_name=$(basename "$VM_SRC_IMG")
-    local dst_name=$(basename "$VM_DST_IMG")
-    local dst_dir=$(dirname "$VM_DST_IMG")
-    local vmx_path="${dst_dir}/$(_src_to_dst_name "${src_name}" ".vmx")"
-    local vmx_file=$(basename "${vmx_path}")
-    local vfile="${VM_TMP_DIR}/Vagrantfile"
-    local box="${dst_dir}/$(_src_to_dst_name "${src_name}" ".box")"
-
-    # Move the disk image to tmp, it won't be a final output file
-    mv "${VM_DST_IMG}" "${VM_TMP_DIR}/${dst_name}"
+    local vmx=$(_dst_path ".vmx")
 
+    mkdir -p "${VM_TMP_DIR}/box"
     _write_vmx_conf ${vm_mem}
-    "${BUILD_LIBRARY_DIR}/virtualbox_ovf.sh" \
-            --vm_name "$VM_NAME" \
-            --disk_vmdk "${VM_TMP_DIR}/${dst_name}" \
-            --memory_size "$vm_mem" \
-            --output_vagrant "$vfile"
+    mv "${vmx}" "${VM_TMP_DIR}/box"
 
-    cat > "${VM_TMP_DIR}"/metadata.json <<EOF
+    cat > "${VM_TMP_DIR}"/box/metadata.json <<EOF
 {"provider": "vmware_fusion"}
 EOF
-
-    mv "${vmx_path}" "${VM_TMP_DIR}/"
-
-    tar -czf "${box}" -C "${VM_TMP_DIR}" "Vagrantfile" "${dst_name}" \
-        "${vmx_file}" "metadata.json"
-
-    cat > "${VM_README}" <<EOF
-Vagrant master (unreleased) currently has full CoreOS support. In the meantime, you may encounter an error about networking that can be ignored
-vagrant box add coreos path/to/$(basename "${box}")
-vagrant init coreos
-vagrant up
-vagrant ssh
-
-You will get a warning about "No guest additions were detected...",
-this is expected and should be ignored. SSH should work just dandy.
-EOF
-
-    # Replace list, not append, since we packaged up the disk image.
-    VM_GENERATED_FILES=( "${box}" "${VM_README}" )
 }
 
 _write_gce_conf() {
@@ -727,6 +706,23 @@ _write_gce_conf() {
     VM_GENERATED_FILES=( "${tar_path}" )
 }
 
+# If this is a bundled format generate it!
+write_vm_bundle() {
+    local bundle_format=$(_get_vm_opt BUNDLE_FORMAT)
+    if [[ -n "${bundle_format}" ]]; then
+        info "Writing ${bundle_format} bundle"
+        _write_${bundle_format}_bundle "$@"
+    fi
+}
+
+_write_box_bundle() {
+    local box=$(_dst_path ".box")
+
+    mv "${VM_DST_IMG}" "${VM_TMP_DIR}/box"
+    tar -czf "${box}" -C "${VM_TMP_DIR}/box" .
+    VM_GENERATED_FILES+=( "${box}" )
+}
+
 vm_cleanup() {
     info "Cleaning up temporary files"
     if mountpoint -q "${VM_TMP_ROOT}"; then
diff --git a/image_to_vm.sh b/image_to_vm.sh
index 585f02abe4..3f764d927f 100755
--- a/image_to_vm.sh
+++ b/image_to_vm.sh
@@ -102,10 +102,12 @@ setup_disk_image "${FLAGS_disk_layout}"
 
 # Optionally install any OEM packages
 install_oem_package
+run_fs_hook
 
 # Changes done, glue it together
 write_vm_disk
 write_vm_conf "${FLAGS_mem}"
+write_vm_bundle
 
 vm_cleanup
 trap - EXIT

From 8183430f6cf7d7369b139a6c90f569746a0d3f12 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Mon, 28 Apr 2014 23:24:30 -0400
Subject: [PATCH 2/5] feat(vm_image_util): Add Vagrant metadata file

This is required for Vagrant to recognize box versions:
http://docs.vagrantup.com/v2/boxes/versioning.html
---
 build_library/release_util.sh  | 14 ++++++++++++++
 build_library/vm_image_util.sh | 19 ++++++++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/build_library/release_util.sh b/build_library/release_util.sh
index f5dba907cb..0210d1e958 100644
--- a/build_library/release_util.sh
+++ b/build_library/release_util.sh
@@ -174,3 +174,17 @@ upload_image() {
     local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}"
     upload_files "${log_msg}" "${def_upload_path}" "" "${uploads[@]}"
 }
+
+# Translate the configured upload URL to a download URL
+# Usage: download_image_url "path/suffix"
+download_image_url() {
+    if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then
+        echo "$1"
+        return 0
+    fi
+    local upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}"
+    if [[ -n "${UPLOAD_PATH}" ]]; then
+        upload_path="${UPLOAD_PATH}"
+    fi
+    echo "http://${upload_path#gs://}/$1"
+}
diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh
index 5fefafaf61..43731cd0f0 100644
--- a/build_library/vm_image_util.sh
+++ b/build_library/vm_image_util.sh
@@ -717,10 +717,27 @@ write_vm_bundle() {
 
 _write_box_bundle() {
     local box=$(_dst_path ".box")
+    local json=$(_dst_path ".json")
 
     mv "${VM_DST_IMG}" "${VM_TMP_DIR}/box"
     tar -czf "${box}" -C "${VM_TMP_DIR}/box" .
-    VM_GENERATED_FILES+=( "${box}" )
+
+    cat >"${json}" <<EOF
+{
+  "name": "coreos-alpha",
+  "description": "CoreOS alpha",
+  "versions": [{
+    "version": "${COREOS_VERSION_ID}",
+    "providers": [{
+      "name": "virtualbox",
+      "url": "$(download_image_url "$(_dst_name ".box")")",
+      "checksum_type": "sha256",
+      "checksum": "$(sha256sum "${box}" | awk '{print $1}')"
+    }]
+  }]
+}
+EOF
+    VM_GENERATED_FILES+=( "${box}" "${json}" )
 }
 
 vm_cleanup() {

From 20f191a804f949dc67563ce8b933cd1b2655cba0 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Tue, 29 Apr 2014 00:25:35 -0400
Subject: [PATCH 3/5] maint(set_lsb_release): Remove gentoo-release (again)

We now require Vagrant 1.5.0 both in documentation and in code which
uses os-release instead of gentoo-release.
---
 build_library/set_lsb_release | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/build_library/set_lsb_release b/build_library/set_lsb_release
index 1a2d7552d1..b74c93aca7 100755
--- a/build_library/set_lsb_release
+++ b/build_library/set_lsb_release
@@ -67,6 +67,3 @@ sudo_clobber "${ROOT_FS_DIR}/usr/share/coreos/update.conf" <<EOF
 SERVER=$COREOS_VERSION_AUSERVER
 GROUP=$FLAGS_group
 EOF
-
-# For old versions of vagrant
-sudo_clobber "${ROOT_FS_DIR}/etc/gentoo-release" <<<"$COREOS_VERSION_NAME"

From 10d98e7b326a3d926f49fdd8c56bf78a511ce127 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Tue, 29 Apr 2014 01:00:45 -0400
Subject: [PATCH 4/5] fix(common): Switch to os-release style names in
 version.txt

The existing version.txt is kinda annoying. The common case of referring
to the current version requires joining three values and the names of
those values only make sense in ChromeOS. Instead just use version as a
string, using VERSION, VERSION_ID, and BUILD_ID just as they appear in
os-release. It is up to the few scripts that need the individual parts
to break the version apart.

The old values remain for the sake of compatibility.
---
 build_image    |  3 +++
 common.sh      | 11 ++++++++---
 image_to_vm.sh |  2 +-
 tag_release    | 18 +++++++++++-------
 4 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/build_image b/build_image
index 32eb1caf61..e54d727a1b 100755
--- a/build_image
+++ b/build_image
@@ -177,6 +177,9 @@ tee "${BUILD_DIR}/version.txt" <<EOF
 COREOS_BUILD=${COREOS_BUILD}
 COREOS_BRANCH=${COREOS_BRANCH}
 COREOS_PATCH=${COREOS_PATCH}
+COREOS_VERSION=${COREOS_VERSION_STRING}
+COREOS_VERSION_ID=${COREOS_VERSION_ID}
+COREOS_BUILD_ID=${COREOS_BUILD_ID}
 COREOS_SDK_VERSION=${COREOS_SDK_VERSION}
 EOF
 
diff --git a/common.sh b/common.sh
index 11b1803a76..7b9ce50379 100644
--- a/common.sh
+++ b/common.sh
@@ -311,6 +311,9 @@ if [[ ! -f "${COREOS_VERSION_FILE}" ]]; then
 fi
 source "$COREOS_VERSION_FILE" || die "Cannot source version.txt"
 
+# Set version based on old variables if undefined
+: ${COREOS_VERSION_ID:=${COREOS_BUILD}.${COREOS_BRANCH}.${COREOS_PATCH}}
+
 # Official builds must set COREOS_OFFICIAL=1 to use an official version.
 # Unofficial builds always appended the date/time as a build identifier.
 # Also do not alter the version if using an alternate version.txt path.
@@ -318,11 +321,13 @@ COREOS_BUILD_ID=""
 if [[ ${COREOS_OFFICIAL:-0} -ne 1 &&
     "${COREOS_VERSION_FILE}" =~ /\.repo/manifests/version.txt ]]; then
   COREOS_BUILD_ID=$(date +%Y-%m-%d-%H%M)
+  COREOS_VERSION="${COREOS_VERSION_ID}+${COREOS_BUILD_ID}"
+else
+  COREOS_VERSION="${COREOS_VERSION_ID}"
 fi
 
-# Full version string.
-COREOS_VERSION_ID="${COREOS_BUILD}.${COREOS_BRANCH}.${COREOS_PATCH}"
-COREOS_VERSION_STRING="${COREOS_VERSION_ID}${COREOS_BUILD_ID:++}${COREOS_BUILD_ID}"
+# Compatibility alias
+COREOS_VERSION_STRING="${COREOS_VERSION}"
 
 # Calculate what today's build version should be, used by release
 # scripts to provide a reasonable default value. The value is the number
diff --git a/image_to_vm.sh b/image_to_vm.sh
index 3f764d927f..198b5ab75c 100755
--- a/image_to_vm.sh
+++ b/image_to_vm.sh
@@ -84,7 +84,7 @@ FLAGS_to=`eval readlink -f $FLAGS_to`
 # If source includes version.txt switch to its version information
 if [ -f "${FLAGS_from}/version.txt" ]; then
     source "${FLAGS_from}/version.txt"
-    COREOS_VERSION_STRING="${COREOS_BUILD}.${COREOS_BRANCH}.${COREOS_PATCH}"
+    COREOS_VERSION_STRING="${COREOS_VERSION}"
 fi
 
 if [ ${FLAGS_prod_image} -eq ${FLAGS_TRUE} ]; then
diff --git a/tag_release b/tag_release
index d134940c0f..7427531c22 100755
--- a/tag_release
+++ b/tag_release
@@ -7,16 +7,17 @@
 SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/common.sh" || exit 1
 
-DEFAULT_BUILD=${TODAYS_VERSION}
-DEFAULT_BRANCH=0
+DEFAULT_MAJOR=${TODAYS_VERSION}
+DEFAULT_MINOR=0
 
-# Increment $BRANCH if we already are on a version from today
-if [[ ${DEFAULT_BUILD} -eq ${COREOS_BUILD} ]]; then
-    DEFAULT_BRANCH=$((COREOS_BRANCH + 1))
+# Increment $MINOR if we already are on a version from today
+CURRENT_VERSION=( ${COREOS_VERSION_ID//./ } )
+if [[ ${DEFAULT_MAJOR} -eq ${CURRENT_VERSION[0]} ]]; then
+    DEFAULT_MINOR=$((${CURRENT_VERSION[1]} + 1))
 fi
 
-DEFINE_integer build ${DEFAULT_BUILD} "Branch major version (aka 'build')"
-DEFINE_integer branch ${DEFAULT_BRANCH} "Branch revision or minor version"
+DEFINE_integer major ${DEFAULT_MAJOR} "Branch major version (aka 'build')"
+DEFINE_integer branch ${DEFAULT_MINOR} "Branch revision or minor version"
 DEFINE_integer patch 0 "Branch patch id, normally 0"
 DEFINE_string track "dev-channel" "Set the given track to this new branch"
 DEFINE_string sdk_version "${COREOS_VERSION_ID}" \
@@ -60,6 +61,9 @@ tee version.txt <<EOF
 COREOS_BUILD=${FLAGS_build}
 COREOS_BRANCH=${FLAGS_branch}
 COREOS_PATCH=${FLAGS_patch}
+COREOS_VERSION=${BRANCH_VERSION}
+COREOS_VERSION_ID=${BRANCH_VERSION}
+COREOS_BUILD_ID=
 COREOS_SDK_VERSION=${FLAGS_sdk_version}
 EOF
 ln -sf "${BRANCH_NAME}.xml" release.xml

From 3fdfe92b5d9e3a5d52fa4f222390526f2e4734e0 Mon Sep 17 00:00:00 2001
From: Michael Marineau <michael.marineau@coreos.com>
Date: Tue, 29 Apr 2014 03:37:09 -0400
Subject: [PATCH 5/5] fix(vm_image_util): Fix building vagrant vmware fusion
 images.

---
 build_library/vm_image_util.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh
index 43731cd0f0..61234cd970 100644
--- a/build_library/vm_image_util.sh
+++ b/build_library/vm_image_util.sh
@@ -535,7 +535,10 @@ usb.generic.autoconnect = "FALSE"
 usb.present = "TRUE"
 rtc.diffFromUTC = 0
 EOF
-    VM_GENERATED_FILES+=( "${vmx_path}" )
+    # Only upload the vmx if it won't be bundled
+    if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then
+        VM_GENERATED_FILES+=( "${vmx_path}" )
+    fi
 }
 
 _write_vmware_zip_conf() {
@@ -722,6 +725,11 @@ _write_box_bundle() {
     mv "${VM_DST_IMG}" "${VM_TMP_DIR}/box"
     tar -czf "${box}" -C "${VM_TMP_DIR}/box" .
 
+    local provider="virtualbox"
+    if [[ "${VM_IMG_TYPE}" == vagrant_vmware_fusion ]]; then
+        provider="vmware_fusion"
+    fi
+
     cat >"${json}" <<EOF
 {
   "name": "coreos-alpha",
@@ -729,7 +737,7 @@ _write_box_bundle() {
   "versions": [{
     "version": "${COREOS_VERSION_ID}",
     "providers": [{
-      "name": "virtualbox",
+      "name": "${provider}",
       "url": "$(download_image_url "$(_dst_name ".box")")",
       "checksum_type": "sha256",
       "checksum": "$(sha256sum "${box}" | awk '{print $1}')"