Merge pull request #162 from marineam/digests

VM image digests

build_image

@@ -170,7 +170,8 @@ COREOS_PATCH=${COREOS_PATCH}
 COREOS_SDK_VERSION=${COREOS_SDK_VERSION}
 EOF
 
-upload_image "${BUILD_DIR}/au-generator.zip" "${BUILD_DIR}/version.txt"
+upload_image -d "${BUILD_DIR}/au-generator.zip.DIGESTS" \
+    "${BUILD_DIR}/au-generator.zip" "${BUILD_DIR}/version.txt"
 
 # Create a named symlink.
 LINK_NAME="${FLAGS_output_root}/${BOARD}/${FLAGS_symlink}"

build_library/release_util.sh

@@ -85,12 +85,25 @@ upload_packages() {
     upload_files packages ${def_upload_path} "pkgs/" "${board_packages}"/*
 }
 
-# Upload a image along with optional supporting files
+# Upload a set of files (usually images) and digest, optionally w/ gpg sig
-# The image file must be the first argument
+# If more than one file is specified -d must be the first argument
+# Usage: upload_image [-d file.DIGESTS] file1 [file2...]
 upload_image() {
     [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
     [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first"
 
+    # The name to use for .DIGESTS and .DIGESTS.asc must be explicit if
+    # there is more than one file to upload to avoid potential confusion.
+    local digests
+    if [[ "$1" == "-d" ]]; then
+        [[ -n "$2" ]] || die "-d requires an argument"
+        digests="$2"
+        shift 2
+    else
+        [[ $# -eq 1 ]] || die "-d is required for multi-file uploads"
+        digests="${1}.DIGESTS"
+    fi
+
     local uploads=()
     local filename
     for filename in "$@"; do
@@ -110,18 +123,18 @@ upload_image() {
 
     # For consistency generate a .DIGESTS file similar to the one catalyst
     # produces for the SDK tarballs and up upload it too.
-    make_digests "${uploads[@]}"
+    make_digests -d "${digests}" "${uploads[@]}"
-    uploads+=( "${uploads[0]}.DIGESTS" )
+    uploads+=( "${digests}" )
 
     # Create signature as ...DIGESTS.asc as Gentoo does.
     if [[ -n "${FLAGS_sign_digests}" ]]; then
-      rm -f "${uploads[0]}.DIGESTS.asc"
+      rm -f "${digests}.asc"
       gpg --batch --local-user "${FLAGS_sign_digests}" \
-          --clearsign "${uploads[0]}.DIGESTS" || die "gpg failed"
+          --clearsign "${digests}" || die "gpg failed"
-      uploads+=( "${uploads[0]}.DIGESTS.asc" )
+      uploads+=( "${digests}.asc" )
     fi
 
-    local log_msg="${1##*/}"
+    local log_msg=$(basename "$digests" .DIGESTS)
     local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}"
     upload_files "${log_msg}" "${def_upload_path}" "" "${uploads[@]}"
 }

build_library/vm_image_util.sh

@@ -631,6 +631,44 @@ vm_cleanup() {
     sudo rm -rf "${VM_TMP_DIR}"
 }
 
+vm_upload() {
+    local digests="$(_dst_dir)/$(_dst_name .DIGESTS)"
+    upload_image -d "${digests}" "${VM_GENERATED_FILES[@]}"
+
+    # FIXME(marineam): Temporary alternate name for .DIGESTS
+    # This used to be derived from the first file listed in
+    # ${VM_GENERATED_FILES[@]}", usually $VM_DST_IMG or similar.
+    # Since not everything actually uploads $VM_DST_IMG this was not very
+    # consistent and relying on ordering was breakable.
+    # Now the common prefix, output by $(_dst_name) is used above.
+    # Some download/install scripts may still refer to the old name.
+    local uploaded legacy_digests
+    for uploaded in "${VM_GENERATED_FILES[@]}"; do
+        if [[ "${uploaded}" == "${VM_DST_IMG}" ]]; then
+            legacy_digests="$(_dst_dir)/$(basename ${VM_DST_IMG}).DIGESTS"
+            break
+        fi
+    done
+
+    # Since depending on the ordering of $VM_GENERATED_FILES is brittle only
+    # use it if $VM_DST_IMG isn't included in the uploaded files.
+    if [[ -z "${legacy_digests}" ]]; then
+        legacy_digests="${VM_GENERATED_FILES[0]}.DIGESTS"
+    fi
+
+    [[ "${legacy_digests}" != "${digests}" ]] || return
+
+    local legacy_uploads=( "${legacy_digests}" )
+    cp "${digests}" "${legacy_digests}"
+    if [[ -e "${digests}.asc" ]]; then
+        legacy_uploads+=( "${legacy_digests}.asc" )
+        cp "${digests}.asc" "${legacy_digests}.asc"
+    fi
+
+    local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}"
+    upload_files "$(_dst_name)" "${def_upload_path}" "" "${legacy_uploads[@]}"
+}
+
 print_readme() {
     local filename
     info "Files written to $(relpath "$(dirname "${VM_DST_IMG}")")"

common.sh

@@ -693,40 +693,45 @@ enable_rw_mount() {
 
 # Generate a DIGESTS file, as normally used by Gentoo.
 # This is an alternative to shash which doesn't know how to report errors.
-# Usage: make_digests file1 [file2...]
+# Usage: make_digests -d file.DIGESTS file1 [file2...]
-# Output: file1.DIGESTS
-# Any extra files be hashed and listed in file1.DIGESTS
 _digest_types="md5 sha1 sha512"
 make_digests() {
-    local dirname=$(dirname "$1")
+    [[ "$1" == "-d" ]] || die
-    local basename=$(basename "$1")
+    local digests="$(readlink -f "$2")"
+    shift 2
 
-    pushd "${dirname}" >/dev/null
+    pushd "$(dirname "$1")" >/dev/null
-    echo -n > "${basename}.DIGESTS"
+    echo -n > "${digests}"
     for filename in "$@"; do
         filename=$(basename "$filename")
         info "Computing DIGESTS for ${filename}"
         for hash_type in $_digest_types; do
-            echo "# $hash_type HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS"
+            echo "# $hash_type HASH" | tr "a-z" "A-Z" >> "${digests}"
-            ${hash_type}sum "${filename}" >> "${basename}.DIGESTS"
+            ${hash_type}sum "${filename}" >> "${digests}"
         done
     done
     popd >/dev/null
 }
 
 # Validate a DIGESTS file. Essentially the inverse of make_digests.
-# Usage: verify_digests file1 [file2...]
+# Usage: verify_digests [-d file.DIGESTS] file1 [file2...]
-# Checks the hash of all given files using file1.DIGESTS
+# If -d is not specified file1.DIGESTS will be used
 verify_digests() {
-    local dirname=$(dirname "$1")
+    local digests
-    local basename=$(basename "$1")
+    if [[ "$1" == "-d" ]]; then
+        [[ -n "$2" ]] || die "-d requires an argument"
+        digests="$(readlink -f "$2")"
+        shift 2
+    else
+        digests=$(basename "${1}.DIGESTS")
+    fi
 
-    pushd "${dirname}" >/dev/null
+    pushd "$(dirname "$1")" >/dev/null
     for filename in "$@"; do
         filename=$(basename "$filename")
         info "Validating DIGESTS for ${filename}"
         for hash_type in $_digest_types; do
-            grep -A1 -i "^# ${hash_type} HASH$" "${basename}.DIGESTS" | \
+            grep -A1 -i "^# ${hash_type} HASH$" "${digests}" | \
                 grep "$filename$" | ${hash_type}sum -c - --strict || return 1
             # Also check that none of the greps failed in the above pipeline
             [[ -z ${PIPESTATUS[*]#0} ]] || return 1

image_to_vm.sh

@@ -106,7 +106,7 @@ vm_cleanup
 trap - EXIT
 
 # Optionally upload all of our hard work
-upload_image "${VM_GENERATED_FILES[@]}"
+vm_upload
 
 # Ready to set sail!
 okboat