common: Split binpkg and SDK tarball URLs

Use FLATCAR_DEV_BUILDS only for setting up binpkg URLs, ceding the SDK
tarball URL role to the new FLATCAR_SDK_SERVERS variable. That way we
can still set up binpkg URLs the way we used to do so far and set up
SDK tarball URLs differently.

For two-phase SDK build, we would like to use intermediate SDK as a
seed. This SDK is only available on bincache, but previously only
nightly builds could use bincache as the source of SDK tarballs. Now,
with the URL split, we can set up the builds to use both bincache and
the release server, where release builds will prioritize release
server over bincache, and developer builds - bincache over release
server.

common.sh

@@ -338,7 +338,25 @@ FLATCAR_VERSION_STRING="${FLATCAR_VERSION}"
 readonly COREOS_EPOCH=1372636800
 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
 
-# Download URL prefix for SDK and board binary packages
+# Download URL prefixes for SDK
+if [[ -n "${FLATCAR_BUILD_ID}" ]] ; then
+    # For dev builds, first try bincache, then release to allow a
+    # bincache overwrite.
+    FLATCAR_SDK_SERVERS=(
+        "${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}"
+        "${SETTING_BINPKG_SERVER_PROD}"
+    )
+else
+    # For release builds, first try release, then bincache to allow
+    # downloading intermediate SDKs if using two-phase builds for
+    # releases.
+    FLATCAR_SDK_SERVERS=(
+        "${SETTING_BINPKG_SERVER_PROD}"
+        "${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}"
+    )
+fi
+
+# Download URL prefix for board binary packages
 if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
     : ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}}
 else

sdk_lib/sdk_util.sh

@@ -13,7 +13,6 @@ FLATCAR_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks"
 FLATCAR_SDK_TARBALL_PATH="${FLATCAR_SDK_TARBALL_CACHE}/${FLATCAR_SDK_TARBALL}"
 FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-$FLATCAR_DEV_BUILDS/sdk}"
 FLATCAR_SDK_URL="${FLATCAR_DEV_BUILDS_SDK}/${FLATCAR_SDK_ARCH}/${FLATCAR_SDK_VERSION}/${FLATCAR_SDK_TARBALL}"
-FLATCAR_SDK_RELEASE_URL="https://mirror.release.flatcar-linux.net/sdk/${FLATCAR_SDK_ARCH}/${FLATCAR_SDK_VERSION}/${FLATCAR_SDK_TARBALL}"
 
 # Download the current SDK tarball (if required) and verify digests/sig
 sdk_download_tarball() {
@@ -22,21 +21,53 @@ sdk_download_tarball() {
     fi
 
     info "Downloading ${FLATCAR_SDK_TARBALL}"
-    info "URL: ${FLATCAR_SDK_URL}"
-    local suffix
-    for suffix in "" ".DIGESTS"; do # TODO(marineam): download .asc
-        # First try bincache then release to allow a bincache overwrite
-        wget --tries=3 --timeout=30 --continue \
-            -O  "${FLATCAR_SDK_TARBALL_PATH}${suffix}" \
-            "${FLATCAR_SDK_URL}${suffix}" \
-            || wget --tries=3 --timeout=30 --continue \
-            -O "${FLATCAR_SDK_TARBALL_PATH}${suffix}" \
-            "${FLATCAR_SDK_RELEASE_URL}${suffix}" \
-            || die_notrace "SDK download failed!"
-    done
+    local server url suffix
+    local -a suffixes
 
-    sdk_verify_digests || die_notrace "SDK digest verification failed!"
-    sdk_clean_cache
+    suffixes=('' '.DIGESTS') # TODO(marineam): download .asc
+    for server in "${FLATCAR_SDK_SERVERS[@]}"; do
+        url="${server}/sdk/${FLATCAR_SDK_ARCH}/${FLATCAR_SDK_VERSION}/${FLATCAR_SDK_TARBALL}"
+        info "URL: ${url}"
+        for suffix in "${suffixes[@]}"; do
+            # If all downloads fail, we will detect it later.
+            if ! wget --tries=3 --timeout=30 --continue \
+                 -O "${FLATCAR_SDK_TARBALL_PATH}${suffix}" \
+                 "${url}${suffix}"; then
+                break
+            fi
+        done
+        if _sdk_check_downloads "${FLATCAR_SDK_TARBALL_PATH}" "${suffixes[@]}"; then
+            if sdk_verify_digests; then
+                sdk_clean_cache
+                return 0
+            fi
+            info "SDK digest verification failed, cleaning up and will try another server"
+        else
+            info "Downloading SDK from ${url} failed, cleaning up and will try another server"
+        fi
+        _sdk_remove_downloads "${FLATCAR_SDK_TARBALL_PATH}" "${suffixes[@]}"
+    done
+    die_notrace "SDK download failed!"
+}
+
+_sdk_remove_downloads() {
+    local path="${1}"; shift
+    # rest of the params are suffixes
+
+    rm -f "${@/#/${path}}"
+}
+
+_sdk_check_downloads() {
+    local path="${1}"; shift
+    # rest of the params are suffixes
+    local suffix
+
+    for suffix; do
+        if [[ ! -s "${path}${suffix}" ]]; then
+            return 1
+        fi
+    done
+    return 0
 }
 
 sdk_verify_digests() {