fix(*): Migrate image storage to new buckets

- Automated builds drop SDK and binary packages into
  gs://builds.developer.core-os.net/ and the new download URL is
  http://builds.developer.core-os.net/ (COREOS_DEV_BUILDS)
- Change default upload path to gs://users.developer.core-os.net/ for
  misc developer builds. Official builds go elsewhere and will just be
  configured in buildbot/jenkins so some COREOS_OFFICIAL stuff is gone.
- Automated builds of images go to a private bucket,
  gs://builds.release.core-os.net which later gets copied to
  gs://alpha.release.core-os.net and friends by core_promote.

build_image

@@ -18,10 +18,6 @@ restart_in_chroot_if_needed "$@"
 assert_not_root_user
 
 DEFAULT_GROUP=developer
-if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then
-  # TODO: update to beta, and then stable once those actually exist
-  DEFAULT_GROUP=alpha
-fi
 
 # Developer-visible flags.
 DEFINE_string board "${DEFAULT_BOARD}" \

build_library/release_util.sh

@@ -6,9 +6,12 @@ GSUTIL_OPTS=
 UPLOAD_ROOT=
 UPLOAD_PATH=
 UPLOAD_DEFAULT=${FLAGS_FALSE}
-if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then
-  UPLOAD_DEFAULT=${FLAGS_TRUE}
-fi
+
+# Default upload root can be overridden from the environment.
+_user="${USER}"
+[[ ${USER} == "root" ]] && _user="${SUDO_USER}"
+: ${COREOS_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}}
+unset _user
 
 IMAGE_ZIPPER="lbzip2 --compress --keep"
 IMAGE_ZIPEXT=".bz2"

build_library/toolchain_util.sh

@@ -121,9 +121,9 @@ get_board_binhost() {
 
     for ver in "$@"; do
         if [[ $toolchain_only -eq 0 ]]; then
-            echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/pkgs/"
+            echo "${COREOS_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
         fi
-        echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/toolchain/"
+        echo "${COREOS_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
     done
 }
 
@@ -144,8 +144,8 @@ get_sdk_binhost() {
     fi
 
     for ver in "$@"; do
-        echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/pkgs/"
-        echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/toolchain/"
+        echo "${COREOS_DEV_BUILDS}/sdk/${arch}/${ver}/pkgs/"
+        echo "${COREOS_DEV_BUILDS}/sdk/${arch}/${ver}/toolchain/"
     done
 }
 

common.sh

@@ -335,12 +335,8 @@ COREOS_VERSION_STRING="${COREOS_VERSION}"
 readonly COREOS_EPOCH=1372636800
 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
 
-# Builds are uploaded to our Google Cloud Storage space,
-# can be overridden from the environment.
-: ${COREOS_UPLOAD_ROOT:=gs://storage.core-os.net/coreos}
-
-# And the corresponding http download url
-: ${COREOS_DOWNLOAD_ROOT:=http://storage.core-os.net/coreos}
+# Download URL prefix for SDK and board binary packages
+: ${COREOS_DEV_BUILDS:=http://builds.developer.core-os.net}
 
 # Load developer's custom settings.  Default location is in scripts dir,
 # since that's available both inside and outside the chroot.  By convention,

core_promote

@@ -25,9 +25,9 @@ DEFINE_string api_key "" \
     "API key for roller."
 DEFINE_string endpoint "https://public.update.core-os.net" \
     "Roller endpoint to update."
-DEFINE_string build_storage "gs://storage.core-os.net/coreos" \
+DEFINE_string build_storage "gs://builds.release.core-os.net" \
     "GS bucket with official build artifacts."
-DEFINE_string release_storage "gs://storage.core-os.net/coreos" \
+DEFINE_string release_storage "" \
     "GS bucket for release downloads."
 
 # Allow toggling the assorted actions.
@@ -62,10 +62,15 @@ FLAGS_release_storage="${FLAGS_release_storage%%/}"
 roller_channel="${FLAGS_channel^}"
 lower_channel="${FLAGS_channel,,}"
 
-gs_build_path="${FLAGS_build_storage}/${FLAGS_board}/${FLAGS_version}"
-# TODO(marineam): once build_storage and release_storage are different
-# this will become two paths, one for version and the other for channel.
-gs_release_path="${FLAGS_release_storage}/${FLAGS_board}/${lower_channel}"
+# Full GS URL of the original build
+gs_build="${FLAGS_build_storage}/${lower_channel}/${FLAGS_board}/${FLAGS_version}"
+
+if [[ -z "${FLAGS_release_storage}" ]]; then
+    FLAGS_release_storage="gs://${lower_channel}.release.core-os.net"
+fi
+# Full GS URL of the public release locations
+gs_release="${FLAGS_release_storage}/${FLAGS_board}/${FLAGS_version}"
+gs_current="${FLAGS_release_storage}/${FLAGS_board}/current"
 
 if [[ ${FLAGS_do_roller} -eq ${FLAGS_TRUE} ]]; then
     rollerctl \
@@ -78,6 +83,10 @@ if [[ ${FLAGS_do_roller} -eq ${FLAGS_TRUE} ]]; then
         "${FLAGS_version}"
 fi
 
+if [[ ${FLAGS_do_storage} -eq ${FLAGS_TRUE} ]]; then
+    gsutil -m cp "${gs_build}/*" "${gs_release}/"
+fi
+
 if [[ ${FLAGS_do_gce} -eq ${FLAGS_TRUE} ]]; then
     gce_name="coreos-${lower_channel}-${FLAGS_version//./-}-v$(date -u +%Y%m%d)"
     gce_desc="CoreOS ${lower_channel} ${FLAGS_version}"
@@ -86,19 +95,16 @@ if [[ ${FLAGS_do_gce} -eq ${FLAGS_TRUE} ]]; then
         addimage \
         --description="${gce_desc}" \
         "${gce_name}" \
-        "${gs_build_path}/coreos_production_gce.tar.gz"
+        "${gs_build}/coreos_production_gce.tar.gz"
     gce_path="projects/coreos-cloud/global/images/${gce_name}"
     gce_temp=$(mktemp --suffix=.txt)
     trap "rm -f '${gce_temp}'" EXIT
     echo "${gce_path}" > "${gce_temp}"
-    gsutil cp "${gce_temp}" "${gs_build_path}/coreos_production_gce.txt"
+    gsutil cp "${gce_temp}" "${gs_release}/coreos_production_gce.txt"
     rm -f "${gce_temp}"
     trap - EXIT
 fi
 
 if [[ ${FLAGS_do_storage} -eq ${FLAGS_TRUE} ]]; then
-    gsutil -m cp \
-        "${gs_build_path}/coreos_production*" \
-        "${gs_build_path}/version.*" \
-        "${gs_release_path}/"
+    gsutil -m cp "${gs_release}/*" "${gs_current}/"
 fi

oem/ami/build_ebs_on_ec2.sh

@@ -28,7 +28,7 @@ VERSION="master"
 BOARD="amd64-usr"
 GROUP="alpha"
 IMAGE="coreos_production_ami_image.bin.bz2"
-GS_URL="gs://storage.core-os.net/coreos"
+GS_URL="gs://builds.release.core-os.net"
 IMG_URL=""
 IMG_PATH=""
 
@@ -75,11 +75,18 @@ if [[ -n "$IMG_PATH" ]]; then
     IMG_URL=$(basename "$IMG_PATH")
 else
     if [[ -z "$IMG_URL" ]]; then
-        IMG_URL="http://${GS_URL#gs://}/$BOARD/$VERSION/$IMAGE"
+        IMG_URL="$GS_URL/$GROUP/$BOARD/$VERSION/$IMAGE"
     fi
-    if ! curl --fail -s --head "$IMG_URL" >/dev/null; then
-        echo "$0: Image URL unavailable: $IMG_URL" >&2
-        exit 1
+    if [[ "$IMG_URL" == gs://* ]]; then
+        if ! gsutil -q stat "$IMG_URL"; then
+            echo "$0: Image URL unavailable: $IMG_URL" >&2
+            exit 1
+        fi
+    else
+        if ! curl --fail -s --head "$IMG_URL" >/dev/null; then
+            echo "$0: Image URL unavailable: $IMG_URL" >&2
+            exit 1
+        fi
     fi
 fi
 
@@ -146,12 +153,14 @@ echo "Writing image from $IMG_URL to $dev"
 # if it is on the local fs, just use it, otherwise try to download it
 if [[ -n "$IMG_PATH" ]]; then
     if [[ "$IMG_PATH" =~ \.bz2$ ]]; then
-        bunzip2 -c "$IMG_PATH" | dd of=$dev bs=1M
+        bunzip2 -c "$IMG_PATH" > $dev
     else
         dd if="$IMG_PATH" of=$dev bs=1M
     fi
+elif [[ "$IMG_URL" == gs://* ]]; then
+    gsutil cat "$IMG_URL" | bunzip2 > $dev
 else
-    curl --fail "$IMG_URL" | bunzip2 | dd of=$dev bs=1M
+    curl --fail "$IMG_URL" | bunzip2 > $dev
 fi
 
 echo "Detaching $volumeid and creating snapshot"

oem/ami/upload_ami_txt.sh

@@ -28,7 +28,7 @@ This script must be run from an ec2 host with the ec2 tools installed.
 "
 
 IMAGE="coreos_production_ami"
-GS_URL="gs://storage.core-os.net/coreos"
+GS_URL="gs://builds.release.core-os.net"
 AMI=
 VER=
 BOARD="amd64-usr"
@@ -72,7 +72,7 @@ done
 
 OUT=
 for r in "${!AMIS[@]}"; do
-    url="$GS_URL/$BOARD/$VER/${IMAGE}_${r}.txt"
+    url="$GS_URL/$GROUP/$BOARD/$VER/${IMAGE}_${r}.txt"
     tmp=$(mktemp --suffix=.txt)
     trap "rm -f '$tmp'" EXIT
     echo "${AMIS[$r]}" > "$tmp"
@@ -84,7 +84,7 @@ for r in "${!AMIS[@]}"; do
         OUT="${OUT}|${r}=${AMIS[$r]}"
     fi
 done
-url="$GS_URL/$BOARD/$VER/${IMAGE}_all.txt"
+url="$GS_URL/$GROUP/$BOARD/$VER/${IMAGE}_all.txt"
 tmp=$(mktemp --suffix=.txt)
 trap "rm -f '$tmp'" EXIT
 echo "$OUT" > "$tmp"

sdk_lib/sdk_util.sh

@@ -11,7 +11,7 @@ COREOS_SDK_ARCH="amd64" # We are unlikely to support anything else.
 COREOS_SDK_TARBALL="coreos-sdk-${COREOS_SDK_ARCH}-${COREOS_SDK_VERSION}.tar.bz2"
 COREOS_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks"
 COREOS_SDK_TARBALL_PATH="${COREOS_SDK_TARBALL_CACHE}/${COREOS_SDK_TARBALL}"
-COREOS_SDK_URL="${COREOS_DOWNLOAD_ROOT}/sdk/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}"
+COREOS_SDK_URL="${COREOS_DEV_BUILDS}/sdk/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}"
 
 # Download the current SDK tarball (if required) and verify digests/sig
 sdk_download_tarball() {