diff --git a/build_image b/build_image index 52a8ddd10a..c1d4bdf752 100755 --- a/build_image +++ b/build_image @@ -18,10 +18,6 @@ restart_in_chroot_if_needed "$@" assert_not_root_user DEFAULT_GROUP=developer -if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then - # TODO: update to beta, and then stable once those actually exist - DEFAULT_GROUP=alpha -fi # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 0210d1e958..1d34f5c3b5 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -6,9 +6,12 @@ GSUTIL_OPTS= UPLOAD_ROOT= UPLOAD_PATH= UPLOAD_DEFAULT=${FLAGS_FALSE} -if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then - UPLOAD_DEFAULT=${FLAGS_TRUE} -fi + +# Default upload root can be overridden from the environment. +_user="${USER}" +[[ ${USER} == "root" ]] && _user="${SUDO_USER}" +: ${COREOS_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}} +unset _user IMAGE_ZIPPER="lbzip2 --compress --keep" IMAGE_ZIPEXT=".bz2" diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 49eb872400..01dbd7f810 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -121,9 +121,9 @@ get_board_binhost() { for ver in "$@"; do if [[ $toolchain_only -eq 0 ]]; then - echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/pkgs/" + echo "${COREOS_DEV_BUILDS}/boards/${board}/${ver}/pkgs/" fi - echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/toolchain/" + echo "${COREOS_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" done } @@ -144,8 +144,8 @@ get_sdk_binhost() { fi for ver in "$@"; do - echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/pkgs/" - echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/toolchain/" + echo "${COREOS_DEV_BUILDS}/sdk/${arch}/${ver}/pkgs/" + echo "${COREOS_DEV_BUILDS}/sdk/${arch}/${ver}/toolchain/" done } diff --git a/common.sh b/common.sh index 3a175d22aa..bf2d340242 100644 --- a/common.sh +++ b/common.sh @@ -335,12 +335,8 @@ COREOS_VERSION_STRING="${COREOS_VERSION}" readonly COREOS_EPOCH=1372636800 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) -# Builds are uploaded to our Google Cloud Storage space, -# can be overridden from the environment. -: ${COREOS_UPLOAD_ROOT:=gs://storage.core-os.net/coreos} - -# And the corresponding http download url -: ${COREOS_DOWNLOAD_ROOT:=http://storage.core-os.net/coreos} +# Download URL prefix for SDK and board binary packages +: ${COREOS_DEV_BUILDS:=http://builds.developer.core-os.net} # Load developer's custom settings. Default location is in scripts dir, # since that's available both inside and outside the chroot. By convention, diff --git a/core_promote b/core_promote index 94136984bf..4aa16bfa28 100755 --- a/core_promote +++ b/core_promote @@ -25,9 +25,9 @@ DEFINE_string api_key "" \ "API key for roller." DEFINE_string endpoint "https://public.update.core-os.net" \ "Roller endpoint to update." -DEFINE_string build_storage "gs://storage.core-os.net/coreos" \ +DEFINE_string build_storage "gs://builds.release.core-os.net" \ "GS bucket with official build artifacts." -DEFINE_string release_storage "gs://storage.core-os.net/coreos" \ +DEFINE_string release_storage "" \ "GS bucket for release downloads." # Allow toggling the assorted actions. @@ -62,10 +62,15 @@ FLAGS_release_storage="${FLAGS_release_storage%%/}" roller_channel="${FLAGS_channel^}" lower_channel="${FLAGS_channel,,}" -gs_build_path="${FLAGS_build_storage}/${FLAGS_board}/${FLAGS_version}" -# TODO(marineam): once build_storage and release_storage are different -# this will become two paths, one for version and the other for channel. -gs_release_path="${FLAGS_release_storage}/${FLAGS_board}/${lower_channel}" +# Full GS URL of the original build +gs_build="${FLAGS_build_storage}/${lower_channel}/${FLAGS_board}/${FLAGS_version}" + +if [[ -z "${FLAGS_release_storage}" ]]; then + FLAGS_release_storage="gs://${lower_channel}.release.core-os.net" +fi +# Full GS URL of the public release locations +gs_release="${FLAGS_release_storage}/${FLAGS_board}/${FLAGS_version}" +gs_current="${FLAGS_release_storage}/${FLAGS_board}/current" if [[ ${FLAGS_do_roller} -eq ${FLAGS_TRUE} ]]; then rollerctl \ @@ -78,6 +83,10 @@ if [[ ${FLAGS_do_roller} -eq ${FLAGS_TRUE} ]]; then "${FLAGS_version}" fi +if [[ ${FLAGS_do_storage} -eq ${FLAGS_TRUE} ]]; then + gsutil -m cp "${gs_build}/*" "${gs_release}/" +fi + if [[ ${FLAGS_do_gce} -eq ${FLAGS_TRUE} ]]; then gce_name="coreos-${lower_channel}-${FLAGS_version//./-}-v$(date -u +%Y%m%d)" gce_desc="CoreOS ${lower_channel} ${FLAGS_version}" @@ -86,19 +95,16 @@ if [[ ${FLAGS_do_gce} -eq ${FLAGS_TRUE} ]]; then addimage \ --description="${gce_desc}" \ "${gce_name}" \ - "${gs_build_path}/coreos_production_gce.tar.gz" + "${gs_build}/coreos_production_gce.tar.gz" gce_path="projects/coreos-cloud/global/images/${gce_name}" gce_temp=$(mktemp --suffix=.txt) trap "rm -f '${gce_temp}'" EXIT echo "${gce_path}" > "${gce_temp}" - gsutil cp "${gce_temp}" "${gs_build_path}/coreos_production_gce.txt" + gsutil cp "${gce_temp}" "${gs_release}/coreos_production_gce.txt" rm -f "${gce_temp}" trap - EXIT fi if [[ ${FLAGS_do_storage} -eq ${FLAGS_TRUE} ]]; then - gsutil -m cp \ - "${gs_build_path}/coreos_production*" \ - "${gs_build_path}/version.*" \ - "${gs_release_path}/" + gsutil -m cp "${gs_release}/*" "${gs_current}/" fi diff --git a/oem/ami/build_ebs_on_ec2.sh b/oem/ami/build_ebs_on_ec2.sh index f919e97d19..7a01f54c91 100755 --- a/oem/ami/build_ebs_on_ec2.sh +++ b/oem/ami/build_ebs_on_ec2.sh @@ -28,7 +28,7 @@ VERSION="master" BOARD="amd64-usr" GROUP="alpha" IMAGE="coreos_production_ami_image.bin.bz2" -GS_URL="gs://storage.core-os.net/coreos" +GS_URL="gs://builds.release.core-os.net" IMG_URL="" IMG_PATH="" @@ -75,11 +75,18 @@ if [[ -n "$IMG_PATH" ]]; then IMG_URL=$(basename "$IMG_PATH") else if [[ -z "$IMG_URL" ]]; then - IMG_URL="http://${GS_URL#gs://}/$BOARD/$VERSION/$IMAGE" + IMG_URL="$GS_URL/$GROUP/$BOARD/$VERSION/$IMAGE" fi - if ! curl --fail -s --head "$IMG_URL" >/dev/null; then - echo "$0: Image URL unavailable: $IMG_URL" >&2 - exit 1 + if [[ "$IMG_URL" == gs://* ]]; then + if ! gsutil -q stat "$IMG_URL"; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi + else + if ! curl --fail -s --head "$IMG_URL" >/dev/null; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi fi fi @@ -146,12 +153,14 @@ echo "Writing image from $IMG_URL to $dev" # if it is on the local fs, just use it, otherwise try to download it if [[ -n "$IMG_PATH" ]]; then if [[ "$IMG_PATH" =~ \.bz2$ ]]; then - bunzip2 -c "$IMG_PATH" | dd of=$dev bs=1M + bunzip2 -c "$IMG_PATH" > $dev else dd if="$IMG_PATH" of=$dev bs=1M fi +elif [[ "$IMG_URL" == gs://* ]]; then + gsutil cat "$IMG_URL" | bunzip2 > $dev else - curl --fail "$IMG_URL" | bunzip2 | dd of=$dev bs=1M + curl --fail "$IMG_URL" | bunzip2 > $dev fi echo "Detaching $volumeid and creating snapshot" diff --git a/oem/ami/upload_ami_txt.sh b/oem/ami/upload_ami_txt.sh index 259ca462cc..85e427eb86 100755 --- a/oem/ami/upload_ami_txt.sh +++ b/oem/ami/upload_ami_txt.sh @@ -28,7 +28,7 @@ This script must be run from an ec2 host with the ec2 tools installed. " IMAGE="coreos_production_ami" -GS_URL="gs://storage.core-os.net/coreos" +GS_URL="gs://builds.release.core-os.net" AMI= VER= BOARD="amd64-usr" @@ -72,7 +72,7 @@ done OUT= for r in "${!AMIS[@]}"; do - url="$GS_URL/$BOARD/$VER/${IMAGE}_${r}.txt" + url="$GS_URL/$GROUP/$BOARD/$VER/${IMAGE}_${r}.txt" tmp=$(mktemp --suffix=.txt) trap "rm -f '$tmp'" EXIT echo "${AMIS[$r]}" > "$tmp" @@ -84,7 +84,7 @@ for r in "${!AMIS[@]}"; do OUT="${OUT}|${r}=${AMIS[$r]}" fi done -url="$GS_URL/$BOARD/$VER/${IMAGE}_all.txt" +url="$GS_URL/$GROUP/$BOARD/$VER/${IMAGE}_all.txt" tmp=$(mktemp --suffix=.txt) trap "rm -f '$tmp'" EXIT echo "$OUT" > "$tmp" diff --git a/sdk_lib/sdk_util.sh b/sdk_lib/sdk_util.sh index 1dd4ff0376..5c60e46159 100644 --- a/sdk_lib/sdk_util.sh +++ b/sdk_lib/sdk_util.sh @@ -11,7 +11,7 @@ COREOS_SDK_ARCH="amd64" # We are unlikely to support anything else. COREOS_SDK_TARBALL="coreos-sdk-${COREOS_SDK_ARCH}-${COREOS_SDK_VERSION}.tar.bz2" COREOS_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks" COREOS_SDK_TARBALL_PATH="${COREOS_SDK_TARBALL_CACHE}/${COREOS_SDK_TARBALL}" -COREOS_SDK_URL="${COREOS_DOWNLOAD_ROOT}/sdk/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}" +COREOS_SDK_URL="${COREOS_DEV_BUILDS}/sdk/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}" # Download the current SDK tarball (if required) and verify digests/sig sdk_download_tarball() {