Merge pull request #169 from marineam/binhost

binhost config update

build_library/release_util.sh

@@ -3,7 +3,7 @@
 # found in the LICENSE file.
 
 GSUTIL_OPTS=
-UPLOAD_ROOT="${COREOS_UPLOAD_ROOT:-gs://storage.core-os.net/coreos}"
+UPLOAD_ROOT=
 UPLOAD_PATH=
 UPLOAD_DEFAULT=${FLAGS_FALSE}
 if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then
@@ -17,8 +17,10 @@ DEFINE_boolean parallel ${FLAGS_TRUE} \
   "Enable parallelism in gsutil."
 DEFINE_boolean upload ${UPLOAD_DEFAULT} \
   "Upload all packages/images via gsutil."
+DEFINE_string upload_root "${COREOS_UPLOAD_ROOT}" \
+  "Upload prefix, board/version/etc will be appended. Must be a gs:// URL."
 DEFINE_string upload_path "" \
-  "Upload files to an alternative location. Must be a full gs:// URL."
+  "Full upload path, overrides --upload_root. Must be a full gs:// URL."
 DEFINE_string sign_digests "" \
   "Sign image DIGESTS files with the given GPG key."
 
@@ -29,7 +31,18 @@ check_gsutil_opts() {
         GSUTIL_OPTS="-m"
     fi
 
+    if [[ -n "${FLAGS_upload_root}" ]]; then
+        if [[ "${FLAGS_upload_root}" != gs://* ]]; then
+            die_notrace "--upload_root must be a gs:// URL"
+        fi
+        # Make sure the path doesn't end with a slash
+        UPLOAD_ROOT="${FLAGS_upload_root%%/}"
+    fi
+
     if [[ -n "${FLAGS_upload_path}" ]]; then
+        if [[ "${FLAGS_upload_path}" != gs://* ]]; then
+            die_notrace "--upload_path must be a gs:// URL"
+        fi
         # Make sure the path doesn't end with a slash
         UPLOAD_PATH="${FLAGS_upload_path%%/}"
     fi

build_library/set_lsb_release

@@ -90,14 +90,12 @@ EOF
 # Aaaannd for the new systemd world order
 # os-release provides a separate build-id field, so split it from version
 OS_ID=$(tr '[:upper:]' '[:lower:]' <<<"$COREOS_VERSION_NAME")
-OS_VERSION_ID="${COREOS_VERSION_STRING%%+*}"
-OS_BUILD_ID="${COREOS_VERSION_STRING#*+}"
 sudo_clobber "${ROOT_FS_DIR}/etc/os-release" <<EOF
 NAME=$COREOS_VERSION_NAME
 ID=$OS_ID
 VERSION=$COREOS_VERSION_STRING
-VERSION_ID=$OS_VERSION_ID
-BUILD_ID=$OS_BUILD_ID
+VERSION_ID=$COREOS_VERSION_ID
+BUILD_ID=$COREOS_BUILD_ID
 PRETTY_NAME="$COREOS_VERSION_NAME $COREOS_VERSION_DESCRIPTION"
 ANSI_COLOR="1;32"
 HOME_URL="http://www.coreos.com/"

build_library/toolchain_util.sh

@@ -105,6 +105,30 @@ get_board_profile() {
     done
 }
 
+# Usage: get_board_binhost [-t] board [version...]
+# -t: toolchain only, full rebuilds re-using toolchain pkgs
+# If no versions are specified the current and SDK versions are used.
+get_board_binhost() {
+    local toolchain_only=0 board ver
+    if [[ "$1" == "-t" ]]; then
+        toolchain_only=1
+        shift
+    fi
+    board="$1"
+    shift
+
+    if [[ $# -eq 0 ]]; then
+        set -- "${COREOS_VERSION_ID}" "${COREOS_SDK_VERSION}"
+    fi
+
+    for ver in "$@"; do
+        if [[ $toolchain_only -eq 0 ]]; then
+            echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/pkgs/"
+        fi
+        echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/toolchain/"
+    done
+}
+
 # Usage: get_cross_pkgs chost [chost2...]
 get_cross_pkgs() {
     local cross_chost native_pkg
@@ -149,7 +173,6 @@ _configure_sysroot() {
     fi
 
     $sudo mkdir -p "${ROOT}/etc/portage"
-    echo "eselect will report '!!! Warning: Strange path.' but that's OK"
     $sudo eselect profile set --force "$profile"
 
     $sudo tee "${ROOT}/etc/portage/make.conf" >/dev/null <<EOF

build_packages

@@ -47,6 +47,10 @@ show_help_if_requested "$@"
 # The following options are advanced options, only available to those willing
 # to read the source code. They are not shown in help output, since they are
 # not needed for the typical developer workflow.
+DEFINE_string getbinpkgver "" \
+  "Use binary packages from a specific version."
+DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \
+  "Use binary packages only for the board toolchain."
 DEFINE_string accept_licenses "" \
   "Licenses to append to the accept list."
 DEFINE_boolean fast "${DEFAULT_FAST}" \
@@ -91,7 +95,7 @@ check_gsutil_opts
 CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin"
 
 # Before we can run any tools, we need to update chroot or setup_board.
-UPDATE_ARGS=()
+UPDATE_ARGS=( --regen_configs )
 if [[ -n ${FLAGS_accept_licenses} ]]; then
   UPDATE_ARGS+=( --accept_licenses "${FLAGS_accept_licenses}" )
 fi
@@ -107,6 +111,14 @@ if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
   else
     UPDATE_ARGS+=( --nogetbinpkg )
   fi
+  if [[ "${FLAGS_toolchainpkgonly}" -eq "${FLAGS_TRUE}" ]]; then
+    UPDATE_ARGS+=( --toolchainpkgonly )
+  else
+    UPDATE_ARGS+=( --notoolchainpkgonly )
+  fi
+  if [[ -n "${FLAGS_getbinpkgver}" ]]; then
+    UPDATE_ARGS+=( --getbinpkgver="${FLAGS_getbinpkgver}" )
+  fi
 else
   UPDATE_ARGS+=( --nousepkg )
 fi

common.sh

@@ -314,13 +314,15 @@ source "$COREOS_VERSION_FILE" || die "Cannot source version.txt"
 # Official builds must set COREOS_OFFICIAL=1 to use an official version.
 # Unofficial builds always appended the date/time as a build identifier.
 # Also do not alter the version if using an alternate version.txt path.
+COREOS_BUILD_ID=""
 if [[ ${COREOS_OFFICIAL:-0} -ne 1 &&
     "${COREOS_VERSION_FILE}" =~ /\.repo/manifests/version.txt ]]; then
-  COREOS_PATCH="${COREOS_PATCH}+$(date +%Y-%m-%d-%H%M)"
+  COREOS_BUILD_ID=$(date +%Y-%m-%d-%H%M)
 fi
 
 # Full version string.
-COREOS_VERSION_STRING="${COREOS_BUILD}.${COREOS_BRANCH}.${COREOS_PATCH}"
+COREOS_VERSION_ID="${COREOS_BUILD}.${COREOS_BRANCH}.${COREOS_PATCH}"
+COREOS_VERSION_STRING="${COREOS_VERSION_ID}${COREOS_BUILD_ID++}${COREOS_BUILD_ID}"
 
 # Calculate what today's build version should be, used by release
 # scripts to provide a reasonable default value. The value is the number
@@ -328,6 +330,13 @@ COREOS_VERSION_STRING="${COREOS_BUILD}.${COREOS_BRANCH}.${COREOS_PATCH}"
 readonly COREOS_EPOCH=1372636800
 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
 
+# Builds are uploaded to our Google Cloud Storage space,
+# can be overridden from the environment.
+: ${COREOS_UPLOAD_ROOT:=gs://storage.core-os.net/coreos}
+
+# And the corresponding http download url
+: ${COREOS_DOWNLOAD_ROOT:=http://storage.core-os.net/coreos}
+
 # Load developer's custom settings.  Default location is in scripts dir,
 # since that's available both inside and outside the chroot.  By convention,
 # settings from this file are variables starting with 'CHROMEOS_'

core_upload_update

@@ -77,5 +77,5 @@ core-admin new-version \
 
 # Copy the vagrant boxes and pre-built images over to the track mirror
 gsutil cp \
-    "gs://storage.core-os.net/coreos/amd64-generic/${FLAGS_version}/*" \
-    "gs://storage.core-os.net/coreos/amd64-generic/${FLAGS_track}/"
+    "${COREOS_UPLOAD_ROOT}/amd64-generic/${FLAGS_version}/*" \
+    "${COREOS_UPLOAD_ROOT}/amd64-generic/${FLAGS_track}/"

sdk_lib/sdk_util.sh

@@ -11,8 +11,7 @@ COREOS_SDK_ARCH="amd64" # We are unlikely to support anything else.
 COREOS_SDK_TARBALL="coreos-sdk-${COREOS_SDK_ARCH}-${COREOS_SDK_VERSION}.tar.bz2"
 COREOS_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks"
 COREOS_SDK_TARBALL_PATH="${COREOS_SDK_TARBALL_CACHE}/${COREOS_SDK_TARBALL}"
-COREOS_SDK_URL_PREFIX="http://storage.core-os.net/coreos/sdk"
-COREOS_SDK_URL="${COREOS_SDK_URL_PREFIX}/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}"
+COREOS_SDK_URL="${COREOS_DOWNLOAD_ROOT}/sdk/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}"
 
 # Download the current SDK tarball (if required) and verify digests/sig
 sdk_download_tarball() {

setup_board

@@ -33,6 +33,10 @@ show_help_if_requested "$@"
 # The following options are advanced options, only available to those willing
 # to read the source code. They are not shown in help output, since they are
 # not needed for the typical developer workflow.
+DEFINE_string getbinpkgver "" \
+  "Use binary packages from a specific version."
+DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \
+  "Use binary packages only for the board toolchain."
 DEFINE_string accept_licenses "" \
   "Licenses to append to the accept list."
 DEFINE_boolean fast "${DEFAULT_FAST}" \
@@ -124,6 +128,16 @@ EOF
   sudo_multi "${cmds[@]}"
 }
 
+generate_binhost_list() {
+  local t
+  [[ "${FLAGS_toolchainpkgonly}" -eq "${FLAGS_TRUE}" ]] && t="-t"
+  FLAGS_getbinpkgver="${FLAGS_getbinpkgver/current/${COREOS_VERSION_ID}}"
+  FLAGS_getbinpkgver="${FLAGS_getbinpkgver/latest/${COREOS_VERSION_ID}}"
+  FLAGS_getbinpkgver="${FLAGS_getbinpkgver/sdk/${COREOS_SDK_VERSION}}"
+
+  get_board_binhost $t "${BOARD}" ${FLAGS_getbinpkgver}
+}
+
 # Parse command line flags
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
@@ -180,6 +194,7 @@ BOARD_PROFILE="${BOARD_ETC}/portage/profile"
 BOARD_ARCH=$(get_board_arch "$BOARD")
 BOARD_CHOST=$(get_board_chost ${BOARD})
 PORTAGE_PROFILE=$(get_board_profile "$BOARD")
+BOARD_BINHOST=$(generate_binhost_list)
 
 if [ -d "${BOARD_ROOT}" ]; then
   if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then
@@ -202,7 +217,6 @@ else
 fi
 
 info "Configuring portage in ${BOARD_ROOT}"
-echo "eselect will report '!!! Warning: Strange path.' but that's OK"
 cmds=(
   "mkdir -p '${BOARD_ROOT}' '${BOARD_PROFILE}' '${BOARD_ETC}/portage/hooks'"
   "ROOT='${BOARD_ROOT}' eselect profile set --force '${PORTAGE_PROFILE}'"
@@ -237,6 +251,7 @@ ROOT="${BOARD_ROOT}/"
 MAKEOPTS="--jobs=${NUM_JOBS} --load-average=${NUM_JOBS}"
 PKG_CONFIG="pkg-config-${BOARD_VARIANT}"
 BOARD_USE="${BOARD_VARIANT}"
+PORTAGE_BINHOST="${BOARD_BINHOST}"
 ${ACCEPT_LICENSE}
 EOF
 

tag_release

@@ -7,15 +7,12 @@
 SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/common.sh" || exit 1
 
-# Default to the current release for the new SDK version (minus dev build id)
-DEFAULT_SDK="${COREOS_VERSION_STRING%+*}"
-
 DEFINE_integer build "${TODAYS_VERSION}" \
     "Branch name (aka 'build'), should be days since 2013-7-1"
 DEFINE_integer branch 0 "Branch revision, normally 0"
 DEFINE_integer patch 0 "Branch patch id, normally 0"
 DEFINE_string track "dev-channel" "Set the given track to this new branch"
-DEFINE_string sdk_version "${DEFAULT_SDK}" \
+DEFINE_string sdk_version "${COREOS_VERSION_ID}" \
     "Set the SDK version to use. (current: ${COREOS_SDK_VERSION})"
 DEFINE_boolean push ${FLAGS_FALSE} "Push to public manifest repository."
 DEFINE_string remote "origin" "Remote name or URL to push to."