From 50e54cea859efea796c3fab5303f465174496ae7 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Thu, 6 Feb 2014 13:51:09 -0800 Subject: [PATCH 1/4] fix(release_util): Move default upload root to common, add --upload_root Make it possible for other scripts to share the same value for our release repository and equally easy to override with a custom value. Also allow setting the root from the command line in addition to the environment. Usually --upload_root is better to use than --upload_path. --- build_library/release_util.sh | 17 +++++++++++++++-- common.sh | 7 +++++++ core_upload_update | 4 ++-- sdk_lib/sdk_util.sh | 3 +-- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index b9790bb7fc..1015438b54 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -3,7 +3,7 @@ # found in the LICENSE file. GSUTIL_OPTS= -UPLOAD_ROOT="${COREOS_UPLOAD_ROOT:-gs://storage.core-os.net/coreos}" +UPLOAD_ROOT= UPLOAD_PATH= UPLOAD_DEFAULT=${FLAGS_FALSE} if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then @@ -17,8 +17,10 @@ DEFINE_boolean parallel ${FLAGS_TRUE} \ "Enable parallelism in gsutil." DEFINE_boolean upload ${UPLOAD_DEFAULT} \ "Upload all packages/images via gsutil." +DEFINE_string upload_root "${COREOS_UPLOAD_ROOT}" \ + "Upload prefix, board/version/etc will be appended. Must be a gs:// URL." DEFINE_string upload_path "" \ - "Upload files to an alternative location. Must be a full gs:// URL." + "Full upload path, overrides --upload_root. Must be a full gs:// URL." DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." @@ -29,7 +31,18 @@ check_gsutil_opts() { GSUTIL_OPTS="-m" fi + if [[ -n "${FLAGS_upload_root}" ]]; then + if [[ "${FLAGS_upload_root}" != gs://* ]]; then + die_notrace "--upload_root must be a gs:// URL" + fi + # Make sure the path doesn't end with a slash + UPLOAD_ROOT="${FLAGS_upload_root%%/}" + fi + if [[ -n "${FLAGS_upload_path}" ]]; then + if [[ "${FLAGS_upload_path}" != gs://* ]]; then + die_notrace "--upload_path must be a gs:// URL" + fi # Make sure the path doesn't end with a slash UPLOAD_PATH="${FLAGS_upload_path%%/}" fi diff --git a/common.sh b/common.sh index bd94e09500..c7201ea148 100644 --- a/common.sh +++ b/common.sh @@ -328,6 +328,13 @@ COREOS_VERSION_STRING="${COREOS_BUILD}.${COREOS_BRANCH}.${COREOS_PATCH}" readonly COREOS_EPOCH=1372636800 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) +# Builds are uploaded to our Google Cloud Storage space, +# can be overridden from the environment. +: ${COREOS_UPLOAD_ROOT:=gs://storage.core-os.net/coreos} + +# And the corresponding http download url +: ${COREOS_DOWNLOAD_ROOT:=http://storage.core-os.net/coreos} + # Load developer's custom settings. Default location is in scripts dir, # since that's available both inside and outside the chroot. By convention, # settings from this file are variables starting with 'CHROMEOS_' diff --git a/core_upload_update b/core_upload_update index 9310fe0e03..5cf866fc88 100755 --- a/core_upload_update +++ b/core_upload_update @@ -77,5 +77,5 @@ core-admin new-version \ # Copy the vagrant boxes and pre-built images over to the track mirror gsutil cp \ - "gs://storage.core-os.net/coreos/amd64-generic/${FLAGS_version}/*" \ - "gs://storage.core-os.net/coreos/amd64-generic/${FLAGS_track}/" + "${COREOS_UPLOAD_ROOT}/amd64-generic/${FLAGS_version}/*" \ + "${COREOS_UPLOAD_ROOT}/amd64-generic/${FLAGS_track}/" diff --git a/sdk_lib/sdk_util.sh b/sdk_lib/sdk_util.sh index 401b818d52..1dd4ff0376 100644 --- a/sdk_lib/sdk_util.sh +++ b/sdk_lib/sdk_util.sh @@ -11,8 +11,7 @@ COREOS_SDK_ARCH="amd64" # We are unlikely to support anything else. COREOS_SDK_TARBALL="coreos-sdk-${COREOS_SDK_ARCH}-${COREOS_SDK_VERSION}.tar.bz2" COREOS_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks" COREOS_SDK_TARBALL_PATH="${COREOS_SDK_TARBALL_CACHE}/${COREOS_SDK_TARBALL}" -COREOS_SDK_URL_PREFIX="http://storage.core-os.net/coreos/sdk" -COREOS_SDK_URL="${COREOS_SDK_URL_PREFIX}/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}" +COREOS_SDK_URL="${COREOS_DOWNLOAD_ROOT}/sdk/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}" # Download the current SDK tarball (if required) and verify digests/sig sdk_download_tarball() { From 5524a7ce9a9be6f5720d29156b3859388e92d350 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 9 Feb 2014 13:56:41 -0800 Subject: [PATCH 2/4] fix(common): Define VERSION_ID and BUILD_ID Scripts can now use COREOS_VERSION_ID to get the base version instead of having to strip off the BUILD_ID that is appended for unofficial builds. --- build_library/set_lsb_release | 6 ++---- common.sh | 6 ++++-- tag_release | 5 +---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/build_library/set_lsb_release b/build_library/set_lsb_release index 49fea724b5..b29e25e213 100755 --- a/build_library/set_lsb_release +++ b/build_library/set_lsb_release @@ -90,14 +90,12 @@ EOF # Aaaannd for the new systemd world order # os-release provides a separate build-id field, so split it from version OS_ID=$(tr '[:upper:]' '[:lower:]' <<<"$COREOS_VERSION_NAME") -OS_VERSION_ID="${COREOS_VERSION_STRING%%+*}" -OS_BUILD_ID="${COREOS_VERSION_STRING#*+}" sudo_clobber "${ROOT_FS_DIR}/etc/os-release" < Date: Thu, 6 Feb 2014 14:36:25 -0800 Subject: [PATCH 3/4] feat(setup_board): Configure binary package sources on the fly. Generate list of board binhosts based on the new $COREOS_DOWNLOAD_ROOT variable and optionally use packages only from a specific version --- build_library/toolchain_util.sh | 24 ++++++++++++++++++++++++ build_packages | 14 +++++++++++++- setup_board | 16 ++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index e841f33057..74f40d7c3e 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -105,6 +105,30 @@ get_board_profile() { done } +# Usage: get_board_binhost [-t] board [version...] +# -t: toolchain only, full rebuilds re-using toolchain pkgs +# If no versions are specified the current and SDK versions are used. +get_board_binhost() { + local toolchain_only=0 board ver + if [[ "$1" == "-t" ]]; then + toolchain_only=1 + shift + fi + board="$1" + shift + + if [[ $# -eq 0 ]]; then + set -- "${COREOS_VERSION_ID}" "${COREOS_SDK_VERSION}" + fi + + for ver in "$@"; do + if [[ $toolchain_only -eq 0 ]]; then + echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/pkgs/" + fi + echo "${COREOS_DOWNLOAD_ROOT}/${board}/${ver}/toolchain/" + done +} + # Usage: get_cross_pkgs chost [chost2...] get_cross_pkgs() { local cross_chost native_pkg diff --git a/build_packages b/build_packages index ed9e383a54..0f408aedb8 100755 --- a/build_packages +++ b/build_packages @@ -47,6 +47,10 @@ show_help_if_requested "$@" # The following options are advanced options, only available to those willing # to read the source code. They are not shown in help output, since they are # not needed for the typical developer workflow. +DEFINE_string getbinpkgver "" \ + "Use binary packages from a specific version." +DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \ + "Use binary packages only for the board toolchain." DEFINE_string accept_licenses "" \ "Licenses to append to the accept list." DEFINE_boolean fast "${DEFAULT_FAST}" \ @@ -91,7 +95,7 @@ check_gsutil_opts CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin" # Before we can run any tools, we need to update chroot or setup_board. -UPDATE_ARGS=() +UPDATE_ARGS=( --regen_configs ) if [[ -n ${FLAGS_accept_licenses} ]]; then UPDATE_ARGS+=( --accept_licenses "${FLAGS_accept_licenses}" ) fi @@ -107,6 +111,14 @@ if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then else UPDATE_ARGS+=( --nogetbinpkg ) fi + if [[ "${FLAGS_toolchainpkgonly}" -eq "${FLAGS_TRUE}" ]]; then + UPDATE_ARGS+=( --toolchainpkgonly ) + else + UPDATE_ARGS+=( --notoolchainpkgonly ) + fi + if [[ -n "${FLAGS_getbinpkgver}" ]]; then + UPDATE_ARGS+=( --getbinpkgver="${FLAGS_getbinpkgver}" ) + fi else UPDATE_ARGS+=( --nousepkg ) fi diff --git a/setup_board b/setup_board index ff278bc58a..dcde4975a3 100755 --- a/setup_board +++ b/setup_board @@ -33,6 +33,10 @@ show_help_if_requested "$@" # The following options are advanced options, only available to those willing # to read the source code. They are not shown in help output, since they are # not needed for the typical developer workflow. +DEFINE_string getbinpkgver "" \ + "Use binary packages from a specific version." +DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \ + "Use binary packages only for the board toolchain." DEFINE_string accept_licenses "" \ "Licenses to append to the accept list." DEFINE_boolean fast "${DEFAULT_FAST}" \ @@ -124,6 +128,16 @@ EOF sudo_multi "${cmds[@]}" } +generate_binhost_list() { + local t + [[ "${FLAGS_toolchainpkgonly}" -eq "${FLAGS_TRUE}" ]] && t="-t" + FLAGS_getbinpkgver="${FLAGS_getbinpkgver/current/${COREOS_VERSION_ID}}" + FLAGS_getbinpkgver="${FLAGS_getbinpkgver/latest/${COREOS_VERSION_ID}}" + FLAGS_getbinpkgver="${FLAGS_getbinpkgver/sdk/${COREOS_SDK_VERSION}}" + + get_board_binhost $t "${BOARD}" ${FLAGS_getbinpkgver} +} + # Parse command line flags FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" @@ -180,6 +194,7 @@ BOARD_PROFILE="${BOARD_ETC}/portage/profile" BOARD_ARCH=$(get_board_arch "$BOARD") BOARD_CHOST=$(get_board_chost ${BOARD}) PORTAGE_PROFILE=$(get_board_profile "$BOARD") +BOARD_BINHOST=$(generate_binhost_list) if [ -d "${BOARD_ROOT}" ]; then if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then @@ -237,6 +252,7 @@ ROOT="${BOARD_ROOT}/" MAKEOPTS="--jobs=${NUM_JOBS} --load-average=${NUM_JOBS}" PKG_CONFIG="pkg-config-${BOARD_VARIANT}" BOARD_USE="${BOARD_VARIANT}" +PORTAGE_BINHOST="${BOARD_BINHOST}" ${ACCEPT_LICENSE} EOF From 30b72754ff967fa0425c8bc1d82d3813a538a315 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 9 Feb 2014 14:48:45 -0800 Subject: [PATCH 4/4] fix(setup_board): Remove note about annoying eselect warning Silenced in eselect 1.4 --- build_library/toolchain_util.sh | 1 - setup_board | 1 - 2 files changed, 2 deletions(-) diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 74f40d7c3e..8ed3162de3 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -173,7 +173,6 @@ _configure_sysroot() { fi $sudo mkdir -p "${ROOT}/etc/portage" - echo "eselect will report '!!! Warning: Strange path.' but that's OK" $sudo eselect profile set --force "$profile" $sudo tee "${ROOT}/etc/portage/make.conf" >/dev/null <