diff --git a/bootstrap_sdk b/bootstrap_sdk index 24c5afd86d..668f3ef4e1 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -36,17 +36,11 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) TYPE="coreos-sdk" ARCH=$(portageq envvar ARCH) DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" -UPLOAD_ROOT="gs://storage.core-os.net/coreos/sdk/${ARCH}" -UPLOAD_DEFAULT=${FLAGS_FALSE} -if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then - UPLOAD_DEFAULT=${FLAGS_TRUE} -fi . "${SCRIPT_ROOT}/lib/catalyst.sh" || exit 1 -DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil." -DEFINE_boolean upload ${UPLOAD_DEFAULT} \ - "Upload final tarball and all packages via gsutil." +# include upload options +. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 ## Define the stage4 config template catalyst_stage4() { @@ -60,21 +54,7 @@ catalyst_stage_default } catalyst_init "$@" - -# Search for .boto, this runs under sudo so ~/.boto may not be right -if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then - for boto in "$HOME/.boto" "/home/$SUDO_USER/.boto" /etc/boto.cfg; do - if [[ -f "$boto" ]]; then - info "Using boto config $boto" - export BOTO_CONFIG="$boto" - break - fi - done - if [[ ! -f "$BOTO_CONFIG" ]]; then - die_notrace "Please run gsutil config to create ~/.boto" - fi -fi - +check_gsutil_opts catalyst_build info "Build complete! Changing output name to something more sensible." @@ -100,21 +80,9 @@ done info "SDK ready: $BUILDS/${release_name}" -GSUTIL_OPTS= -if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then - GSUTIL_OPTS="-m" -fi - -if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then - info "Uploading tarball" - gsutil ${GSUTIL_OPTS} cp \ - "$BUILDS/${release_name}" \ - "$BUILDS/${release_name}.CONTENTS" \ - "$BUILDS/${release_name}.DIGESTS" \ - "${UPLOAD_ROOT}/${FLAGS_version}/" - info "Uploading packages" - gsutil ${GSUTIL_OPTS} cp -R "${BINPKGS}"/* \ - "${UPLOAD_ROOT}/${FLAGS_version}/pkgs/" -fi +def_upload_path="${UPLOAD_ROOT}/coreos/sdk/${ARCH}/${FLAGS_version}" +upload_files "tarball" "${def_upload_path}" "" "$BUILDS/${release_name}" \ + "$BUILDS/${release_name}.CONTENTS" "$BUILDS/${release_name}.DIGESTS" +upload_files "packages" "${def_upload_path}" "pkgs/" "${BINPKGS}"/* command_completed diff --git a/build_library/release_util.sh b/build_library/release_util.sh index cbd75476fb..bab54f07f6 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -32,20 +32,55 @@ check_gsutil_opts() { UPLOAD_PATH="${FLAGS_upload_path%%/}" fi - if [[ ! -f "$HOME/.boto" ]]; then + # Search for .boto, may be run via sudo + local boto + for boto in "$HOME/.boto" "/home/$SUDO_USER/.boto"; do + if [[ -f "$boto" ]]; then + info "Using boto config $boto" + export BOTO_CONFIG="$boto" + break + fi + done + if [[ ! -f "$BOTO_CONFIG" ]]; then die_notrace "Please run gsutil config to create ~/.boto" fi } +# Generic upload function +# Usage: upload_files "file type" "${UPLOAD_ROOT}/default/path" "" files... +# arg1: file type reported via log +# arg2: default upload path, overridden by --upload_path +# arg3: upload path suffix that can't be overridden, must end in / +# argv: remaining args are files or directories to upload +upload_files() { + [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 + + local msg="$1" + local local_upload_path="$2" + local extra_upload_suffix="$3" + shift 3 + + if [[ -n "${UPLOAD_PATH}" ]]; then + local_upload_path="${UPLOAD_PATH}" + fi + + if [[ -n "${extra_upload_suffix}" && "${extra_upload_suffix}" != */ ]] + then + die "upload suffix '${extra_upload_suffix}' doesn't end in /" + fi + + info "Uploading ${msg} to ${local_upload_path}" + gsutil ${GSUTIL_OPTS} cp -R "$@" \ + "${local_upload_path}/${extra_upload_suffix}" +} + upload_packages() { [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" - local BOARD_PACKAGES="${1:-"${BOARD_ROOT}/packages"}" - : ${UPLOAD_PATH:="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}"} - - info "Uploading packages" - gsutil ${GSUTIL_OPTS} cp -R "${BOARD_PACKAGES}"/* "${UPLOAD_PATH}/pkgs/" + local board_packages="${1:-"${BOARD_ROOT}/packages"}" + local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}" + upload_files packages ${def_upload_path} "pkgs/" "${board_packages}"/* } make_digests() { @@ -66,7 +101,6 @@ upload_image() { [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" local BUILT_IMAGE="$1" - : ${UPLOAD_PATH:="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}"} if [[ ! -f "${BUILT_IMAGE}" ]]; then die "Image '${BUILT_IMAGE}' does not exist!" @@ -83,7 +117,8 @@ upload_image() { # produces for the SDK tarballs and up upload it too. make_digests "${BUILT_IMAGE}" - info "Uploading ${BUILT_IMAGE##*/}" - gsutil ${GSUTIL_OPTS} cp "${BUILT_IMAGE}" \ - "${BUILT_IMAGE}.DIGESTS" "${UPLOAD_PATH}/" + local log_msg="${BUILT_IMAGE##*/}" + local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}" + upload_files "${log_msg}" "${def_upload_path}" "" \ + "${BUILT_IMAGE}" "${BUILT_IMAGE}.DIGESTS" } diff --git a/lib/catalyst.sh b/lib/catalyst.sh index 334b39410c..7b514bd745 100644 --- a/lib/catalyst.sh +++ b/lib/catalyst.sh @@ -41,8 +41,6 @@ DEFINE_string profile "${DEFAULT_PROFILE}" \ "Portage profile, may be prefixed with repo:" DEFINE_boolean rebuild ${FLAGS_FALSE} \ "Rebuild and overwrite stages that already exist." -DEFINE_integer make_jobs ${NUM_JOBS} "Tune make's concurrency." -DEFINE_integer emerge_jobs ${NUM_JOBS} "Tune emerge's concurrency." DEFINE_boolean debug ${FLAGS_FALSE} "Enable verbose output from catalyst." ##################### @@ -73,9 +71,9 @@ EOF } catalystrc() { -cat "${SCRIPTS_DIR}/lib/catalystrc" -echo "export MAKEOPTS=-j$FLAGS_make_jobs" -echo "export EMERGE_DEFAULT_OPTS=--jobs=$FLAGS_emerge_jobs" +echo "export MAKEOPTS='--jobs=${NUM_JOBS} --load-average=${NUM_JOBS}'" +echo "export EMERGE_DEFAULT_OPTS=--jobs=${NUM_JOBS}" +echo "export FEATURES='parallel-install -ebuild-locks'" } # Common values for all stage spec files diff --git a/lib/catalyst_sdk_stage4.sh b/lib/catalyst_sdk_stage4.sh index 44ad54587f..2012daa05f 100644 --- a/lib/catalyst_sdk_stage4.sh +++ b/lib/catalyst_sdk_stage4.sh @@ -11,12 +11,11 @@ for cross_chost in x86_64-cros-linux-gnu; do --stable --ex-gdb --init-target \ --target "${cross_chost}" || exit 1 - # If PKGCACHE is enabled check to see if binary packages are available - # or (due to --noreplace) the packages are already installed. If so then - # we don't need to perform a full bootstrap and just call emerge instead. + # If PKGCACHE is enabled check to see if binary packages are available. + # If so then don't perform a full bootstrap and just call emerge instead. if [[ -n "${clst_PKGCACHE}" ]] && \ emerge ${clst_myemergeopts} --usepkgonly --binpkg-respect-use=y \ - --noreplace "${cross_pkgs[@]}" &>/dev/null + --pretend "${cross_pkgs[@]}" &>/dev/null then run_merge -u "${cross_pkgs[@]}" else @@ -26,6 +25,11 @@ for cross_chost in x86_64-cros-linux-gnu; do --stable --ex-gdb --stage4 \ --target "${cross_chost}" || exit 1 fi + + # There is no point to including the built packages in the final tarball + # because the packages will have to be downloaded anyway due to how the + # cross toolchains are managed in board sysroots. + crossdev --force -C "${cross_chost}" done echo "Double checking everything is fresh and happy." diff --git a/lib/catalystrc b/lib/catalystrc deleted file mode 100644 index 06f12c11cd..0000000000 --- a/lib/catalystrc +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# setup some configs catalyst doesn't handle -# i.e: update locale.gen to speed up builds -maybe_append() { - [[ -f "$1" && ! -w "$1" ]] && return - if ! grep -q "^$2" "$1"; then - echo "$2" >> "$1" - fi -} - -fixup_rootfs() { - mkdir -p "$1/etc" 2>/dev/null || return - maybe_append "$1/etc/locale.gen" "en_US ISO-8859-1" - maybe_append "$1/etc/locale.gen" "en_US.UTF-8 UTF-8" -} - -# Fix both / and $ROOT (if it exists) -fixup_rootfs -[[ "${ROOT:-/}" != / ]] && fixup_rootfs "$ROOT" -unset fixup_rootfs maybe_append diff --git a/update_chroot b/update_chroot index ac146ad916..c6c69160eb 100755 --- a/update_chroot +++ b/update_chroot @@ -92,7 +92,7 @@ fi # can cause serious issues later. if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" ]; then # First update crossdev. - sudo -E ${EMERGE_CMD} ${EMERGE_FLAGS} crossdev + sudo -E ${EMERGE_CMD} ${EMERGE_FLAGS} crossdev sysroot-wrappers TOOLCHAIN_FLAGS=( "--include-boards=${FLAGS_toolchain_boards}" ) # This should really only be skipped while bootstrapping.