diff --git a/bootstrap_sdk b/bootstrap_sdk index 5f20dd929b..189cd689b3 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -36,9 +36,14 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) TYPE="coreos-sdk" ARCH=$(portageq envvar ARCH) DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" +UPLOAD_ROOT="gs://storage.core-os.net/coreos/sdk/${ARCH}" . "${SCRIPT_ROOT}/lib/catalyst.sh" || exit 1 +DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil." +DEFINE_boolean upload ${FLAGS_FALSE} \ + "Upload final tarball and all packages via gsutil." + ## Define the stage4 config template catalyst_stage4() { cat < "$BUILDS/${release_name}.DIGESTS" + +# Validate we didn't break the DIGESTS with sed +for hash_type in md5 sha1 sha512; do + info "Validating ${hash_type} DIGESTS" + # shash is what's used to generate these multi-hash digests but it + # doesn't exit with non-zero on failure. I mean seriously... + #shash -c "$BUILDS/${release_name}.DIGESTS" -a "${hash_type}" + # So we do it the hard way... + grep -qi "^# ${hash_type} HASH$" "$BUILDS/${release_name}.DIGESTS" + (cd "$BUILDS" && grep -A1 -i "^# ${hash_type} HASH$" \ + "${release_name}.DIGESTS" | grep -v '^--$' | \ + ${hash_type}sum -c - --strict) +done + +info "SDK ready: $BUILDS/${release_name}" + +GSUTIL_OPTS= +if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then + GSUTIL_OPTS="-m" +fi + +if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then + info "Uploading tarball" + gsutil ${GSUTIL_OPTS} cp \ + "$BUILDS/${release_name}" \ + "$BUILDS/${release_name}.CONTENTS" \ + "$BUILDS/${release_name}.DIGESTS" \ + "${UPLOAD_ROOT}/${FLAGS_version}/" + info "Uploading packages" + gsutil ${GSUTIL_OPTS} cp -R "${BINPKGS}"/* \ + "${UPLOAD_ROOT}/${FLAGS_version}/pkgs/" +fi + +info "Done!" diff --git a/lib/catalyst.sh b/lib/catalyst.sh index 73c195e256..a8e536cfd5 100644 --- a/lib/catalyst.sh +++ b/lib/catalyst.sh @@ -22,6 +22,12 @@ DISTDIR= TEMPDIR= STAGES= +# For searching for alternatives when DEFAULT_SEED doesn't exist +# unset SDK_SEARCH=1 to disable this fallback +SDK_VERSION_FILE="coreos/binhost/host/sdk_version.conf" +SDK_TARBALL_FMT="coreos-sdk-${ARCH}-%s.tar.bz2" +SDK_SEARCH=1 + DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \ "Path to directory for all catalyst images and other files." DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \ @@ -166,7 +172,10 @@ catalyst_init() { TEMPDIR="$CATALYST_ROOT/tmp/$TYPE" DISTDIR="$CATALYST_ROOT/distfiles" - # check for recent seed + # possibly search for existing seeds + search_for_sdk_seed + + # confirm seed exists if [[ ! -f "$FLAGS_seed_tarball" ]]; then die_notrace "Seed tarball not found: $FLAGS_seed_tarball" fi @@ -192,6 +201,36 @@ catalyst_init() { fi } +# search_for_sdk_seed +# As a fallback search around for an existing SDK tarball we +# can use as a seed when the default doesn't exist. +search_for_sdk_seed() { + # Search disabled + [[ "${SDK_SEARCH}" != 1 ]] && return + # Seed already exists + [[ -f "${FLAGS_seed_tarball}" ]] && return + # User set the option so we shouldn't change it + [[ "${FLAGS_seed_tarball}" != "${DEFAULT_SEED}" ]] && return + + local SDK_LATEST_VERSION SDK_TARBALL check_path + eval $(grep "^SDK_LATEST_VERSION=" \ + "${FLAGS_coreos_overlay}/${SDK_VERSION_FILE}") + SDK_TARBALL=$(printf "${SDK_TARBALL_FMT}" "${SDK_LATEST_VERSION}") + + for check_path in \ + "${CATALYST_ROOT}/builds/coreos-sdk/${SDK_TARBALL}" \ + "${CATALYST_ROOT}/builds/seeds/${SDK_TARBALL}" \ + "/var/cache/chromeos-cache/sdks/${SDK_TARBALL}" \ + "/mnt/host/source/.cache/sdks/${SDK_TARBALL}" + do + if [[ -f "${check_path}" ]]; then + info "Using SDK for seed: ${check_path}" + FLAGS_seed_tarball="${check_path}" + return + fi + done +} + write_configs() { # No catalyst config option, so defined via environment export CCACHE_DIR="$TEMPDIR/ccache" diff --git a/sdk_lib/enter_chroot.sh b/sdk_lib/enter_chroot.sh index 128fdeba0c..d4d421be44 100755 --- a/sdk_lib/enter_chroot.sh +++ b/sdk_lib/enter_chroot.sh @@ -83,6 +83,8 @@ FILES_TO_COPY_TO_CHROOT=( .gdata_token # Auth token for Google Docs on chromium.org .disable_build_stats_upload # Presence of file disables command stats upload .netrc # May contain required source fetching credentials + .boto # Auth information for gsutil + .boto-key.p12 # Service account key for gsutil ) INNER_CHROME_ROOT=$FLAGS_chrome_root_mount # inside chroot @@ -434,18 +436,6 @@ setup_env() { # semaphores. chmod -R 777 "${FLAGS_chroot}/dev/shm" - # If the private overlays are installed, gsutil can use those credentials. - # We're also installing credentials for use by sudoed invocations. - boto='src/private-overlays/coreos-overlay/googlestorage_account.boto' - if [ -s "${FLAGS_trunk}/${boto}" ]; then - if [ ! -L "${FLAGS_chroot}/home/${SUDO_USER}/.boto" ]; then - user_symlink "trunk/${boto}" "${FLAGS_chroot}/home/${SUDO_USER}/.boto" - fi - if [ ! -L "${FLAGS_chroot}/root/.boto" ]; then - ln -sf "${CHROOT_TRUNK_DIR}/${boto}" "${FLAGS_chroot}/root/.boto" - fi - fi - # Have found a few chroots where ~/.gsutil is owned by root:root, probably # as a result of old gsutil or tools. This causes permission errors when # gsutil cp tries to create its cache files, so ensure the user can diff --git a/update_distfiles b/update_distfiles index a61ba63c02..488950e6b6 100755 --- a/update_distfiles +++ b/update_distfiles @@ -8,8 +8,14 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes." +DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil." DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil." +# FIXME(marineam): We need to add gs support to emirrordist so it +# doesn't have to operate on a local copy of the complete mirror. +DEFINE_boolean download ${FLAGS_FALSE} \ + "Download the current mirror before making updates to it." + MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror" UPLOAD_ROOT="gs://storage.core-os.net/mirror" @@ -22,6 +28,12 @@ if [[ $# -eq 0 ]]; then eval set -- portage-stable coreos fi +GSUTIL_OPTS= +if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then + GSUTIL_OPTS="-m" +fi + + update_local_mirror() { local repo_name="$1" local repo_mirror="${MIRROR_ROOT}/$repo_name" @@ -34,7 +46,7 @@ update_local_mirror() { info "Starting distfiles update for $repo_name" fi - mkdir -p "${repo_mirror}/"{distfiles,info,log} + mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp} emirrordist --mirror --verbose $extra_flags \ --jobs=${NUM_JOBS} --repo="${repo_name}" \ --distfiles="${repo_mirror}/distfiles" \ @@ -47,6 +59,7 @@ update_local_mirror() { --distfiles-db="${repo_mirror}/info/distfiles.db" \ --deletion-delay=$((86400 * 14)) \ --restrict-mirror-exemptions="gentoo" \ + --temp-dir="${repo_mirror}/tmp" \ --verify-existing-digest } upload_mirror() { @@ -55,13 +68,35 @@ upload_mirror() { local remote_mirror="${UPLOAD_ROOT}/$repo_name" info "Uploading public distfiles for $repo_name" - gsutil -m cp -n \ + gsutil ${GSUTIL_OPTS} cp -n \ "${local_mirror}/distfiles/*" "${remote_mirror}/distfiles" info "Uploading private metadata for $repo_name" - gsutil cp -a project-private \ + gsutil ${GSUTIL_OPTS} cp -a project-private \ "${local_mirror}/info/*" "${remote_mirror}/info" } +download_mirror() { + local repo_name="$1" + local local_mirror="${MIRROR_ROOT}/$repo_name" + local remote_mirror="${UPLOAD_ROOT}/$repo_name" + + info "Downloading public distfiles for $repo_name" + mkdir -p "${local_mirror}/"{distfiles,info} + gsutil ${GSUTIL_OPTS} cp -n \ + "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles" + + info "Downloading private metadata for $repo_name" + gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info" +} + +if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then + if [[ ! -f "$HOME/.boto" ]]; then + die_notrace "Please run gsutil config to create ~/.boto" + fi + for repo in "$@"; do + download_mirror "$repo" + done +fi for repo in "$@"; do if ! portageq get_repo_path / "$repo" >/dev/null; then @@ -70,6 +105,11 @@ for repo in "$@"; do update_local_mirror "$repo" done +if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then + info "Dry-run complete." + exit +fi + if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then if [[ ! -f "$HOME/.boto" ]]; then die_notrace "Please run gsutil config to create ~/.boto"