Merge pull request #8 from marineam/buildbot

Buildbot

bootstrap_sdk

@@ -36,9 +36,14 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 TYPE="coreos-sdk"
 ARCH=$(portageq envvar ARCH)
 DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
+UPLOAD_ROOT="gs://storage.core-os.net/coreos/sdk/${ARCH}"
 
 . "${SCRIPT_ROOT}/lib/catalyst.sh" || exit 1
 
+DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil."
+DEFINE_boolean upload ${FLAGS_FALSE} \
+    "Upload final tarball and all packages via gsutil."
+
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
@@ -51,4 +56,61 @@ catalyst_stage_default
 }
 
 catalyst_init "$@"
+
+# Search for .boto, this runs under sudo so ~/.boto may not be right
+if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
+    for boto in "$HOME/.boto" "/home/$SUDO_USER/.boto" /etc/boto.cfg; do
+        if [[ -f "$boto" ]]; then
+            info "Using boto config $boto"
+            export BOTO_CONFIG="$boto"
+            break
+        fi
+    done
+    if [[ ! -f "$BOTO_CONFIG" ]]; then
+        die_notrace "Please run gsutil config to create ~/.boto"
+    fi
+fi
+
 catalyst_build
+
+info "Build complete! Changing output name to something more sensible."
+build_name="stage4-${ARCH}-${FLAGS_version}.tar.bz2"
+release_name="${TYPE}-${ARCH}-${FLAGS_version}.tar.bz2"
+ln -f "$BUILDS/${build_name}" "$BUILDS/${release_name}"
+ln -f "$BUILDS/${build_name}.CONTENTS" "$BUILDS/${release_name}.CONTENTS"
+sed -e "s/${build_name}/${release_name}/" \
+    "$BUILDS/${build_name}.DIGESTS" > "$BUILDS/${release_name}.DIGESTS"
+
+# Validate we didn't break the DIGESTS with sed
+for hash_type in md5 sha1 sha512; do
+    info "Validating ${hash_type} DIGESTS"
+    # shash is what's used to generate these multi-hash digests but it
+    # doesn't exit with non-zero on failure. I mean seriously...
+    #shash -c "$BUILDS/${release_name}.DIGESTS" -a "${hash_type}"
+    # So we do it the hard way...
+    grep -qi "^# ${hash_type} HASH$" "$BUILDS/${release_name}.DIGESTS"
+    (cd "$BUILDS" && grep -A1 -i "^# ${hash_type} HASH$" \
+        "${release_name}.DIGESTS" | grep -v '^--$' | \
+        ${hash_type}sum -c - --strict)
+done
+
+info "SDK ready: $BUILDS/${release_name}"
+
+GSUTIL_OPTS=
+if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
+    GSUTIL_OPTS="-m"
+fi
+
+if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
+    info "Uploading tarball"
+    gsutil ${GSUTIL_OPTS} cp \
+        "$BUILDS/${release_name}" \
+        "$BUILDS/${release_name}.CONTENTS" \
+        "$BUILDS/${release_name}.DIGESTS" \
+        "${UPLOAD_ROOT}/${FLAGS_version}/"
+    info "Uploading packages"
+    gsutil ${GSUTIL_OPTS} cp -R "${BINPKGS}"/* \
+        "${UPLOAD_ROOT}/${FLAGS_version}/pkgs/"
+fi
+
+info "Done!"

lib/catalyst.sh

@@ -22,6 +22,12 @@ DISTDIR=
 TEMPDIR=
 STAGES=
 
+# For searching for alternatives when DEFAULT_SEED doesn't exist
+# unset SDK_SEARCH=1 to disable this fallback
+SDK_VERSION_FILE="coreos/binhost/host/sdk_version.conf"
+SDK_TARBALL_FMT="coreos-sdk-${ARCH}-%s.tar.bz2"
+SDK_SEARCH=1
+
 DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \
     "Path to directory for all catalyst images and other files."
 DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \
@@ -166,7 +172,10 @@ catalyst_init() {
     TEMPDIR="$CATALYST_ROOT/tmp/$TYPE"
     DISTDIR="$CATALYST_ROOT/distfiles"
 
-    # check for recent seed
+    # possibly search for existing seeds
+    search_for_sdk_seed
+
+    # confirm seed exists
     if [[ ! -f "$FLAGS_seed_tarball" ]]; then
         die_notrace "Seed tarball not found: $FLAGS_seed_tarball"
     fi
@@ -192,6 +201,36 @@ catalyst_init() {
     fi
 }
 
+# search_for_sdk_seed
+# As a fallback search around for an existing SDK tarball we
+# can use as a seed when the default doesn't exist.
+search_for_sdk_seed() {
+    # Search disabled
+    [[ "${SDK_SEARCH}" != 1 ]] && return
+    # Seed already exists
+    [[ -f "${FLAGS_seed_tarball}" ]] && return
+    # User set the option so we shouldn't change it
+    [[ "${FLAGS_seed_tarball}" != "${DEFAULT_SEED}" ]] && return
+
+    local SDK_LATEST_VERSION SDK_TARBALL check_path
+    eval $(grep "^SDK_LATEST_VERSION=" \
+            "${FLAGS_coreos_overlay}/${SDK_VERSION_FILE}")
+    SDK_TARBALL=$(printf "${SDK_TARBALL_FMT}" "${SDK_LATEST_VERSION}")
+
+    for check_path in \
+        "${CATALYST_ROOT}/builds/coreos-sdk/${SDK_TARBALL}" \
+        "${CATALYST_ROOT}/builds/seeds/${SDK_TARBALL}" \
+        "/var/cache/chromeos-cache/sdks/${SDK_TARBALL}" \
+        "/mnt/host/source/.cache/sdks/${SDK_TARBALL}"
+    do
+        if [[ -f "${check_path}" ]]; then
+            info "Using SDK for seed: ${check_path}"
+            FLAGS_seed_tarball="${check_path}"
+            return
+        fi
+    done
+}
+
 write_configs() {
     # No catalyst config option, so defined via environment
     export CCACHE_DIR="$TEMPDIR/ccache"

sdk_lib/enter_chroot.sh

@@ -83,6 +83,8 @@ FILES_TO_COPY_TO_CHROOT=(
   .gdata_token                # Auth token for Google Docs on chromium.org
   .disable_build_stats_upload # Presence of file disables command stats upload
   .netrc                      # May contain required source fetching credentials
+  .boto                       # Auth information for gsutil
+  .boto-key.p12               # Service account key for gsutil
 )
 
 INNER_CHROME_ROOT=$FLAGS_chrome_root_mount  # inside chroot
@@ -434,18 +436,6 @@ setup_env() {
     # semaphores.
     chmod -R 777 "${FLAGS_chroot}/dev/shm"
 
-    # If the private overlays are installed, gsutil can use those credentials.
-    # We're also installing credentials for use by sudoed invocations.
-    boto='src/private-overlays/coreos-overlay/googlestorage_account.boto'
-    if [ -s "${FLAGS_trunk}/${boto}" ]; then
-      if [ ! -L "${FLAGS_chroot}/home/${SUDO_USER}/.boto" ]; then
-        user_symlink "trunk/${boto}" "${FLAGS_chroot}/home/${SUDO_USER}/.boto"
-      fi
-      if [ ! -L "${FLAGS_chroot}/root/.boto" ]; then
-        ln -sf "${CHROOT_TRUNK_DIR}/${boto}" "${FLAGS_chroot}/root/.boto"
-      fi
-    fi
-
     # Have found a few chroots where ~/.gsutil is owned by root:root, probably
     # as a result of old gsutil or tools. This causes permission errors when
     # gsutil cp tries to create its cache files, so ensure the user can

update_distfiles

@@ -8,8 +8,14 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/common.sh" || exit 1
 
 DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes."
+DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil."
 DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil."
 
+# FIXME(marineam): We need to add gs support to emirrordist so it
+# doesn't have to operate on a local copy of the complete mirror.
+DEFINE_boolean download ${FLAGS_FALSE} \
+    "Download the current mirror before making updates to it."
+
 MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror"
 UPLOAD_ROOT="gs://storage.core-os.net/mirror"
 
@@ -22,6 +28,12 @@ if [[ $# -eq 0 ]]; then
     eval set -- portage-stable coreos
 fi
 
+GSUTIL_OPTS=
+if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
+    GSUTIL_OPTS="-m"
+fi
+
+
 update_local_mirror() {
     local repo_name="$1"
     local repo_mirror="${MIRROR_ROOT}/$repo_name"
@@ -34,7 +46,7 @@ update_local_mirror() {
         info "Starting distfiles update for $repo_name"
     fi
 
-    mkdir -p "${repo_mirror}/"{distfiles,info,log}
+    mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp}
     emirrordist --mirror --verbose $extra_flags \
         --jobs=${NUM_JOBS} --repo="${repo_name}" \
         --distfiles="${repo_mirror}/distfiles" \
@@ -47,6 +59,7 @@ update_local_mirror() {
         --distfiles-db="${repo_mirror}/info/distfiles.db" \
         --deletion-delay=$((86400 * 14)) \
         --restrict-mirror-exemptions="gentoo" \
+        --temp-dir="${repo_mirror}/tmp" \
         --verify-existing-digest
 }
 upload_mirror() {
@@ -55,13 +68,35 @@ upload_mirror() {
     local remote_mirror="${UPLOAD_ROOT}/$repo_name"
 
     info "Uploading public distfiles for $repo_name"
-    gsutil -m cp -n \
+    gsutil ${GSUTIL_OPTS} cp -n \
         "${local_mirror}/distfiles/*" "${remote_mirror}/distfiles"
 
     info "Uploading private metadata for $repo_name"
-    gsutil cp -a project-private \
+    gsutil ${GSUTIL_OPTS} cp -a project-private \
         "${local_mirror}/info/*" "${remote_mirror}/info"
 }
+download_mirror() {
+    local repo_name="$1"
+    local local_mirror="${MIRROR_ROOT}/$repo_name"
+    local remote_mirror="${UPLOAD_ROOT}/$repo_name"
+
+    info "Downloading public distfiles for $repo_name"
+    mkdir -p "${local_mirror}/"{distfiles,info}
+    gsutil ${GSUTIL_OPTS} cp -n \
+        "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles"
+
+    info "Downloading private metadata for $repo_name"
+    gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info"
+}
+
+if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then
+    if [[ ! -f "$HOME/.boto" ]]; then
+        die_notrace "Please run gsutil config to create ~/.boto"
+    fi
+    for repo in "$@"; do
+        download_mirror "$repo"
+    done
+fi
 
 for repo in "$@"; do
     if ! portageq get_repo_path / "$repo" >/dev/null; then
@@ -70,6 +105,11 @@ for repo in "$@"; do
     update_local_mirror "$repo"
 done
 
+if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
+    info "Dry-run complete."
+    exit
+fi
+
 if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
     if [[ ! -f "$HOME/.boto" ]]; then
         die_notrace "Please run gsutil config to create ~/.boto"