From 9f8eac6540dd7b499b003222ef8fd2874a0f8815 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Fri, 22 Nov 2024 12:56:55 +0000 Subject: [PATCH] Drop lots of obsolete release signing and upload code Most of this hinges on the --upload option being passed, and it never is any more. Much of it also uses Google Buckets, which we no longer use, save for some GCE-specific bits. Signed-off-by: James Le Cuirot --- bootstrap_sdk | 13 -- build_image | 6 - build_library/build_image_util.sh | 25 +-- build_library/dev_container_util.sh | 16 +- build_library/generate_au_zip.py | 2 - build_library/modify_image_util.sh | 18 +- build_library/prod_image_util.sh | 64 +------ build_library/release_util.sh | 275 ---------------------------- build_library/vm_image_util.sh | 49 +---- build_packages | 5 - build_toolchains | 10 - core_date | 53 ------ core_dev_sign_update | 71 ------- core_pre_alpha | 41 ----- core_roller_upload | 64 ------- image_inject_bootchain | 2 - image_set_group | 2 - image_to_vm.sh | 6 +- rebuild_packages | 5 - signing/devel.key.pem | 27 --- signing/devel.pub.pem | 9 - signing/new_key.sh | 6 - signing/print_key.sh | 8 - signing/prod-2.pub.pem | 9 - signing/transfer.sh | 251 ------------------------- update_distfiles | 55 ------ 26 files changed, 9 insertions(+), 1083 deletions(-) delete mode 100755 core_date delete mode 100755 core_dev_sign_update delete mode 100755 core_pre_alpha delete mode 100755 core_roller_upload delete mode 100644 signing/devel.key.pem delete mode 100644 signing/devel.pub.pem delete mode 100755 signing/new_key.sh delete mode 100755 signing/print_key.sh delete mode 100644 signing/prod-2.pub.pem delete mode 100755 signing/transfer.sh diff --git a/bootstrap_sdk b/bootstrap_sdk index 812a86209e..834506ee6d 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -60,7 +60,6 @@ GENTOO_MIRRORS="${GENTOO_MIRRORS//https:\/\//http://}" export GENTOO_MIRRORS catalyst_init "$@" -check_gsutil_opts ROOT_OVERLAY=${TEMPDIR}/stage4_overlay if [[ "$STAGES" =~ stage4 ]]; then @@ -108,18 +107,6 @@ if [[ "$STAGES" =~ stage4 ]]; then verify_digests "${release_image}" "${release_contents}" info "SDK ready: ${release_image}" - - def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}" - sign_and_upload_files "tarball" "${def_upload_path}" "" \ - "${release_image}" "${release_contents}" "${release_digests}" - sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \ - "${BINPKGS}"/* - - if [ -d "${BINPKGS}/crossdev" ]; then - # Upload the SDK toolchain packages - sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \ - "toolchain/" "${BINPKGS}/crossdev"/* - fi fi command_completed diff --git a/build_image b/build_image index fb68cf3623..a1356b03f0 100755 --- a/build_image +++ b/build_image @@ -87,8 +87,6 @@ eval set -- "${FLAGS_ARGV:-prod}" # so will die prematurely if 'switch_to_strict_mode' is specified before now. switch_to_strict_mode -check_gsutil_opts - # If downloading packages is enabled ensure the board is configured properly. if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \ @@ -202,8 +200,6 @@ FLATCAR_BUILD_ID="${FLATCAR_BUILD_ID}" FLATCAR_SDK_VERSION=${FLATCAR_SDK_VERSION} EOF -upload_image "${BUILD_DIR}/version.txt" - # Create a named symlink. set_build_symlinks latest "${FLAGS_group}-latest" @@ -230,5 +226,3 @@ if [[ "${PROD_IMAGE}" -eq 1 ]]; then fi command_completed - - diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 8edaaf355e..a29470dc82 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -62,22 +62,13 @@ extract_update() { local image_name="$1" local disk_layout="$2" local update_path="${BUILD_DIR}/${image_name%_image.bin}_update.bin" - local digest_path="${update_path}.DIGESTS" "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ extract "${BUILD_DIR}/${image_name}" "USR-A" "${update_path}" # Compress image files_to_evaluate+=( "${update_path}" ) - declare -a compressed_images - declare -a extra_files - compress_disk_images files_to_evaluate compressed_images extra_files - - # Upload compressed image - upload_image -d "${digest_path}" "${compressed_images[@]}" "${extra_files[@]}" - - # Upload legacy digests - upload_legacy_digests "${digest_path}" compressed_images + compress_disk_images files_to_evaluate # For production as well as dev builds we generate a dev-key-signed update # payload for running tests (the signature won't be accepted by production systems). @@ -87,8 +78,6 @@ extract_update() { -new_image "${update_path}" \ -new_kernel "${BUILD_DIR}/${image_name%.bin}.vmlinuz" \ -out_file "${update_test}" - - upload_image "${update_test}" } zip_update_tools() { @@ -100,8 +89,6 @@ zip_update_tools() { export REPO_MANIFESTS_DIR SCRIPTS_DIR "${BUILD_LIBRARY_DIR}/generate_au_zip.py" \ --arch "$(get_sdk_arch)" --output-dir "${BUILD_DIR}" --zip-name "${update_zip}" - - upload_image "${BUILD_DIR}/${update_zip}" } generate_update() { @@ -123,16 +110,8 @@ generate_update() { # Compress image declare -a files_to_evaluate - declare -a compressed_images - declare -a extra_files files_to_evaluate+=( "${update}.bin" ) - compress_disk_images files_to_evaluate compressed_images extra_files - - # Upload images - upload_image -d "${update}.DIGESTS" "${update}".{gz,zip} "${compressed_images[@]}" "${extra_files[@]}" - - # Upload legacy digests - upload_legacy_digests "${update}.DIGESTS" compressed_images + compress_disk_images files_to_evaluate } # ldconfig cannot generate caches for non-native arches. diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 3d702c32eb..f87bd1235a 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -115,20 +115,6 @@ create_dev_container() { finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}" declare -a files_to_evaluate - declare -a compressed_images - declare -a extra_files - files_to_evaluate+=( "${BUILD_DIR}/${image_name}" ) - compress_disk_images files_to_evaluate compressed_images extra_files - - upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \ - "${BUILD_DIR}/${image_contents}" \ - "${BUILD_DIR}/${image_contents_wtd}" \ - "${BUILD_DIR}/${image_packages}" \ - "${BUILD_DIR}/${image_licenses}" \ - "${compressed_images[@]}" \ - "${extra_files[@]}" - - # Upload legacy digests - upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images + compress_disk_images files_to_evaluate } diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 3e84a36ae0..1206af5457 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -22,8 +22,6 @@ SCRIPTS_DIR = os.environ['SCRIPTS_DIR'] # GLOBALS STATIC_FILES = ['%s/version.txt' % REPO_MANIFESTS_DIR, '%s/common.sh' % SCRIPTS_DIR, - '%s/core_pre_alpha' % SCRIPTS_DIR, - '%s/core_roller_upload' % SCRIPTS_DIR, '%s/core_sign_update' % SCRIPTS_DIR, ] diff --git a/build_library/modify_image_util.sh b/build_library/modify_image_util.sh index 53fa0cc81e..9099b02e10 100755 --- a/build_library/modify_image_util.sh +++ b/build_library/modify_image_util.sh @@ -85,24 +85,8 @@ finish_modify_image() { declare -a files_to_evaluate - declare -a compressed_images - declare -a extra_files - files_to_evaluate+=( "${DST_IMAGE}" ) - compress_disk_images files_to_evaluate compressed_images extra_files - - upload_image -d "${DST_IMAGE}.DIGESTS" \ - "${compressed_images[@]}" \ - "${extra_files[@]}" - - # Upload legacy digests - upload_legacy_digests "${DST_IMAGE}.DIGESTS" compressed_images - - for filename in "${EXTRA_FILES[@]}"; do - if [[ -e "${BUILD_DIR}/${filename}" ]]; then - upload_image "${BUILD_DIR}/${filename}" - fi - done + compress_disk_images files_to_evaluate set_build_symlinks "${FLAGS_group}-latest" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 49bc6b0c91..561766aa21 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -131,7 +131,7 @@ create_prod_image() { sudo rsync -a --delete "${BUILD_DIR}/configroot/etc/portage" "${BUILD_DIR}/root_fs_dir2/etc" sudo mksquashfs "${BUILD_DIR}/root_fs_dir2" "${BUILD_DIR}/${image_sysext_base}" -noappend -xattrs-exclude '^btrfs.' sudo rm -rf "${BUILD_DIR}/root_fs_dir2" - + # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env sudo rm -rf ${root_fs_dir}/etc/env.d @@ -180,56 +180,8 @@ EOF "${image_initrd_contents_wtd}" \ "${image_disk_usage}" - # Upload - local to_upload=( - "${BUILD_DIR}/${image_contents}" - "${BUILD_DIR}/${image_contents_wtd}" - "${BUILD_DIR}/${image_packages}" - "${BUILD_DIR}/${image_sbom}" - "${BUILD_DIR}/${image_licenses}" - "${BUILD_DIR}/${image_kernel}" - "${BUILD_DIR}/${image_pcr_policy}" - "${BUILD_DIR}/${image_grub}" - "${BUILD_DIR}/${image_kconfig}" - "${BUILD_DIR}/${image_initrd_contents}" - "${BUILD_DIR}/${image_initrd_contents_wtd}" - "${BUILD_DIR}/${image_disk_usage}" - "${BUILD_DIR}/${image_sysext_base}" - ) - - # append sysext inventories to uploads - if [[ -n "${base_sysexts}" ]] ; then - local inventory_file="" image_basename="${image_name%.bin}" - - for inventory_file in "${image_contents}" "${image_contents_wtd}" "${image_disk_usage}" "${image_packages}" ; do - local suffix="${inventory_file/${image_basename}/}" sysext="" - - for sysext in ${base_sysexts//,/ }; do - local name="${sysext%:*}" - local sysext_inventory="${root_fs_sysexts_output_dir}/${name}${suffix}" - if [[ ! -f "${sysext_inventory}" ]] ; then - die "Sysext inventory file '${name}${suffix}' for '${inventory_file}' not found in '${root_fs_sysexts_output_dir}'" - fi - to_upload+=( "${sysext_inventory}" ) - done - done - fi - local files_to_evaluate=( "${BUILD_DIR}/${image_name}" ) - declare -a compressed_images - declare -a extra_files - compress_disk_images files_to_evaluate compressed_images extra_files - to_upload+=( "${compressed_images[@]}" ) - to_upload+=( "${extra_files[@]}" ) - - # FIXME(bgilbert): no shim on arm64 - if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then - to_upload+=("${BUILD_DIR}/${image_shim}") - fi - upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" "${to_upload[@]}" - - # Upload legacy digests - upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images + compress_disk_images files_to_evaluate } create_prod_tar() { @@ -246,13 +198,11 @@ create_prod_tar() { sudo umount "/mnt/${lodevbase}p9" sudo rmdir "/mnt/${lodevbase}p9" sudo losetup --detach "${lodev}" - upload_image "${container}" } create_prod_sysexts() { local image_name="$1" local image_sysext_base="${image_name%.bin}_sysext.squashfs" - local to_upload=() for sysext in "${EXTRA_SYSEXTS[@]}"; do local name="flatcar-${sysext%:*}" local pkgs="${sysext#*:}" @@ -273,15 +223,5 @@ create_prod_sysexts() { -private_key "/usr/share/update_engine/update-payload-key.key.pem" \ -new_image "${BUILD_DIR}/${name}.raw" \ -out_file "${BUILD_DIR}/flatcar_test_update-${name}.gz" - to_upload+=( - "${BUILD_DIR}/${name}.raw" - "${BUILD_DIR}/${name}_contents.txt" - "${BUILD_DIR}/${name}_contents_wtd.txt" - "${BUILD_DIR}/${name}_disk_usage.txt" - "${BUILD_DIR}/${name}_packages.txt" - "${BUILD_DIR}/flatcar_test_update-${name}.gz" - ) done - upload_image -d ${BUILD_DIR}/sysexts.DIGESTS "${to_upload[@]}" } - diff --git a/build_library/release_util.sh b/build_library/release_util.sh index faa84425f3..ebe1163606 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -2,36 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -GSUTIL_OPTS= -UPLOAD_ROOT= -UPLOAD_PATH= -UPLOAD_DEFAULT=${FLAGS_FALSE} DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2" -# Default upload root can be overridden from the environment. -_user="${USER}" -[[ ${USER} == "root" ]] && _user="${SUDO_USER}" -: ${FLATCAR_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}} -unset _user - -DEFINE_boolean parallel ${FLAGS_TRUE} \ - "Enable parallelism in gsutil." -DEFINE_boolean upload ${UPLOAD_DEFAULT} \ - "Upload all packages/images via gsutil." -DEFINE_boolean private ${FLAGS_TRUE} \ - "Upload the image as a private object." -DEFINE_string upload_root "${FLATCAR_UPLOAD_ROOT}" \ - "Upload prefix, board/version/etc will be appended. Must be a gs:// URL." -DEFINE_string upload_path "" \ - "Full upload path, overrides --upload_root. Must be a full gs:// URL." -DEFINE_string download_root "" \ - "HTTP download prefix, board/version/etc will be appended." -DEFINE_string download_path "" \ - "HTTP download path, overrides --download_root." -DEFINE_string sign "" \ - "Sign all files to be uploaded with the given GPG key." -DEFINE_string sign_digests "" \ - "Sign image DIGESTS files with the given GPG key." DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zst" DEFINE_boolean only_store_compressed ${FLAGS_TRUE} \ @@ -89,14 +61,6 @@ compress_disk_images() { # among them. local -n local_files_to_evaluate="$1" - # An array that will hold the path on disk to the resulting disk image archives. - # Multiple compression formats may be requested, so this array may hold - # multiple archives for the same image. - local -n local_resulting_archives="$2" - - # Files that did not match the filter for disk images. - local -n local_extra_files="$3" - info "Compressing ${#local_files_to_evaluate[@]} images" # We want to compress images, but we also want to remove the uncompressed files # from the list of uploadable files. @@ -118,7 +82,6 @@ compress_disk_images() { if [ -z "${processed_format[${format}]}" ]; then info "Compressing ${filename##*/} to ${format}" COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") - local_resulting_archives+=( "$COMPRESSED_FILENAME" ) processed_format["${format}"]=1 fi done @@ -133,244 +96,6 @@ compress_disk_images() { else info "Keeping ${filename}" fi - else - local_extra_files+=( "${filename}" ) fi done } - -upload_legacy_digests() { - [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 - - local local_digest_file="$1" - local -n local_compressed_files="$2" - - [[ "${#local_compressed_files[@]}" -gt 0 ]] || return 0 - - # Upload legacy digests - declare -a digests_to_upload - for file in "${local_compressed_files[@]}";do - legacy_digest_file="${file}.DIGESTS" - cp "${local_digest_file}" "${legacy_digest_file}" - digests_to_upload+=( "${legacy_digest_file}" ) - done - local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" - upload_files "digests" "${def_upload_path}" "" "${digests_to_upload[@]}" -} - -check_gsutil_opts() { - [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 - - if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then - GSUTIL_OPTS="-m" - fi - - if [[ -n "${FLAGS_upload_root}" ]]; then - if [[ "${FLAGS_upload_root}" != gs://* ]] \ - && [[ "${FLAGS_upload_root}" != rsync://* ]] ; then - die_notrace "--upload_root must be a gs:// or rsync:// URL" - fi - # Make sure the path doesn't end with a slash - UPLOAD_ROOT="${FLAGS_upload_root%%/}" - fi - - if [[ -n "${FLAGS_upload_path}" ]]; then - if [[ "${FLAGS_upload_path}" != gs://* ]] \ - && [[ "${FLAGS_upload_path}" != rsync://* ]] ; then - die_notrace "--upload_path must be a gs:// or rsync:// URL" - fi - # Make sure the path doesn't end with a slash - UPLOAD_PATH="${FLAGS_upload_path%%/}" - fi - - # Ensure scripts run via sudo can use the user's gsutil/boto configuration. - if [[ -n "${SUDO_USER}" ]]; then - : ${BOTO_PATH:="$HOME/.boto:/home/$SUDO_USER/.boto"} - export BOTO_PATH - fi -} - -# Generic upload function -# Usage: upload_files "file type" "${UPLOAD_ROOT}/default/path" "" files... -# arg1: file type reported via log -# arg2: default upload path, overridden by --upload_path -# arg3: upload path suffix that can't be overridden, must end in / -# argv: remaining args are files or directories to upload -upload_files() { - [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 - - local msg="$1" - local local_upload_path="$2" - local extra_upload_suffix="$3" - shift 3 - - if [[ -n "${UPLOAD_PATH}" ]]; then - local_upload_path="${UPLOAD_PATH}" - fi - - if [[ -n "${extra_upload_suffix}" && "${extra_upload_suffix}" != */ ]] - then - die "upload suffix '${extra_upload_suffix}' doesn't end in /" - fi - - info "Uploading ${msg} to ${local_upload_path}" - - if [[ "${local_upload_path}" = 'rsync://'* ]]; then - local rsync_upload_path="${local_upload_path#rsync://}" - local sshcmd="ssh -o BatchMode=yes " - sshcmd="$sshcmd -o StrictHostKeyChecking=no" - sshcmd="$sshcmd -o UserKnownHostsFile=/dev/null" - sshcmd="$sshcmd -o NumberOfPasswordPrompts=0" - - # ensure the target path exists - local sshuserhost="${rsync_upload_path%:*}" - local destpath="${rsync_upload_path#*:}" - ${sshcmd} "${sshuserhost}" \ - "mkdir -p ${destpath}/${extra_upload_suffix}" - - # now sync - rsync -Pav -e "${sshcmd}" "$@" \ - "${rsync_upload_path}/${extra_upload_suffix}" - else - gsutil ${GSUTIL_OPTS} cp -R "$@" \ - "${local_upload_path}/${extra_upload_suffix}" - fi -} - - -# Identical to upload_files but GPG signs every file if enabled. -# Usage: sign_and_upload_files "file type" "${UPLOAD_ROOT}/default/path" "" files... -# arg1: file type reported via log -# arg2: default upload path, overridden by --upload_path -# arg3: upload path suffix that can't be overridden, must end in / -# argv: remaining args are files or directories to upload -sign_and_upload_files() { - [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 - - local msg="$1" - local path="$2" - local suffix="$3" - shift 3 - - # run a subshell to possibly clean the temporary directory with - # signatures without clobbering the global EXIT trap - ( - # Create simple GPG detached signature for all uploads. - local sigs=() - if [[ -n "${FLAGS_sign}" ]]; then - local file - local sigfile - local sigdir=$(mktemp --directory) - trap "rm -rf ${sigdir}" EXIT - for file in "$@"; do - if [[ "${file}" =~ \.(asc|gpg|sig)$ ]]; then - continue - fi - - for sigfile in $(find "${file}" ! -type d); do - mkdir -p "${sigdir}/${sigfile%/*}" - gpg --batch --local-user "${FLAGS_sign}" \ - --output "${sigdir}/${sigfile}.sig" \ - --detach-sign "${sigfile}" || die "gpg failed" - done - - [ -d "${file}" ] && - sigs+=( "${sigdir}/${file}" ) || - sigs+=( "${sigdir}/${file}.sig" ) - done - fi - - upload_files "${msg}" "${path}" "${suffix}" "$@" "${sigs[@]}" - ) -} - -upload_packages() { - [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 - [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" - - local board_packages="${1:-"${BOARD_ROOT}/packages"}" - local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" - sign_and_upload_files packages ${def_upload_path} "pkgs/" \ - "${board_packages}"/* -} - -# Upload a set of files (usually images) and digest, optionally w/ gpg sig -# If more than one file is specified -d must be the first argument -# Usage: upload_image [-d file.DIGESTS] file1 [file2...] -upload_image() { - [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 - [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" - - # The name to use for .DIGESTS and .DIGESTS.asc must be explicit if - # there is more than one file to upload to avoid potential confusion. - local digests - if [[ "$1" == "-d" ]]; then - [[ -n "$2" ]] || die "-d requires an argument" - digests="$2" - shift 2 - else - [[ $# -eq 1 ]] || die "-d is required for multi-file uploads" - # digests is assigned after image is possibly compressed/renamed - fi - - local uploads=() - local filename - for filename in "$@"; do - if [[ ! -f "${filename}" ]]; then - die "File '${filename}' does not exist!" - fi - uploads+=( "${filename}" ) - done - - if [[ -z "${digests}" ]]; then - digests="${uploads[0]}.DIGESTS" - fi - - # For consistency generate a .DIGESTS file similar to the one catalyst - # produces for the SDK tarballs and up upload it too. - make_digests -d "${digests}" "${uploads[@]}" - uploads+=( "${digests}" ) - - # Create signature as ...DIGESTS.asc as Gentoo does. - if [[ -n "${FLAGS_sign_digests}" ]]; then - rm -f "${digests}.asc" - gpg --batch --local-user "${FLAGS_sign_digests}" \ - --clearsign "${digests}" || die "gpg failed" - uploads+=( "${digests}.asc" ) - fi - - local log_msg=$(basename "$digests" .DIGESTS) - local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" - sign_and_upload_files "${log_msg}" "${def_upload_path}" "" "${uploads[@]}" -} - -# Translate the configured upload URL to a download URL -# Usage: download_image_url "path/suffix" -download_image_url() { - if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then - echo "$1" - return 0 - fi - - local download_root="${FLAGS_download_root:-${UPLOAD_ROOT}}" - - local download_path - local download_channel - if [[ -n "${FLAGS_download_path}" ]]; then - download_path="${FLAGS_download_path%%/}" - elif [[ "${download_root}" == *flatcar-jenkins* ]]; then - download_channel="${download_root##*/}" - download_root="gs://${download_channel}.release.flatcar-linux.net" - # Official release download paths don't include the boards directory - download_path="${download_root%%/}/${BOARD}/${FLATCAR_VERSION}" - else - download_path="${download_root%%/}/boards/${BOARD}/${FLATCAR_VERSION}" - fi - - # Just in case download_root was set from UPLOAD_ROOT - if [[ "${download_path}" == gs://* ]]; then - download_path="https://${download_path#gs://}" - fi - - echo "${download_path}/$1" -} diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 11c7c746cd..cd44a9d36d 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -1224,7 +1224,7 @@ EOF "version": "${FLATCAR_VERSION_ID}", "providers": [{ "name": "${provider}", - "url": "$(download_image_url "$(_dst_name ".box")")", + "url": "https://${BUILDCACHE_SERVER:-bincache.flatcar-linux.net}/images/${BOARD%-usr}/${FLATCAR_VERSION}/$(_dst_name ".box")", "checksum_type": "sha256", "checksum": "$(sha256sum "${box}" | awk '{print $1}')" }] @@ -1271,53 +1271,6 @@ vm_cleanup() { sudo rm -rf "${VM_TMP_DIR}" } -vm_upload() { - - declare -a legacy_uploads - declare -a uploadable_files - declare -a compressed_images - declare -a image_files - declare -a digest_uploads - - compress_disk_images VM_GENERATED_FILES compressed_images uploadable_files - - if [ "${#compressed_images[@]}" -gt 0 ]; then - uploadable_files+=( "${compressed_images[@]}" ) - legacy_uploads+=( "${compressed_images[@]}" ) - fi - - local digests="$(_dst_dir)/$(_dst_name .DIGESTS)" - upload_image -d "${digests}" "${uploadable_files[@]}" - - [[ -e "${digests}" ]] || return 0 - - # Since depending on the ordering of $VM_GENERATED_FILES is brittle only - # use it if $VM_DST_IMG isn't included in the uploaded files. - if [ "${#legacy_uploads[@]}" -eq 0 ];then - legacy_uploads+=( "${VM_GENERATED_FILES[0]}" ) - fi - - for legacy_upload in "${legacy_uploads[@]}";do - local legacy_digest_file="${legacy_upload}.DIGESTS" - [[ "${legacy_digest_file}" == "${digests}" ]] && continue - - cp "${digests}" "${legacy_digest_file}" - digest_uploads+=( "${legacy_digest_file}" ) - - if [[ -e "${digests}.asc" ]]; then - digest_uploads+=( "${legacy_digest_file}.asc" ) - cp "${digests}.asc" "${legacy_digest_file}.asc" - fi - done - - if [ "${#digest_uploads[@]}" -gt 0 ];then - legacy_uploads+=( "${digest_uploads[@]}" ) - fi - - local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" - upload_files "$(_dst_name)" "${def_upload_path}" "" "${legacy_uploads[@]}" -} - print_readme() { local filename info "Files written to $(relpath "$(dirname "${VM_DST_IMG}")")" diff --git a/build_packages b/build_packages index 4ca38aec2f..77c7742b7b 100755 --- a/build_packages +++ b/build_packages @@ -86,8 +86,6 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then FLAGS_workon="${FLAGS_FALSE}" fi -check_gsutil_opts - # Before we can run any tools, we need to update chroot or setup_board. UPDATE_ARGS=( --regen_configs ) if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then @@ -333,8 +331,5 @@ eclean-$BOARD -d packages info "Checking build root" test_image_content "${BOARD_ROOT}" -# upload packages if enabled -upload_packages - info "Builds complete" command_completed diff --git a/build_toolchains b/build_toolchains index 910e797b33..8deb452410 100755 --- a/build_toolchains +++ b/build_toolchains @@ -45,7 +45,6 @@ create_provenance_overlay() { } catalyst_init "$@" -check_gsutil_opts ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay" @@ -55,13 +54,4 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" create_provenance_overlay "${ROOT_OVERLAY}" catalyst_build - -# TODO: Actually just TOOLCHAIN_PKGS and the exact dependencies should be uploaded -for board in $(get_board_list); do - board_packages="${BINPKGS}/target/${board}" - def_upload_path="${UPLOAD_ROOT}/boards/${board}/${FLAGS_version}" - sign_and_upload_files "board toolchain packages" "${def_upload_path}" \ - "toolchain/" "${board_packages}"/* -done - command_completed diff --git a/core_date b/core_date deleted file mode 100755 index bddf452f73..0000000000 --- a/core_date +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2016 The CoreOS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -SCRIPT_ROOT=$(dirname $(readlink -f "$0")) -. "${SCRIPT_ROOT}/common.sh" || exit 1 - -USAGE="USAGE: $0 [-v] [date flags] - -This script calculates the date given the CoreOS major version or calculates -the version given a date. - -Examples: - - $0 -v 1000 - Sun Mar 27 00:00:00 UTC 2016 - - $0 -v 1000 --iso-8601 - 2016-03-27 - - $0 --date 'Jun 1, 2016' - 1066 -" - -case "$1" in - "-h") - echo "$USAGE" - ;; - - "-v") - shift - if [[ $# -ge 1 ]] && [[ "$1" != [-+]* ]]; then - v="$1" - shift - else - v="${FLATCAR_VERSION}" - fi - - # strip of a v prefix or .0.0 suffix - v="${v#v}" - v="${v%%.*}" - - export TZ=${TZ:=UTC} - date -d @$(( (v * 86400) + COREOS_EPOCH )) "$@" - ;; - - *) - t=$(date +%s "$@") - echo $(( (t - COREOS_EPOCH) / 86400 )) - ;; -esac diff --git a/core_dev_sign_update b/core_dev_sign_update deleted file mode 100755 index 6ed848ee34..0000000000 --- a/core_dev_sign_update +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_ROOT=$(dirname $(readlink -f "$0")) -. "${SCRIPT_ROOT}/common.sh" || exit 1 - -assert_inside_chroot - -DEFINE_string data_dir "" "Directory containing downloaded release artifacts" -DEFINE_string board "" "Board to sign artifacts for" -DEFINE_string version "" "Version to sign artifacts for" -DEFINE_integer n_signatures "2" "Number of signatures this release will be signed with" -DEFINE_string output_dir "" "Output directory" -DEFINE_string gpg_key "" "Value for '--default-key' argument to gpg --sign" - -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" - -set -e - -data_dir="${FLAGS_data_dir}/${FLAGS_board}/${FLAGS_version}" -output_dir="${FLAGS_output_dir}/${FLAGS_board}/${FLAGS_version}" -mkdir -p "$output_dir" - -cleanup() { - # core_sign_update expects to unpack this too, so we'll clean it up. - rm -f "${data_dir}/coreos_production_update.bin" - - rm -f "${data_dir}/update" - rm -f "${data_dir}/update.hash" -} - -trap cleanup INT TERM EXIT - -# delta_generator expects a list of colon-separated sizes for signature hash algorithms in order to -# build the update payload protobuf properly. Since we already assume sha256 elsewhere in -# core_sign_update, do it here as well. -signature_sizes="" -for i in $(seq 1 $FLAGS_n_signatures); do - signature_sizes="${signature_sizes}:256" -done -signature_sizes="${signature_sizes:1:${#signature_sizes}}" - -echo "=== Verifying update payload... ===" -gpg2 --verify "${data_dir}/coreos_production_update.bin.bz2.sig" -gpg2 --verify "${data_dir}/coreos_production_image.vmlinuz.sig" -gpg2 --verify "${data_dir}/coreos_production_update.zip.sig" -echo "=== Decompressing update payload... ===" -bunzip2 --keep "${data_dir}/coreos_production_update.bin.bz2" - -echo "=== Creating signable update payload... ===" -delta_generator \ - -new_image "${data_dir}/coreos_production_update.bin" \ - -new_kernel "${data_dir}/coreos_production_image.vmlinuz" \ - -out_file "${data_dir}/update" -delta_generator \ - --signature_size ${signature_sizes} \ - --in_file "${data_dir}/update" \ - --out_hash_file "${data_dir}/update.hash" - -echo "=== Signing update payload... ===" -if [[ -z "${FLAGS_gpg_key}" ]]; then - gpg2 \ - --output "${output_dir}/update.sig.$(whoami)" \ - --armor --detach-sign "${data_dir}/update.hash" -else - gpg2 \ - --local-user "$FLAGS_gpg_key" \ - --output "${output_dir}/update.sig.$(whoami)" \ - --armor --detach-sign "${data_dir}/update.hash" -fi -echo "=== Update payload signed successfully. ===" diff --git a/core_pre_alpha b/core_pre_alpha deleted file mode 100755 index c11bd7954f..0000000000 --- a/core_pre_alpha +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2014 The CoreOS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -SCRIPT_ROOT=$(dirname $(readlink -f "$0")) -# We have to simple-mindedly set GCLIENT_ROOT in case we're running from -# au-generator.zip because common.sh will fail while auto-detect it. -export GCLIENT_ROOT=$(readlink -f "${SCRIPT_ROOT}/../../") -. "${SCRIPT_ROOT}/common.sh" || exit 1 - -DEFINE_string board "amd64-usr" \ - "Board type of the image" -DEFINE_string version "${FLATCAR_VERSION}" \ - "Version number to promote." - -DEFINE_string build_storage "gs://builds.release.core-os.net" \ - "GS bucket with official build artifacts." -DEFINE_string release_storage "gs://alpha.release.core-os.net" \ - "GS bucket for release downloads." -DEFINE_string legacy_storage "gs://storage.core-os.net/coreos" \ - "Legacy 'storage' GS bucket." - -# Parse flags -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" -switch_to_strict_mode - -# Ensure GS URL doesn't have a trailing / -FLAGS_build_storage="${FLAGS_build_storage%%/}" -FLAGS_release_storage="${FLAGS_release_storage%%/}" -FLAGS_legacy_storage="${FLAGS_legacy_storage%%/}" - -# Full GS URLs -gs_build="${FLAGS_build_storage}/alpha/boards/${FLAGS_board}/${FLAGS_version}" -gs_release="${FLAGS_release_storage}/${FLAGS_board}/${FLAGS_version}" -gs_legacy="${FLAGS_legacy_storage}/${FLAGS_board}/${FLAGS_version}" - -gsutil -m cp "${gs_build}/*" "${gs_release}/" -gsutil -m cp "${gs_release}/*" "${gs_legacy}/" diff --git a/core_roller_upload b/core_roller_upload deleted file mode 100755 index a20ef5e845..0000000000 --- a/core_roller_upload +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2013 The CoreOS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -SCRIPT_ROOT=$(dirname $(readlink -f "$0")) -# We have to simple-mindedly set GCLIENT_ROOT in case we're running from -# au-generator.zip because common.sh will fail while auto-detect it. -export GCLIENT_ROOT=$(readlink -f "${SCRIPT_ROOT}/../../") -. "${SCRIPT_ROOT}/common.sh" || exit 1 - -DEFINE_string board "amd64-usr" \ - "Board type of the image" -DEFINE_string payload "coreos_production_update.gz" \ - "Path to the update payload" -DEFINE_string version "${FLATCAR_VERSION}" \ - "Version number of this build." -DEFINE_string app_id "e96281a6-d1af-4bde-9a0a-97b76e56dc57" \ - "CoreOS AppId in roller." - -DEFINE_string user "" \ - "User for roller." -DEFINE_string api_key "" \ - "API key for roller." -DEFINE_string endpoint "https://public.update.core-os.net" \ - "Roller endpoint to update." -DEFINE_string storage "gs://coreos-update" \ - "Google Storage location to host the payload." -DEFINE_string download "https://update.release.core-os.net" \ - "HTTPS location users will download payload from." - -FLAGS_HELPS="usage: $SCRIPTNAME [flags] - -Setting everything up for use\n - -1) Run 'gsutil config' and use project id coreos-net-storage\n -2) Ensure updateservicectl is installed in your path\n -" - -# Parse flags -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" -switch_to_strict_mode - -[[ -z "${FLAGS_api_key}" ]] && die "--api_key is required" -[[ -z "${FLAGS_user}" ]] && die "--user is required" - -# Ensure GS URL doesn't have a trailing / -FLAGS_storage="${FLAGS_storage%%/}" - -GS_URL="${FLAGS_storage}/${FLAGS_board}/${FLAGS_version}/update.gz" -HTTP_URL="${FLAGS_download}/${FLAGS_board}/${FLAGS_version}/update.gz" -gsutil cp "${FLAGS_payload}" "${GS_URL}" - -updateservicectl \ - --server="${FLAGS_endpoint}" \ - --user="${FLAGS_user}" \ - --key="${FLAGS_api_key}" \ - package create \ - --app-id="${FLAGS_app_id}" \ - --file="${FLAGS_payload}" \ - --url="${HTTP_URL}" \ - --version="${FLAGS_version}" diff --git a/image_inject_bootchain b/image_inject_bootchain index 721f86da1d..45f115aba3 100755 --- a/image_inject_bootchain +++ b/image_inject_bootchain @@ -50,8 +50,6 @@ eval set -- "${FLAGS_ARGV}" # Die on any errors. switch_to_strict_mode -check_gsutil_opts - if [[ -z "${FLAGS_kernel_path}" && -z "${FLAGS_efi_grub_path}" && -z "${FLAGS_shim_path}" ]]; then die_notrace "Specify at least one of --kernel_path, --efi_grub_path, --shim_path" diff --git a/image_set_group b/image_set_group index 77a6fd1653..a034167b11 100755 --- a/image_set_group +++ b/image_set_group @@ -42,8 +42,6 @@ eval set -- "${FLAGS_ARGV}" # Die on any errors. switch_to_strict_mode -check_gsutil_opts - if [[ -z "${FLAGS_board}" ]] ; then die_notrace "--board is required." fi diff --git a/image_to_vm.sh b/image_to_vm.sh index d496fe8ecb..db205f76bf 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -55,8 +55,6 @@ eval set -- "${FLAGS_ARGV}" # Die on any errors. switch_to_strict_mode -check_gsutil_opts - if [[ -z "${FLAGS_format}" ]]; then FLAGS_format="$(get_default_vm_type ${FLAGS_board})" fi @@ -128,8 +126,8 @@ write_vm_bundle vm_cleanup trap - EXIT -# Optionally upload all of our hard work -vm_upload +declare -a compressed_images uploadable_files +compress_disk_images VM_GENERATED_FILES # Ready to set sail! okboat diff --git a/rebuild_packages b/rebuild_packages index 424431676b..bdd128f436 100755 --- a/rebuild_packages +++ b/rebuild_packages @@ -29,8 +29,6 @@ if [[ -z "${FLAGS_board}" ]]; then exit 1 fi -check_gsutil_opts - # set BOARD and BOARD_ROOT . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 @@ -78,9 +76,6 @@ else info "Checking build root" test_image_content "${BOARD_ROOT}" - - # upload packages if enabled - upload_packages fi command_completed diff --git a/signing/devel.key.pem b/signing/devel.key.pem deleted file mode 100644 index ac660a755e..0000000000 --- a/signing/devel.key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAzFS5uVJ+pgibcFLD3kbYk02Edj0HXq31ZT/Bva1sLp3Ysv+Q -Tv/ezjf0gGFfASdgpz6G+zTipS9AIrQr0yFR+tdp1ZsHLGxVwvUoXFftdapqlyj8 -uQcWjjbN7qJsZu0Ett/qo93hQ5nHW7Sv5dRm/ZsDFqk2Uvyaoef4bF9r03wYpZq7 -K3oALZ2smETv+A5600mj1Xg5M52QFU67UHlsEFkZphrGjiqiCdp9AAbAvE7a5rFc -Jf86YR73QX08K8BX7OMzkn3DsqdnWvLB3l3W6kvIuP+75SrMNeYAcU8PI1+bzLcA -G3VN3jA78zeKALgynUNH50mxuiiU3DO4DZ+p5QIDAQABAoIBAH7ENbE+9+nkPyMx -hekaBPVmSz7b3/2iaTNWmckmlY5aSX3LxejtH3rLBjq7rihWGMXJqg6hodcfeGfP -Zb0H2AeKq1Nlac7qq05XsKGRv3WXs6dyO1BDkH/Minh5dk1o0NrwEm91kXLSLfe8 -IsCwxPCjwgfGFTjpFLpL4zjA/nFmWRyk2eyvs5VYRGKbbC83alUy7LutyRdZfw1b -nwXldw2m8k/HPbGhaAqPpXTOjckIXZS5Dcp3smrOzwObZ6c3gQzg8upaRmxJVOmk -cgCFTe0yUB2GMTEE3SUmuWJyZqECoyQtuiu0yT3igH8MZQpjg9NXm0eho/bXjN36 -frH+ikUCgYEA7VdCRcisnYWct29j+Bnaio9yXwwxhfoee53a4LQgjw5RLGUe1mXe -j56oZ1Mak3Hh55sVQLNXZBuXHQqPsr7KkWXJXedDNFfq1u6by4LeJV0YYiDjjaCM -T5G4Tcs7xhBWszLMCjhpJCrwHdGk3aa65UQ+angZlxhyziULCjpb5rMCgYEA3GUb -VkqlVuNkHoogOMwg+h1jUSkwtWvP/z/FOXrKjivuwSgQ+i6PsildI3FL/WQtJxgd -arB+l0L8TZJ6spFdNXwGmdCLqEcgEBYl11EojOXYLa7oLONI41iRQ3/nBBIqC38P -Cs6CZQG/ZpKSoOzXE34BwcrOL99MA2oaVpGHuQcCgYA1IIk3Mbph8FyqOwb3rGHd -Dksdt48GXHyiUy2BixCWtS+6blA+0cLGB0/PAS07wAw/WdmiCAMR55Ml7w1Hh6m0 -bkJrAK9schmhTvwUzBCJ8JLatF37f+qojQfichHJPjMKHd7KkuIGNI5XPmxXKVFA -rMwD7SpdRh28w1H7UiDsPQKBgGebnFtXohyTr2hv9K/evo32LM9ltsFC2rga6YOZ -BwoI+yeQx1JleyX9LgzQYTHQ2y0quAGE0S4YznVFLCswDQpssMm0cUL9lMQbNVTg -kViTYKoxNHKNsqE17Kw3v4l5ZIydAZxJ8qC7TphQxV+jl4RRU1AgIAf/SEO+qH0T -0yMXAoGBAN+y9QpGnGX6cgwLQQ7IC6MC+3NRed21s+KxHzpyF+Zh/q6NTLUSgp8H -dBmeF4wAZTY+g/fdB9drYeaSdRs3SZsM7gMEvjspjYgE2rV/5gkncFyGKRAiNOR4 -bsy1Gm/UYLTc8+S3fq/xjg9RCjW9JMwavAwL6oVNNt7nyAXPfvSu ------END RSA PRIVATE KEY----- diff --git a/signing/devel.pub.pem b/signing/devel.pub.pem deleted file mode 100644 index ccee9ee90b..0000000000 --- a/signing/devel.pub.pem +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFS5uVJ+pgibcFLD3kbY -k02Edj0HXq31ZT/Bva1sLp3Ysv+QTv/ezjf0gGFfASdgpz6G+zTipS9AIrQr0yFR -+tdp1ZsHLGxVwvUoXFftdapqlyj8uQcWjjbN7qJsZu0Ett/qo93hQ5nHW7Sv5dRm -/ZsDFqk2Uvyaoef4bF9r03wYpZq7K3oALZ2smETv+A5600mj1Xg5M52QFU67UHls -EFkZphrGjiqiCdp9AAbAvE7a5rFcJf86YR73QX08K8BX7OMzkn3DsqdnWvLB3l3W -6kvIuP+75SrMNeYAcU8PI1+bzLcAG3VN3jA78zeKALgynUNH50mxuiiU3DO4DZ+p -5QIDAQAB ------END PUBLIC KEY----- diff --git a/signing/new_key.sh b/signing/new_key.sh deleted file mode 100755 index 51ba500b42..0000000000 --- a/signing/new_key.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -ex -KEY="$1" -openssl genrsa -rand /dev/random -out "${KEY}.key.pem" 2048 -openssl rsa -in "${KEY}.key.pem" -pubout -out "${KEY}.pub.pem" diff --git a/signing/print_key.sh b/signing/print_key.sh deleted file mode 100755 index 40e6c5ea7f..0000000000 --- a/signing/print_key.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# If there is no default printer use ./print_key.sh prod-2 -d printer_name -# List available printers with lpstat -a - -set -ex -KEY="$1" -shift -qrencode -8 -o - < "${KEY}.key.pem" | lp -E -o fit-to-page "$@" diff --git a/signing/prod-2.pub.pem b/signing/prod-2.pub.pem deleted file mode 100644 index 10fcaff58b..0000000000 --- a/signing/prod-2.pub.pem +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7pO21yN+b6yx9P+wHzS2 -clxGs18eWxfoleETLbFVmcXu783rgBP0bFjYfqrNZNaI1Ip6bxEYkPacg0xFg7ri -lNdO/cxJV5Ltj40wFlpmzJOAH8hx5SF8KWg2NV1I6TS8pp+CQqcvvOKu6AIcWfeY -11V7eJ8rWcDsnqpTg8T1VRxytsg2UjTMfQwzcGLTb8cQ8AV39ED5WC5NdS9Bld4h -XqS9Dx6Pe3JOQLZze6XIIwWuB2jxGpM1GWfRNm5nxvne3l7ggC970482a7STGK10 -fD8//k8myVxleMAeQoMRXoRq9p3C84H4Bw8v2dX13kFFCgfEQj6SOZ5huXZKLPpB -LwIDAQAB ------END PUBLIC KEY----- diff --git a/signing/transfer.sh b/signing/transfer.sh deleted file mode 100755 index c5a788a0ed..0000000000 --- a/signing/transfer.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -APPID=e96281a6-d1af-4bde-9a0a-97b76e56dc57 - -declare -A RELEASE_CHANNEL -RELEASE_CHANNEL[alpha]=Alpha -RELEASE_CHANNEL[beta]=Beta -RELEASE_CHANNEL[stable]=Stable - -download() { - local channel="$1" - local version="$2" - local board="$3" - - local gs="gs://builds.release.core-os.net/${channel}/boards/${board}/${version}" - local dir="${BASEDIR}/${board}/${version}" - mkdir -p "${dir}" - pushd "${dir}" >/dev/null - - gsutil -m cp \ - "${gs}/flatcar_production_image.vmlinuz" \ - "${gs}/flatcar_production_image.vmlinuz.sig" \ - "${gs}/flatcar_production_update.bin.bz2" \ - "${gs}/flatcar_production_update.bin.bz2.sig" \ - "${gs}/flatcar_production_update.zip" \ - "${gs}/flatcar_production_update.zip.sig" ./ - - # torcx manifest: try embargoed release bucket first - local torcx_base="gs://builds.release.core-os.net/embargoed/devfiles/torcx/manifests/${board}/${version}" - if ! gsutil -q stat "${torcx_base}/torcx_manifest.json"; then - # Non-embargoed release - local torcx_base="gs://builds.developer.core-os.net/torcx/manifests/${BOARD}/${version}" - fi - gsutil -m cp \ - "${torcx_base}/torcx_manifest.json" \ - "${torcx_base}/torcx_manifest.json.sig" \ - ./ - - gpg2 --verify "flatcar_production_image.vmlinuz.sig" - gpg2 --verify "flatcar_production_update.bin.bz2.sig" - gpg2 --verify "flatcar_production_update.zip.sig" - gpg2 --verify "torcx_manifest.json.sig" - - popd >/dev/null -} - -devsign() { - local channel="$1" - local version="$2" - local board="$3" - - "$(dirname $0)/../core_dev_sign_update" \ - --data_dir "${BASEDIR}" \ - --version "${version}" \ - --output_dir "${SIGDIR}" - - git -C "${SIGDIR}" add . - git -C "${SIGDIR}" commit -m "Add sigs from ${USER} for ${channel} ${version}" -} - -upload() { - local channel="$1" - local version="$2" - local board="$3" - - local dir="${BASEDIR}/${board}/${version}" - local payload="${dir}/flatcar_production_update.gz" - local torcx_manifest="${dir}/torcx_manifest.json" - local torcx_manifest_sig="${dir}/torcx_manifest.json.asc" - local path - for path in "${payload}" "${torcx_manifest}" "${torcx_manifest_sig}"; do - if [[ ! -e "${path}" ]]; then - echo "No such file: ${path}" >&2 - exit 1 - fi - done - - "$(dirname $0)/../core_roller_upload" \ - --user="${ROLLER_USERNAME}" \ - --api_key="${ROLLER_API_KEY}" \ - --app_id="${APPID}" \ - --board="${board}" \ - --version="${version}" \ - --payload="${payload}" - - # Upload torcx manifests - gsutil cp \ - "${torcx_manifest}" \ - "${torcx_manifest_sig}" \ - "gs://coreos-tectonic-torcx/manifests/${board}/${version}/" - - # Update version in a canary channel if one is defined. - local -n canary_channel="ROLLER_CANARY_CHANNEL_${channel^^}" - if [[ -n "${canary_channel}" ]]; then - updateservicectl \ - --server="https://public.update.core-os.net" \ - --user="${ROLLER_USERNAME}" \ - --key="${ROLLER_API_KEY}" \ - channel update \ - --app-id="${APPID}" \ - --channel="${canary_channel}" \ - --version="${version}" - fi -} - -ready() { - local channel="$1" - local version="$2" - local board="$3" - - # setting the percent will deactivate (not delete) any existing rollouts for - # this specific group. - echo "Rollout set to 0% for ${board}" - updateservicectl \ - --server="https://public.update.core-os.net" \ - --user="${ROLLER_USERNAME}" \ - --key="${ROLLER_API_KEY}" \ - group percent \ - --app-id="${APPID}" \ - --group-id="${channel}" \ - --update-percent=0 - - # FIXME(bgilbert): We set --publish=true because there's no way to - # say --publish=unchanged - updateservicectl \ - --server="https://public.update.core-os.net" \ - --user="${ROLLER_USERNAME}" \ - --key="${ROLLER_API_KEY}" \ - channel update \ - --app-id="${APPID}" \ - --channel="${RELEASE_CHANNEL[${channel}]}" \ - --publish=true \ - --version="${version}" -} - -roll() { - local channel="$1" - local hours="$2" - local board="$3" - - local seconds=$((${hours} * 3600)) - - # Only ramp rollouts on AMD64; ARM64 is too small - if [[ "$board" = "arm64-usr" ]]; then - echo "Setting rollout for arm64-usr to 100%" - updateservicectl \ - --server="https://public.update.core-os.net" \ - --user="${ROLLER_USERNAME}" \ - --key="${ROLLER_API_KEY}" \ - group percent \ - --app-id="${APPID}" \ - --group-id="${channel}" \ - --update-percent=100 - else - # creating a new rollout deletes any existing rollout for this group and - # automatically activates the new one. - echo "Creating linear rollout for ${board} that will get to 100% in ${hours}h" - updateservicectl \ - --server="https://public.update.core-os.net" \ - --user="${ROLLER_USERNAME}" \ - --key="${ROLLER_API_KEY}" \ - rollout create linear \ - --app-id="${APPID}" \ - --group-id="${channel}" \ - --duration="${seconds}" \ - --frame-size="60" - fi -} - -usage() { - echo "Usage: $0 {download|upload} [{-a|-b|-s} ]..." >&2 - echo "Usage: $0 devsign [{-a|-b|-s} ]..." >&2 - echo "Usage: $0 ready [{-a|-b|-s} ]..." >&2 - echo "Usage: $0 roll [{-a|-b|-s} ]..." >&2 - exit 1 -} - -# Parse subcommand. -CMD="${1:-}" -shift ||: -case "${CMD}" in - download|devsign) - ;; - upload|ready|roll) - if [[ -e "${HOME}/.config/roller.conf" ]]; then - . "${HOME}/.config/roller.conf" - fi - if [[ -z "${ROLLER_USERNAME:-}" || -z "${ROLLER_API_KEY:-}" ]]; then - echo 'Missing $ROLLER_USERNAME or $ROLLER_API_KEY.' >&2 - echo "Consider adding shell assignments to ~/.config/roller.conf." >&2 - exit 1 - fi - ;; - *) - usage - ;; -esac - -# Parse fixed args if necessary. -case "${CMD}" in - download|devsign|upload) - BASEDIR="${1:-}" - shift ||: - if [[ -z "${BASEDIR}" ]]; then - usage - fi - ;; -esac -case "${CMD}" in - devsign) - SIGDIR="${1:-}" - shift ||: - if [[ -z "${SIGDIR}" ]]; then - usage - fi - ;; -esac - -# Sync SIGDIR exactly once. -case "${CMD}" in - devsign) - echo "Updating ${SIGDIR}..." - git -C "${SIGDIR}" pull -r - ;; -esac - -# Walk argument pairs. -while [[ $# > 0 ]]; do - c="$1" - v="${2?Must provide an argument for each channel (see usage)}" - shift 2 - - case "${c}" in - -a) - $CMD "alpha" "${v}" "amd64-usr" - $CMD "alpha" "${v}" "arm64-usr" - ;; - -b) - $CMD "beta" "${v}" "amd64-usr" - $CMD "beta" "${v}" "arm64-usr" - ;; - -s) - $CMD "stable" "${v}" "amd64-usr" - ;; - *) - usage - ;; - esac -done diff --git a/update_distfiles b/update_distfiles index d6d2cb8444..3591aeb3c7 100755 --- a/update_distfiles +++ b/update_distfiles @@ -8,17 +8,8 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes." -DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil." -DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil." - -# FIXME(marineam): We need to add gs support to emirrordist so it -# doesn't have to operate on a local copy of the complete mirror. -DEFINE_boolean download ${FLAGS_FALSE} \ - "Download the current mirror before making updates to it." MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror" -UPLOAD_ROOT="gs://storage.core-os.net/mirror" -SECOND_ROOT="gs://coreos-net-storage/mirror" # Parse flags FLAGS "$@" || exit 1 @@ -34,11 +25,6 @@ if [[ $# -eq 0 ]]; then set -- "${!repos[@]}" fi -GSUTIL_OPTS= -if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then - GSUTIL_OPTS="-m" -fi - EXIT_CODE=0 @@ -87,40 +73,6 @@ update_local_mirror() { EXIT_CODE=1 } -upload_mirror() { - local repo_name="$1" - local local_mirror="${MIRROR_ROOT}/${repos[$repo_name]}" - local remote_mirror="$2/${repos[$repo_name]}" - - info "Uploading public distfiles for $repo_name" - gsutil ${GSUTIL_OPTS} rsync -c \ - "${local_mirror}/distfiles/" "${remote_mirror}/distfiles" - - info "Uploading private metadata for $repo_name" - # uses cp instead of rsync in order to provide acl - gsutil ${GSUTIL_OPTS} cp -a project-private \ - "${local_mirror}/info/*" "${remote_mirror}/info" -} -download_mirror() { - local repo_name="$1" - local local_mirror="${MIRROR_ROOT}/${repos[$repo_name]}" - local remote_mirror="${UPLOAD_ROOT}/${repos[$repo_name]}" - - info "Downloading public distfiles for $repo_name" - mkdir -p "${local_mirror}/"{distfiles,info} - gsutil ${GSUTIL_OPTS} rsync -c -d \ - "${remote_mirror}/distfiles/" "${local_mirror}/distfiles" - - info "Downloading private metadata for $repo_name" - gsutil ${GSUTIL_OPTS} rsync -c -d \ - "${remote_mirror}/info/" "${local_mirror}/info" -} - -if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then - for repo in "$@"; do - download_mirror "$repo" - done -fi for repo in "$@"; do if ! portageq get_repo_path / "$repo" >/dev/null; then @@ -134,12 +86,5 @@ if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then exit fi -if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then - for repo in "$@"; do - upload_mirror "$repo" "$UPLOAD_ROOT" - upload_mirror "$repo" "$SECOND_ROOT" - done -fi - command_completed exit $EXIT_CODE