From 20325a547f3d103929b5b060ea4b046ff0522883 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 1 Mar 2017 10:52:17 -0800 Subject: [PATCH 1/4] build_image_util: allow unchecked emerges too This is useful for emerges that are meant for incomplete rootfs's, such as ACI building emerges. There are cases where the #! check is expected to fail while doing those. --- build_library/build_image_util.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index d363771a14..0ef508805a 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -159,6 +159,26 @@ emerge_to_image() { test_image_content "${root_fs_dir}" } +# emerge_to_image without a rootfs check; you should use emerge_to_image unless +# here's a good reason not to. +emerge_to_image_unchecked() { + local root_fs_dir="$1"; shift + + if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then + set -- --getbinpkg "$@" + fi + + sudo -E ROOT="${root_fs_dir}" \ + PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ + emerge --root-deps=rdeps --usepkgonly --jobs=$FLAGS_jobs -v "$@" + + # Shortcut if this was just baselayout + [[ "$*" == *sys-apps/baselayout ]] && return + + # Make sure profile.env has been generated + sudo -E ROOT="${root_fs_dir}" env-update --no-ldconfig +} + # Switch to the dev or prod sub-profile set_image_profile() { local suffix="$1" From 1186d2875abbbe1caf7e51a2d3f93f1bd39c8d02 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 1 Mar 2017 11:20:50 -0800 Subject: [PATCH 2/4] Initial checkin of docker-aci build script --- build_docker_aci | 91 ++++++++++++++++++++++++++ build_library/ebuild_aci_manifest.in | 14 ++++ build_library/ebuild_aci_util.sh | 96 ++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100755 build_docker_aci create mode 100644 build_library/ebuild_aci_manifest.in create mode 100644 build_library/ebuild_aci_util.sh diff --git a/build_docker_aci b/build_docker_aci new file mode 100755 index 0000000000..8174eb8bbc --- /dev/null +++ b/build_docker_aci @@ -0,0 +1,91 @@ +#!/bin/bash + +# Copyright (c) 2016 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This is a wrapper around the ebuild_aci_util.sh functions to set up the +# necessary environment, similar to the build_image script. + +SCRIPT_ROOT=$(dirname $(readlink -f "$0")) +. "${SCRIPT_ROOT}/common.sh" || exit 1 + +# Script must run inside the chroot +restart_in_chroot_if_needed "$@" + +assert_not_root_user + +# Developer-visible flags. +DEFINE_string board "${DEFAULT_BOARD}" \ + "The board to build an image for." +DEFINE_string build_dir "" \ + "Directory in which to place image result directories (named by version)" +DEFINE_boolean getbinpkg "${FLAGS_FALSE}" \ + "Download binary packages from remote repository." +DEFINE_string getbinpkgver "" \ + "Use binary packages from a specific version." + +FLAGS_HELP="USAGE: build_docker_aci [flags] [docker version]. +This script is used to build a CoreOS docker-skim ACI. + +The version should identify an existent ebuild (i.e. app-emulation/docker-\$version). + +Examples: + +build_docker_aci --board=amd64-usr --build_dir= 1.12.6 +... +" +show_help_if_requested "$@" + +# The following options are advanced options, only available to those willing +# to read the source code. They are not shown in help output, since they are +# not needed for the typical developer workflow. +DEFINE_integer build_attempt 1 \ + "The build attempt for this image build." +DEFINE_string group "docker-aci" \ + "The update group (not used for actual updates here)" +DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ + "Directory in which to place image result directories (named by version)" +DEFINE_string version "" \ + "Overrides version number in name to this version." + +# Parse command line. +FLAGS "$@" || exit 1 +[ -z "${FLAGS_ARGV}" ] && echo 'No version given' && exit 0 +eval set -- "${FLAGS_ARGV}" + +# Only now can we die on error. shflags functions leak non-zero error codes, +# so will die prematurely if 'switch_to_strict_mode' is specified before now. +switch_to_strict_mode + +# If downloading packages is enabled ensure the board is configured properly. +if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then + "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \ + --getbinpkgver="${FLAGS_getbinpkgver}" --regen_configs_only +fi + +# N.B. Ordering matters for some of the libraries below, because +# some of the files contain initialization used by later files. +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/ebuild_aci_util.sh" || exit 1 + +BUILD_DIR=${FLAGS_build_dir:-$BUILD_DIR} + +# TODO: this should be per-version; not all docker versions have the same binaries +packaged_files=( + "/usr/bin/docker" + "/usr/bin/dockerd" + "/usr/bin/docker-containerd" + "/usr/bin/docker-containerd-shim" + "/usr/bin/docker-proxy" + "/usr/bin/docker-runc" + "/usr/lib/coreos/dockerd" +) + +for version; do + ebuild_aci_create "users.developer.core-os.net/skim/docker" "coreos_docker-${version}-${BOARD}" "app-emulation/docker" "${version}" "${packaged_files[@]}" +done diff --git a/build_library/ebuild_aci_manifest.in b/build_library/ebuild_aci_manifest.in new file mode 100644 index 0000000000..17365203a1 --- /dev/null +++ b/build_library/ebuild_aci_manifest.in @@ -0,0 +1,14 @@ +{ + "acKind": "ImageManifest", + "acVersion": "0.8.6", + "name": "@ACI_NAME@", + "labels": [ + {"name": "arch", "value": "@ACI_ARCH@"}, + {"name": "os", "value": "linux"}, + {"name": "version", "value": "@ACI_VERSION@"} + ], + "app": { + "user": "0", + "group": "0" + } +} diff --git a/build_library/ebuild_aci_util.sh b/build_library/ebuild_aci_util.sh new file mode 100644 index 0000000000..fcafc3b31e --- /dev/null +++ b/build_library/ebuild_aci_util.sh @@ -0,0 +1,96 @@ +# Copyright (c) 2016 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Expects BOARD, BUILD_DIR, BUILD_LIBRARY_DIR, and COREOS_VERSION in env. + +# Copied from create_prod_image() +create_ebuild_aci_image() { + local image_name="$1" + local disk_layout="$2" + local update_group="$3" + local pkg="$4" + + info "Building ACI staging image ${image_name}" + local root_fs_dir="${BUILD_DIR}/rootfs" + local image_contents="${image_name%.bin}_contents.txt" + local image_packages="${image_name%.bin}_packages.txt" + local image_licenses="${image_name%.bin}_licenses.txt" + + start_image \ + "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" + + # Install minimal GCC (libs only) and then everything else + extract_prod_gcc "${root_fs_dir}" + + emerge_to_image_unchecked "${root_fs_dir}" "${pkg}" + run_ldconfig "${root_fs_dir}" + write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" + write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + + cleanup_mounts "${root_fs_dir}" + trap - EXIT +} + +ebuild_aci_write_manifest() { + local manifest="${1?No output path was specified}" + local name="${2?No ACI name was specified}" + local version="${3?No ACI version was specified}" + local appc_arch= + + case "${BOARD}" in + amd64-usr) appc_arch=amd64 ;; + arm64-usr) appc_arch=aarch64 ;; + *) die_notrace "Cannot map \"${BOARD}\" to an appc arch" ;; + esac + + sudo cp "${BUILD_LIBRARY_DIR}/ebuild_aci_manifest.in" "${manifest}" + sudo sed "${manifest}" -i \ + -e "s,@ACI_NAME@,${name}," \ + -e "s,@ACI_VERSION@,${version}," \ + -e "s,@ACI_ARCH@,${appc_arch}," +} + +ebuild_aci_create() { + local aciroot="${BUILD_DIR}" + local aci_name="${1?No aci name was specified}"; shift + local output_image="${1?No output file specified}"; shift + local pkg="${1?No package given}"; shift + local version="${1?No package version given}"; shift + local pkg_files=( "${@}" ) + + local staging_image="coreos_pkg_staging_aci_stage.bin" + + local ebuild_atom="=${pkg}-${version}" + + local ebuild=$(equery-"${BOARD}" w "${ebuild_atom}" 2>/dev/null) + [ -n "${ebuild}" ] || die_notrace "No ebuild exists for ebuild \"${pkg}\"" + + # Build a staging image for this ebuild. + create_ebuild_aci_image "${staging_image}" container stable "${ebuild_atom}" + + # Remount the staging image to brutalize the rootfs for broken services. + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=container \ + mount "${BUILD_DIR}/${staging_image}" "${aciroot}/rootfs" + trap "cleanup_mounts '${aciroot}/rootfs' && delete_prompt" EXIT + + # Substitute variables into the manifest to produce the final version. + # TODO: tag with the right version + ebuild_aci_write_manifest \ + "${aciroot}/manifest" \ + "${aci_name}" \ + "${version}" + + local pkg_files_in_rootfs=( "${pkg_files[@]/#/rootfs}" ) + + # Write a tar ACI file containing the manifest and desired parts of the mounted rootfs + sudo tar -C "${aciroot}" -hczf "${BUILD_DIR}/${output_image}.aci" \ + manifest ${pkg_files_in_rootfs[@]} + + # Unmount the staging image, and delete it to save space. + cleanup_mounts "${aciroot}/rootfs" + trap - EXIT + rm -f "${BUILD_DIR}/${staging_image}" + + echo "Created aci for ${pkg}-${version}: ${BUILD_DIR}/${output_image}.aci" +} From ada33ad411455d0c365a136da918c91ec996f978 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Mon, 6 Mar 2017 14:38:06 -0800 Subject: [PATCH 3/4] build_docker_aci: update version / file logic --- build_docker_aci | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/build_docker_aci b/build_docker_aci index 8174eb8bbc..19e479f50f 100755 --- a/build_docker_aci +++ b/build_docker_aci @@ -47,7 +47,7 @@ DEFINE_string group "docker-aci" \ DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string version "" \ - "Overrides version number in name to this version." + "Sets the docker version to build." # Parse command line. FLAGS "$@" || exit 1 @@ -75,17 +75,23 @@ fi BUILD_DIR=${FLAGS_build_dir:-$BUILD_DIR} -# TODO: this should be per-version; not all docker versions have the same binaries -packaged_files=( - "/usr/bin/docker" - "/usr/bin/dockerd" - "/usr/bin/docker-containerd" - "/usr/bin/docker-containerd-shim" - "/usr/bin/docker-proxy" - "/usr/bin/docker-runc" - "/usr/lib/coreos/dockerd" -) - for version; do - ebuild_aci_create "users.developer.core-os.net/skim/docker" "coreos_docker-${version}-${BOARD}" "app-emulation/docker" "${version}" "${packaged_files[@]}" + case "${version}" in + 1.12.[0-9]*) + packaged_files=( + "/usr/bin/docker" + "/usr/bin/dockerd" + "/usr/bin/docker-containerd" + "/usr/bin/docker-containerd-shim" + "/usr/bin/docker-proxy" + "/usr/bin/docker-runc" + "/usr/lib/coreos/dockerd" + ) + ebuild_aci_create "users.developer.core-os.net/skim/docker" "coreos_docker-${version}-${BOARD}" "app-emulation/docker" "${version}" "${packaged_files[@]}" + ;; + *) + 1>&2 echo "Unrecognized version; please enter a supported version" + exit 1 + ;; + esac done From b8589683b87b31bd5d84c07cd01d914a186cd32f Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Mon, 6 Mar 2017 15:19:15 -0800 Subject: [PATCH 4/4] build_docker_aci: include 'extra' version number This allows for multiple iterations with the same package version if needed. --- build_docker_aci | 59 +++++++++++++++++++------------- build_library/ebuild_aci_util.sh | 4 +-- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/build_docker_aci b/build_docker_aci index 19e479f50f..22207690f4 100755 --- a/build_docker_aci +++ b/build_docker_aci @@ -25,14 +25,18 @@ DEFINE_boolean getbinpkg "${FLAGS_FALSE}" \ DEFINE_string getbinpkgver "" \ "Use binary packages from a specific version." -FLAGS_HELP="USAGE: build_docker_aci [flags] [docker version]. +FLAGS_HELP="USAGE: build_docker_aci [flags] [docker version] [aci version number]. This script is used to build a CoreOS docker-skim ACI. -The version should identify an existent ebuild (i.e. app-emulation/docker-\$version). +The docker version should identify an existent ebuild (i.e. +app-emulation/docker-\$version). + +The aci version number is an atomically incrementing number that will be +appended to the aci version (to create e.g. :v1.12.6_coreos.0). Examples: -build_docker_aci --board=amd64-usr --build_dir= 1.12.6 +build_docker_aci --board=amd64-usr --build_dir= 1.12.6 0 ... " show_help_if_requested "$@" @@ -48,12 +52,18 @@ DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string version "" \ "Sets the docker version to build." +DEFINE_integer aci_version "" \ + "Sets the aci version tag identifier." # Parse command line. FLAGS "$@" || exit 1 [ -z "${FLAGS_ARGV}" ] && echo 'No version given' && exit 0 eval set -- "${FLAGS_ARGV}" +version="${1:?Docker version}" +aci_version="${2:?Docker version}" + + # Only now can we die on error. shflags functions leak non-zero error codes, # so will die prematurely if 'switch_to_strict_mode' is specified before now. switch_to_strict_mode @@ -75,23 +85,26 @@ fi BUILD_DIR=${FLAGS_build_dir:-$BUILD_DIR} -for version; do - case "${version}" in - 1.12.[0-9]*) - packaged_files=( - "/usr/bin/docker" - "/usr/bin/dockerd" - "/usr/bin/docker-containerd" - "/usr/bin/docker-containerd-shim" - "/usr/bin/docker-proxy" - "/usr/bin/docker-runc" - "/usr/lib/coreos/dockerd" - ) - ebuild_aci_create "users.developer.core-os.net/skim/docker" "coreos_docker-${version}-${BOARD}" "app-emulation/docker" "${version}" "${packaged_files[@]}" - ;; - *) - 1>&2 echo "Unrecognized version; please enter a supported version" - exit 1 - ;; - esac -done +case "${version}" in + 1.12.[0-9]*) + packaged_files=( + "/usr/bin/docker" + "/usr/bin/dockerd" + "/usr/bin/docker-containerd" + "/usr/bin/docker-containerd-shim" + "/usr/bin/docker-proxy" + "/usr/bin/docker-runc" + "/usr/lib/coreos/dockerd" + ) + ebuild_aci_create "users.developer.core-os.net/skim/docker" \ + "coreos_docker-${BOARD}-${version}_coreos.${aci_version}" \ + "app-emulation/docker" \ + "${version}" \ + "${aci_version}" \ + "${packaged_files[@]}" + ;; + *) + 1>&2 echo "Unrecognized version; please enter a supported version" + exit 1 + ;; +esac diff --git a/build_library/ebuild_aci_util.sh b/build_library/ebuild_aci_util.sh index fcafc3b31e..91d78fd42d 100644 --- a/build_library/ebuild_aci_util.sh +++ b/build_library/ebuild_aci_util.sh @@ -57,6 +57,7 @@ ebuild_aci_create() { local output_image="${1?No output file specified}"; shift local pkg="${1?No package given}"; shift local version="${1?No package version given}"; shift + local extra_version="${1?No extra version number given}"; shift local pkg_files=( "${@}" ) local staging_image="coreos_pkg_staging_aci_stage.bin" @@ -75,11 +76,10 @@ ebuild_aci_create() { trap "cleanup_mounts '${aciroot}/rootfs' && delete_prompt" EXIT # Substitute variables into the manifest to produce the final version. - # TODO: tag with the right version ebuild_aci_write_manifest \ "${aciroot}/manifest" \ "${aci_name}" \ - "${version}" + "${version}_coreos.${extra_version}" local pkg_files_in_rootfs=( "${pkg_files[@]/#/rootfs}" )