mirror of
https://github.com/flatcar/scripts.git
synced 2025-09-24 15:11:19 +02:00
Merge pull request #475 from flatcar/kai/plume-release
ci-automation/release.sh: Run plume to release cloud images
This commit is contained in:
Kai Lüke
GitHub
commit
d6cb1009c6
@ -310,7 +310,8 @@ function secret_to_file() {
|
||||
exec {fd}<>"${tmpfile}"
|
||||
rm -f "${tmpfile}"
|
||||
echo "${secret}" | base64 --decode >&${fd}
|
||||
config_ref="/proc/${$}/fd/${fd}"
|
||||
# Use BASHPID because we may be in a subshell but $$ is only the main shell's PID
|
||||
config_ref="/proc/${BASHPID}/fd/${fd}"
|
||||
}
|
||||
# --
|
||||
|
||||
|
||||
@ -38,6 +38,16 @@
|
||||
# Defaults to nothing if not set - in such case, artifacts will not be signed.
|
||||
# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
|
||||
#
|
||||
# 3. REGISTRY_USERNAME. Environment variable. The username to use for Docker registry login.
|
||||
# Defaults to nothing if not set - in such case, SDK container will not be pushed.
|
||||
#
|
||||
# 4. REGISTRY_PASSWORD. Environment variable. The password to use for Docker registry login.
|
||||
# Defaults to nothing if not set - in such case, SDK container will not be pushed.
|
||||
#
|
||||
# 5. Cloud credentials as secrets via the environment variables AZURE_PROFILE, AZURE_AUTH_CREDENTIALS,
|
||||
# AWS_CREDENTIALS, AWS_MARKETPLACE_CREDENTIALS, AWS_MARKETPLACE_ARN, AWS_CLOUDFORMATION_CREDENTIALS,
|
||||
# GCP_JSON_KEY, GOOGLE_RELEASE_CREDENTIALS.
|
||||
#
|
||||
# OUTPUT:
|
||||
#
|
||||
# 1. The cloud images are published with mantle's plume and ore tools
|
||||
@ -62,26 +72,133 @@ function _inside_mantle() {
|
||||
(
|
||||
set -euo pipefail
|
||||
|
||||
source sdk_lib/sdk_container_common.sh
|
||||
source ci-automation/ci_automation_common.sh
|
||||
source sdk_container/.repo/manifests/version.txt
|
||||
# Needed because we are not the SDK container here
|
||||
source sdk_container/.env
|
||||
CHANNEL="$(get_git_channel)"
|
||||
VERSION="${FLATCAR_VERSION}"
|
||||
azure_profile_config_file=""
|
||||
secret_to_file azure_profile_config_file "${AZURE_PROFILE}"
|
||||
azure_auth_config_file=""
|
||||
secret_to_file azure_auth_config_file "${AZURE_AUTH_CREDENTIALS}"
|
||||
aws_credentials_config_file=""
|
||||
secret_to_file aws_credentials_config_file "${AWS_CREDENTIALS}"
|
||||
aws_marketplace_credentials_file=""
|
||||
secret_to_file aws_marketplace_credentials_file "${AWS_MARKETPLACE_CREDENTIALS}"
|
||||
gcp_json_key_path=""
|
||||
secret_to_file gcp_json_key_path "${GCP_JSON_KEY}"
|
||||
google_release_credentials_file=""
|
||||
secret_to_file google_release_credentials_file "${GOOGLE_RELEASE_CREDENTIALS}"
|
||||
|
||||
# TODO: set up credentials
|
||||
# TODO: run mantle pre-release and release for all platforms
|
||||
# (needs changes in mantle to consume from buildcache via https)
|
||||
# TODO: run ore for AWS marketplace upload
|
||||
for platform in aws azure; do
|
||||
for arch in amd64 arm64; do
|
||||
# Create a folder where plume stores flatcar_production_ami_*txt and flatcar_production_ami_*json
|
||||
# for later push to bincache
|
||||
rm -rf "${platform}-${arch}"
|
||||
mkdir "${platform}-${arch}"
|
||||
pushd "${platform}-${arch}"
|
||||
|
||||
# For pre-release we don't use the Google Cloud token because it's not needed
|
||||
# and we don't want to upload the AMIs to GCS anymore
|
||||
# (change https://github.com/flatcar/mantle/blob/bc6bc232677c45e389feb221da295cc674882f8c/cmd/plume/prerelease.go#L663-L667
|
||||
# if you want to add GCP release code in plume pre-release instead of plume release)
|
||||
plume pre-release --force \
|
||||
--debug \
|
||||
--platform="${platform}" \
|
||||
--aws-credentials="${aws_credentials_config_file}" \
|
||||
--azure-profile="${azure_profile_config_file}" \
|
||||
--azure-auth="${azure_auth_config_file}" \
|
||||
--gce-json-key=none \
|
||||
--board="${arch}-usr" \
|
||||
--channel="${CHANNEL}" \
|
||||
--version="${FLATCAR_VERSION}" \
|
||||
--write-image-list="images.json"
|
||||
popd
|
||||
done
|
||||
done
|
||||
for arch in amd64 arm64; do
|
||||
# Create a folder where plume stores any temporarily downloaded files
|
||||
rm -rf "release-${arch}"
|
||||
mkdir "release-${arch}"
|
||||
pushd "release-${arch}"
|
||||
|
||||
export product="${CHANNEL}-${arch}"
|
||||
pid=$(jq -r ".[env.product]" ../product-ids.json)
|
||||
|
||||
# If the channel is 'stable' and the arch 'amd64', we add the stable-pro-amd64 product ID to the product IDs.
|
||||
# The published AMI ID is the same for both offer.
|
||||
[[ "${CHANNEL}" == "stable" ]] && [[ "${arch}" == "amd64" ]] && pid="${pid},$(jq -r '.["stable-pro-amd64"]' ../product-ids.json)"
|
||||
|
||||
plume release \
|
||||
--debug \
|
||||
--aws-credentials="${aws_credentials_config_file}" \
|
||||
--aws-marketplace-credentials="${aws_marketplace_credentials_file}" \
|
||||
--publish-marketplace \
|
||||
--access-role-arn="${AWS_MARKETPLACE_ARN}" \
|
||||
--product-ids="${pid}" \
|
||||
--azure-profile="${azure_profile_config_file}" \
|
||||
--azure-auth="${azure_auth_config_file}" \
|
||||
--gce-json-key="${gcp_json_key_path}" \
|
||||
--gce-release-key="${google_release_credentials_file}" \
|
||||
--board="${arch}-usr" \
|
||||
--channel="${CHANNEL}" \
|
||||
--version="${VERSION}"
|
||||
popd
|
||||
done
|
||||
|
||||
# Future: move this to "plume release", in the past this was done in "update-cloudformation-template"
|
||||
aws_cloudformation_credentials_file=""
|
||||
secret_to_file aws_cloudformation_credentials_file "${AWS_CLOUDFORMATION_CREDENTIALS}"
|
||||
export AWS_SHARED_CREDENTIALS_FILE="${aws_cloudformation_credentials_file}"
|
||||
rm -rf cloudformation-files
|
||||
mkdir cloudformation-files
|
||||
for arch in amd64 arm64; do
|
||||
generate_templates "aws-${arch}/flatcar_production_ami_all.json" "${CHANNEL}" "${arch}-usr"
|
||||
done
|
||||
aws s3 cp --recursive --acl public-read cloudformation-files/ "s3://flatcar-prod-ami-import-eu-central-1/dist/aws/"
|
||||
)
|
||||
}
|
||||
|
||||
function publish_sdk() {
|
||||
local docker_sdk_vernum="$1"
|
||||
local sdk_name=""
|
||||
|
||||
# If the registry password or the registry username is not set, we leave early.
|
||||
[[ -z "${REGISTRY_PASSWORD}" ]] || [[ -z "${REGISTRY_USERNAME}" ]] && return
|
||||
|
||||
(
|
||||
# Don't print the password to stderr when logging in
|
||||
set +x
|
||||
local container_registry=""
|
||||
container_registry=$(echo "${sdk_container_common_registry}" | cut -d / -f 1)
|
||||
echo "${REGISTRY_PASSWORD}" | docker login "${container_registry}" -u "${REGISTRY_USERNAME}" --password-stdin
|
||||
)
|
||||
|
||||
# Docker images are pushed in the container registry.
|
||||
for a in all amd64 arm64; do
|
||||
sdk_name="flatcar-sdk-${a}"
|
||||
docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
|
||||
docker push "${sdk_container_common_registry}/flatcar-sdk-${a}:${docker_sdk_vernum}"
|
||||
done
|
||||
}
|
||||
|
||||
function _release_build_impl() {
|
||||
source sdk_lib/sdk_container_common.sh
|
||||
source ci-automation/ci_automation_common.sh
|
||||
source ci-automation/gpg_setup.sh
|
||||
init_submodules
|
||||
|
||||
source sdk_container/.repo/manifests/version.txt
|
||||
# Needed because we are not the SDK container here
|
||||
source sdk_container/.env
|
||||
local sdk_version="${FLATCAR_SDK_VERSION}"
|
||||
local docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
|
||||
local docker_sdk_vernum=""
|
||||
docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
|
||||
local vernum="${FLATCAR_VERSION}"
|
||||
local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
|
||||
local docker_vernum=""
|
||||
docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
|
||||
|
||||
local container_name="flatcar-publish-${docker_vernum}"
|
||||
local mantle_ref
|
||||
@ -92,9 +209,14 @@ function _release_build_impl() {
|
||||
touch sdk_container/.env # This file should already contain the required credentials as env vars
|
||||
docker run --pull always --rm --name="${container_name}" --net host \
|
||||
-w /work -v "$PWD":/work "${mantle_ref}" bash -c "source ci-automation/release.sh; _inside_mantle"
|
||||
# TODO: sign and copy resulting AMI text file to buildcache
|
||||
# TODO: run CF template update
|
||||
# TODO: publish SDK container image if not published yet (i.e., on new majors)
|
||||
# Push flatcar_production_ami_*txt and flatcar_production_ami_*json to the right bincache folder
|
||||
for arch in amd64 arm64; do
|
||||
sudo chown -R "$USER:$USER" "aws-${arch}"
|
||||
create_digests "${SIGNER}" "aws-${arch}/flatcar_production_ami_"*txt "aws-${arch}/flatcar_production_ami_"*json
|
||||
sign_artifacts "${SIGNER}" "aws-${arch}/flatcar_production_ami_"*txt "aws-${arch}/flatcar_production_ami_"*json
|
||||
copy_to_buildcache "images/${arch}/${vernum}/" "aws-${arch}/flatcar_production_ami_"*txt* "aws-${arch}/flatcar_production_ami_"*json*
|
||||
done
|
||||
publish_sdk "${docker_sdk_vernum}"
|
||||
echo "===="
|
||||
echo "Done, now you can copy the images to Origin"
|
||||
echo "===="
|
||||
@ -105,3 +227,179 @@ function _release_build_impl() {
|
||||
# Future: trigger push to nebraska
|
||||
# Future: trigger Origin symlink switch
|
||||
}
|
||||
|
||||
TEMPLATE='
|
||||
{
|
||||
"AWSTemplateFormatVersion": "2010-09-09",
|
||||
"Description": "Flatcar Linux on EC2: https://kinvolk.io/docs/flatcar-container-linux/latest/installing/cloud/aws-ec2/",
|
||||
"Mappings" : {
|
||||
"RegionMap" : {
|
||||
###AMIS###
|
||||
}
|
||||
},
|
||||
"Parameters": {
|
||||
"InstanceType" : {
|
||||
"Description" : "EC2 HVM instance type (m3.medium, etc).",
|
||||
"Type" : "String",
|
||||
"Default" : "m3.medium",
|
||||
"ConstraintDescription" : "Must be a valid EC2 HVM instance type."
|
||||
},
|
||||
"ClusterSize": {
|
||||
"Default": "3",
|
||||
"MinValue": "3",
|
||||
"MaxValue": "12",
|
||||
"Description": "Number of nodes in cluster (3-12).",
|
||||
"Type": "Number"
|
||||
},
|
||||
"DiscoveryURL": {
|
||||
"Description": "An unique etcd cluster discovery URL. Grab a new token from https://discovery.etcd.io/new?size=<your cluster size>",
|
||||
"Type": "String"
|
||||
},
|
||||
"AdvertisedIPAddress": {
|
||||
"Description": "Use 'private' if your etcd cluster is within one region or 'public' if it spans regions or cloud providers.",
|
||||
"Default": "private",
|
||||
"AllowedValues": ["private", "public"],
|
||||
"Type": "String"
|
||||
},
|
||||
"AllowSSHFrom": {
|
||||
"Description": "The net block (CIDR) that SSH is available to.",
|
||||
"Default": "0.0.0.0/0",
|
||||
"Type": "String"
|
||||
},
|
||||
"KeyPair" : {
|
||||
"Description" : "The name of an EC2 Key Pair to allow SSH access to the instance.",
|
||||
"Type" : "String"
|
||||
}
|
||||
},
|
||||
"Resources": {
|
||||
"FlatcarSecurityGroup": {
|
||||
"Type": "AWS::EC2::SecurityGroup",
|
||||
"Properties": {
|
||||
"GroupDescription": "Flatcar Linux SecurityGroup",
|
||||
"SecurityGroupIngress": [
|
||||
{"IpProtocol": "tcp", "FromPort": "22", "ToPort": "22", "CidrIp": {"Ref": "AllowSSHFrom"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"Ingress4001": {
|
||||
"Type": "AWS::EC2::SecurityGroupIngress",
|
||||
"Properties": {
|
||||
"GroupName": {"Ref": "FlatcarSecurityGroup"}, "IpProtocol": "tcp", "FromPort": "4001", "ToPort": "4001", "SourceSecurityGroupId": {
|
||||
"Fn::GetAtt" : [ "FlatcarSecurityGroup", "GroupId" ]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Ingress2379": {
|
||||
"Type": "AWS::EC2::SecurityGroupIngress",
|
||||
"Properties": {
|
||||
"GroupName": {"Ref": "FlatcarSecurityGroup"}, "IpProtocol": "tcp", "FromPort": "2379", "ToPort": "2379", "SourceSecurityGroupId": {
|
||||
"Fn::GetAtt" : [ "FlatcarSecurityGroup", "GroupId" ]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Ingress2380": {
|
||||
"Type": "AWS::EC2::SecurityGroupIngress",
|
||||
"Properties": {
|
||||
"GroupName": {"Ref": "FlatcarSecurityGroup"}, "IpProtocol": "tcp", "FromPort": "2380", "ToPort": "2380", "SourceSecurityGroupId": {
|
||||
"Fn::GetAtt" : [ "FlatcarSecurityGroup", "GroupId" ]
|
||||
}
|
||||
}
|
||||
},
|
||||
"FlatcarServerAutoScale": {
|
||||
"Type": "AWS::AutoScaling::AutoScalingGroup",
|
||||
"Properties": {
|
||||
"AvailabilityZones": {"Fn::GetAZs": ""},
|
||||
"LaunchConfigurationName": {"Ref": "FlatcarServerLaunchConfig"},
|
||||
"MinSize": "3",
|
||||
"MaxSize": "12",
|
||||
"DesiredCapacity": {"Ref": "ClusterSize"},
|
||||
"Tags": [
|
||||
{"Key": "Name", "Value": { "Ref" : "AWS::StackName" }, "PropagateAtLaunch": true}
|
||||
]
|
||||
}
|
||||
},
|
||||
"FlatcarServerLaunchConfig": {
|
||||
"Type": "AWS::AutoScaling::LaunchConfiguration",
|
||||
"Properties": {
|
||||
"ImageId" : { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
|
||||
"InstanceType": {"Ref": "InstanceType"},
|
||||
"KeyName": {"Ref": "KeyPair"},
|
||||
"SecurityGroups": [{"Ref": "FlatcarSecurityGroup"}],
|
||||
"UserData" : { "Fn::Base64":
|
||||
{ "Fn::Join": [ "", [
|
||||
"#cloud-config\n\n",
|
||||
"coreos:\n",
|
||||
" etcd2:\n",
|
||||
" discovery: ", { "Ref": "DiscoveryURL" }, "\n",
|
||||
" advertise-client-urls: http://$", { "Ref": "AdvertisedIPAddress" }, "_ipv4:2379\n",
|
||||
" initial-advertise-peer-urls: http://$", { "Ref": "AdvertisedIPAddress" }, "_ipv4:2380\n",
|
||||
" listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001\n",
|
||||
" listen-peer-urls: http://$", { "Ref": "AdvertisedIPAddress" }, "_ipv4:2380\n",
|
||||
" units:\n",
|
||||
" - name: etcd2.service\n",
|
||||
" command: start\n",
|
||||
" - name: fleet.service\n",
|
||||
" command: start\n"
|
||||
] ]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
function generate_templates() {
|
||||
local IFILE="$1"
|
||||
local CHANNEL="$2"
|
||||
local BOARD="$3"
|
||||
local TMPFILE=""
|
||||
local ARCHTAG=""
|
||||
|
||||
local REGIONS=("eu-central-1"
|
||||
"ap-northeast-1"
|
||||
"ap-northeast-2"
|
||||
# "ap-northeast-3" # Disabled for now because we do not have access
|
||||
"af-south-1"
|
||||
"ca-central-1"
|
||||
"ap-south-1"
|
||||
"sa-east-1"
|
||||
"ap-southeast-1"
|
||||
"ap-southeast-2"
|
||||
"ap-southeast-3"
|
||||
"us-east-1"
|
||||
"us-east-2"
|
||||
"us-west-2"
|
||||
"us-west-1"
|
||||
"eu-west-1"
|
||||
"eu-west-2"
|
||||
"eu-west-3"
|
||||
"eu-north-1"
|
||||
"eu-south-1"
|
||||
"ap-east-1"
|
||||
"me-south-1")
|
||||
|
||||
if [ "${BOARD}" = "amd64-usr" ]; then
|
||||
ARCHTAG=""
|
||||
elif [ "${BOARD}" = "arm64-usr" ]; then
|
||||
ARCHTAG="-arm64"
|
||||
else
|
||||
echo "No architecture tag defined for board \"${BOARD}\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMPFILE=$(mktemp)
|
||||
|
||||
>${TMPFILE}
|
||||
for region in "${REGIONS[@]}"; do
|
||||
echo " \"${region}\" : {" >> ${TMPFILE}
|
||||
echo -n ' "AMI" : ' >> ${TMPFILE}
|
||||
cat "${IFILE}" | jq ".[] | map(select(.name == \"${region}\")) | .[0] | .\"hvm\"" >> ${TMPFILE}
|
||||
echo " }," >> ${TMPFILE}
|
||||
done
|
||||
|
||||
truncate -s-2 ${TMPFILE}
|
||||
|
||||
echo "${TEMPLATE}" | perl -i -0pe "s/###AMIS###/$(cat -- ${TMPFILE})/g" > "cloudformation-files/flatcar-${CHANNEL}${ARCHTAG}-hvm.template"
|
||||
|
||||
rm "${TMPFILE}"
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user