Merge branch 'releasing' into 'master'

Remove misc workarounds, split/rework release stage

See merge request archlinux/archlinux-docker!86

.gitlab-ci.yml

@@ -7,6 +7,7 @@ stages:
   - rootfs
   - image
   - test
+  - pre-release
   - release
   - publish
 
@@ -100,8 +101,6 @@ rootfs:secure:
       - GROUP: [base, base-devel, multilib-devel]
   tags:
     - vm
-  before_script:
-    - pacman -Syu --noconfirm podman
   id_tokens:
     SIGSTORE_ID_TOKEN:
       aud: sigstore
@@ -178,11 +177,6 @@ image:publish:secure:
 .test:
   stage: test
   dependencies: []
-  only:
-    variables:
-      # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/259663
-      # This is fine as at this point we're sure that the release works anyway.
-      - $GITLAB_USER_EMAIL != "project10185_bot2@example.com"
   except:
     refs:
       - releases
@@ -215,8 +209,8 @@ test:base-devel:
     - test -u /usr/bin/sudo   # issue 70
     - test -u /usr/bin/passwd
 
-release:
+pre-release:
-  stage: release
+  stage: pre-release
   image: registry.gitlab.com/gitlab-org/release-cli:latest
   tags:
     - secure
@@ -226,26 +220,27 @@ release:
       - schedules@archlinux/archlinux-docker
     variables:
       - $PUBLISH_ARCHLINUX_REPOSITORY == "TRUE"
-      - $PUBLISH_OFFICIAL_LIBRARY == "TRUE"
   before_script:
     - apk update
-    - apk add jq curl httpie bash
+    - apk add jq curl bash
   script:
     - |
-      # Update the description on https://hub.docker.com/r/archlinux/archlinux
-      TOKEN="$(http --ignore-stdin POST https://hub.docker.com/v2/users/login username="${DOCKERHUB_USERNAME}" password="${DOCKERHUB_PASSWORD}" | jq -er .token)"
-      http --ignore-stdin PATCH https://hub.docker.com/v2/repositories/archlinux/archlinux/ Authorization:"JWT ${TOKEN}" full_description="$(cat README.md)"
-
       # Upload rootfs to the Generic Packages Repository
       for group in base base-devel multilib-devel; do
-        sed -i "s|${group}.tar.zst|${group}-${BUILD_VERSION}.tar.zst|" output/${group}.tar.zst.SHA256
-        echo "Uploading ${group}.tar.zst"
-        curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst
-        echo "Uploading ${group}.tar.zst.SHA256"
-        curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst.SHA256
-        package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.zst)
         rootfs_file="${group}-${BUILD_VERSION}.tar.zst"
-        ./scripts/make-dockerfile.sh "${rootfs_file}" "${group}" "output" "curl -sOJL \"${package_url}\"" "${group}"
+        mv "output/${group}.tar.zst" "output/${rootfs_file}"
+        mv "output/${group}.tar.zst.SHA256" "output/${rootfs_file}.SHA256"
+        sed -i "s|${group}.tar.zst|${rootfs_file}|" "output/${rootfs_file}.SHA256"
+        echo "Uploading ${rootfs_file}"
+        curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "output/${rootfs_file}" "${PACKAGE_REGISTRY_URL}/"
+        echo "Uploading ${rootfs_file}.SHA256"
+        curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file "output/${rootfs_file}.SHA256" "${PACKAGE_REGISTRY_URL}/"
+      done
+
+      # Create the Dockerfiles, commit to the release branch
+      for group in base base-devel multilib-devel; do
+        rootfs_file="${group}-${BUILD_VERSION}.tar.zst"
+        ./scripts/make-dockerfile.sh "${rootfs_file}" "${group}" "output" "curl -sOJL \"${PACKAGE_REGISTRY_URL}/${rootfs_file}\"" "${group}"
         sed -i "/^COPY ${rootfs_file} \/$/d" output/Dockerfile.${group}
       done
     - >
@@ -270,41 +265,51 @@ release:
       --form "actions[][content]=<sigstore-param-file.yaml"
       "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/commits"
     - echo "BUILD_COMMIT=$(jq -r '.id' commit-response.json)" >> build.env
-    - |
-      base_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.zst)
-      echo "${base_url}"
-      base_sha_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.zst.SHA256)
-      echo "${base_sha_url}"
-      base_devel_url=$(./ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.zst)
-      echo "${base_devel_url}"
-      base_devel_sha_url=$(./ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.zst.SHA256)
-      echo "${base_devel_sha_url}"
-
-      multilib_devel_url=$(./ci/get-public-download-for-generic-package.sh multilib-devel-${BUILD_VERSION}.tar.zst)
-      echo "${multilib_devel_url}"
-      multilib_devel_sha_url=$(./ci/get-public-download-for-generic-package.sh multilib-devel-${BUILD_VERSION}.tar.zst.SHA256)
-      echo "${multilib_devel_sha_url}"
-
-      # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.zst\"
-      # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version!
-      echo "Creating release"
-      release-cli --private-token "${GITLAB_PROJECT_TOKEN}" create \
-      --name "Release ${BUILD_VERSION}" \
-      --description "Release ${BUILD_VERSION}" \
-      --tag-name v${BUILD_VERSION} --ref "releases" \
-      --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.zst\",\"url\":\"${base_url}\"}" \
-      --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.zst.SHA256\",\"url\":\"${base_sha_url}\"}" \
-      --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.zst\",\"url\":\"${base_devel_url}\"}" \
-      --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.zst.SHA256\",\"url\":\"${base_devel_sha_url}\"}" \
-      --assets-link "{\"name\":\"multilib-devel-${BUILD_VERSION}.tar.zst\",\"url\":\"${multilib_devel_url}\"}" \
-      --assets-link "{\"name\":\"multilib-devel-${BUILD_VERSION}.tar.zst.SHA256\",\"url\":\"${multilib_devel_sha_url}\"}"
   artifacts:
     reports:
       dotenv: build.env
 
+release:
+  stage: release
+  dependencies:
+    - get_version
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  tags:
+    - secure
+    - docker
+  only:
+    refs:
+      - schedules@archlinux/archlinux-docker
+    variables:
+      - $PUBLISH_ARCHLINUX_REPOSITORY == "TRUE"
+  script:
+    - echo 'Creating release'
+  release:
+    name: 'Release ${BUILD_VERSION}'
+    description: 'Release ${BUILD_VERSION}'
+    tag_name: 'v${BUILD_VERSION}'
+    ref: 'releases'
+    assets:
+      links:
+        - name: 'base-${BUILD_VERSION}.tar.zst'
+          url: '${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.zst'
+        - name: 'base-${BUILD_VERSION}.tar.zst.SHA256'
+          url: '${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.zst.SHA256'
+        - name: 'base-devel-${BUILD_VERSION}.tar.zst'
+          url: '${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.zst'
+        - name: 'base-devel-${BUILD_VERSION}.tar.zst.SHA256'
+          url: '${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.zst.SHA256'
+        - name: 'multilib-devel-${BUILD_VERSION}.tar.zst'
+          url: '${PACKAGE_REGISTRY_URL}/multilib-devel-${BUILD_VERSION}.tar.zst'
+        - name: 'multilib-devel-${BUILD_VERSION}.tar.zst.SHA256'
+          url: '${PACKAGE_REGISTRY_URL}/multilib-devel-${BUILD_VERSION}.tar.zst.SHA256'
+
 # Publish to the official Docker namespace: https://hub.docker.com/_/archlinux
 publish-dockerhub:
   stage: publish
+  dependencies:
+    - get_version
+    - release
   only:
     refs:
       - schedules
@@ -316,10 +321,14 @@ publish-dockerhub:
     - test -n "$BUILD_VERSION"
     - test -n "$BUILD_COMMIT"
     - test -n "$GITHUB_TOKEN"
-    - pacman -Syu --noconfirm github-cli git gettext
+    - pacman -Syu --noconfirm jq httpie github-cli git gettext
     - git config --global user.email "github@archlinux.org"
     - git config --global user.name "Arch Linux Technical User"
   script:
+    - echo "Update the description on https://hub.docker.com/r/archlinux/archlinux"
+    - TOKEN="$(http --ignore-stdin POST https://hub.docker.com/v2/users/login username="${DOCKERHUB_USERNAME}" password="${DOCKERHUB_PASSWORD}" | jq -er .token)"
+    - http --ignore-stdin PATCH https://hub.docker.com/v2/repositories/archlinux/archlinux/ Authorization:"JWT ${TOKEN}" full_description="$(cat README.md)"
+    - echo "Update the Docker library Github repo"
     - mkdir official-images
     - cd official-images
     - git init

ci/get-public-download-for-generic-package.sh

@@ -1,21 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-package_name=$1
-
-package_id=$(curl -sSf --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages?sort=desc&per_page=1" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id")
-
-if [[ -z "${package_id}" ]]; then
-    >&2 echo "Error: No package id found"
-    exit 1
-fi
-
-package_file_id=$(curl -sSf --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"$package_name\") | .id")
-
-if [[ -z "${package_file_id}" ]]; then
-    >&2 echo "Error: No package file id found"
-    exit 1
-fi
-
-echo "https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${package_file_id}/download"

scripts/make-dockerfile.sh

@@ -14,7 +14,7 @@ CI_COMMIT_SHA="${CI_COMMIT_SHA:-$(git rev-parse HEAD)}"
 
 sed -e "s|TEMPLATE_ROOTFS_FILE|$ROOTFS_FILE|" \
     -e "s|TEMPLATE_ROOTFS_DOWNLOAD|$DOWNLOAD|" \
-    -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$GROUP.tar.zst.SHA256)|" \
+    -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$ROOTFS_FILE.SHA256)|" \
     -e "s|TEMPLATE_TITLE|Arch Linux $TITLE Image|" \
     -e "s|TEMPLATE_VERSION_ID|$BUILD_VERSION|" \
     -e "s|TEMPLATE_REVISION|$CI_COMMIT_SHA|" \