From caf091343a9e35aebec1ad6378cdf9b2e135e99b Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 9 Aug 2021 16:36:42 +0200 Subject: [PATCH 1/3] dev-lang/go: Drop the patch for go 1.12 The patch dropped some security changes related to URL parsing in go-1.12 to avoid breaking rkt. Since rkt is gone, the patch could be dropped too. --- .../go-1.12-revert-url-parsing-change.patch | 241 ------------------ ...go-1.12.17.ebuild => go-1.12.17-r1.ebuild} | 4 - 2 files changed, 245 deletions(-) delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-lang/go/files/go-1.12-revert-url-parsing-change.patch rename sdk_container/src/third_party/coreos-overlay/dev-lang/go/{go-1.12.17.ebuild => go-1.12.17-r1.ebuild} (67%) diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/go/files/go-1.12-revert-url-parsing-change.patch b/sdk_container/src/third_party/coreos-overlay/dev-lang/go/files/go-1.12-revert-url-parsing-change.patch deleted file mode 100644 index c51555fd9e..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/go/files/go-1.12-revert-url-parsing-change.patch +++ /dev/null @@ -1,241 +0,0 @@ -From 509793509fee8ada6d2d28cf0cd885a8f270bcf6 Mon Sep 17 00:00:00 2001 -From: Benjamin Gilbert -Date: Tue, 8 Oct 2019 20:43:53 -0400 -Subject: [PATCH] Revert "[release-branch.go1.12-security] net/url: make - Hostname and Port predictable for invalid Host values" - -This breaks rkt for docker:// URLs that don't specify a registry. - -This reverts commit 3226f2d492963d361af9dfc6714ef141ba606713. ---- - src/net/http/transport.go | 2 - - src/net/http/transport_test.go | 2 +- - src/net/url/url.go | 54 ++++++++++++------------ - src/net/url/url_test.go | 76 +++++++++++++++++----------------- - 4 files changed, 65 insertions(+), 69 deletions(-) - -diff --git a/src/net/http/transport.go b/src/net/http/transport.go -index e946760963..07920cfde3 100644 ---- a/src/net/http/transport.go -+++ b/src/net/http/transport.go -@@ -655,8 +655,6 @@ func resetProxyConfig() { - } - - func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { -- // TODO: the validPort check is redundant after CL 189258, as url.URL.Port -- // only returns valid ports now. golang.org/issue/33600 - if port := treq.URL.Port(); !validPort(port) { - return cm, fmt.Errorf("invalid URL port %q", port) - } -diff --git a/src/net/http/transport_test.go b/src/net/http/transport_test.go -index 5c329543e2..f66e72a00f 100644 ---- a/src/net/http/transport_test.go -+++ b/src/net/http/transport_test.go -@@ -4163,7 +4163,7 @@ func TestTransportRejectsAlphaPort(t *testing.T) { - t.Fatalf("got %#v; want *url.Error", err) - } - got := ue.Err.Error() -- want := `invalid port ":123foo" after host` -+ want := `invalid URL port "123foo"` - if got != want { - t.Errorf("got error %q; want %q", got, want) - } -diff --git a/src/net/url/url.go b/src/net/url/url.go -index 337861f80d..64274a0a36 100644 ---- a/src/net/url/url.go -+++ b/src/net/url/url.go -@@ -655,11 +655,6 @@ func parseHost(host string) (string, error) { - } - return host1 + host2 + host3, nil - } -- } else if i := strings.LastIndex(host, ":"); i != -1 { -- colonPort := host[i:] -- if !validOptionalPort(colonPort) { -- return "", fmt.Errorf("invalid port %q after host", colonPort) -- } - } - - var err error -@@ -1058,39 +1053,44 @@ func (u *URL) RequestURI() string { - return result - } - --// Hostname returns u.Host, stripping any valid port number if present. -+// Hostname returns u.Host, without any port number. - // --// If the result is enclosed in square brackets, as literal IPv6 addresses are, --// the square brackets are removed from the result. -+// If Host is an IPv6 literal with a port number, Hostname returns the -+// IPv6 literal without the square brackets. IPv6 literals may include -+// a zone identifier. - func (u *URL) Hostname() string { -- host, _ := splitHostPort(u.Host) -- return host -+ return stripPort(u.Host) - } - - // Port returns the port part of u.Host, without the leading colon. --// --// If u.Host doesn't contain a valid numeric port, Port returns an empty string. -+// If u.Host doesn't contain a port, Port returns an empty string. - func (u *URL) Port() string { -- _, port := splitHostPort(u.Host) -- return port -+ return portOnly(u.Host) - } - --// splitHostPort separates host and port. If the port is not valid, it returns --// the entire input as host, and it doesn't check the validity of the host. --// Unlike net.SplitHostPort, but per RFC 3986, it requires ports to be numeric. --func splitHostPort(hostport string) (host, port string) { -- host = hostport -- -- colon := strings.LastIndexByte(host, ':') -- if colon != -1 && validOptionalPort(host[colon:]) { -- host, port = host[:colon], host[colon+1:] -+func stripPort(hostport string) string { -+ colon := strings.IndexByte(hostport, ':') -+ if colon == -1 { -+ return hostport - } -- -- if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { -- host = host[1 : len(host)-1] -+ if i := strings.IndexByte(hostport, ']'); i != -1 { -+ return strings.TrimPrefix(hostport[:i], "[") - } -+ return hostport[:colon] -+} - -- return -+func portOnly(hostport string) string { -+ colon := strings.IndexByte(hostport, ':') -+ if colon == -1 { -+ return "" -+ } -+ if i := strings.Index(hostport, "]:"); i != -1 { -+ return hostport[i+len("]:"):] -+ } -+ if strings.Contains(hostport, "]") { -+ return "" -+ } -+ return hostport[colon+len(":"):] - } - - // Marshaling interface implementations. -diff --git a/src/net/url/url_test.go b/src/net/url/url_test.go -index b6f4623a52..c5fc90d515 100644 ---- a/src/net/url/url_test.go -+++ b/src/net/url/url_test.go -@@ -422,10 +422,10 @@ var urltests = []URLTest{ - }, - // worst case host, still round trips - { -- "scheme://!$&'()*+,;=hello!:1/path", -+ "scheme://!$&'()*+,;=hello!:port/path", - &URL{ - Scheme: "scheme", -- Host: "!$&'()*+,;=hello!:1", -+ Host: "!$&'()*+,;=hello!:port", - Path: "/path", - }, - "", -@@ -1420,13 +1420,11 @@ func TestParseErrors(t *testing.T) { - {"http://[::1]", false}, - {"http://[::1]:80", false}, - {"http://[::1]:namedport", true}, // rfc3986 3.2.3 -- {"http://x:namedport", true}, // rfc3986 3.2.3 - {"http://[::1]/", false}, - {"http://[::1]a", true}, - {"http://[::1]%23", true}, - {"http://[::1%25en0]", false}, // valid zone id - {"http://[::1]:", false}, // colon, but no port OK -- {"http://x:", false}, // colon, but no port OK - {"http://[::1]:%38%30", true}, // not allowed: % encoding only for non-ASCII - {"http://[::1%25%41]", false}, // RFC 6874 allows over-escaping in zone - {"http://[%10::1]", true}, // no %xx escapes in IP address -@@ -1618,46 +1616,46 @@ func TestURLErrorImplementsNetError(t *testing.T) { - } - } - --func TestURLHostnameAndPort(t *testing.T) { -+func TestURLHostname(t *testing.T) { - tests := []struct { -- in string // URL.Host field -- host string -- port string -+ host string // URL.Host field -+ want string - }{ -- {"foo.com:80", "foo.com", "80"}, -- {"foo.com", "foo.com", ""}, -- {"foo.com:", "foo.com", ""}, -- {"FOO.COM", "FOO.COM", ""}, // no canonicalization -- {"1.2.3.4", "1.2.3.4", ""}, -- {"1.2.3.4:80", "1.2.3.4", "80"}, -- {"[1:2:3:4]", "1:2:3:4", ""}, -- {"[1:2:3:4]:80", "1:2:3:4", "80"}, -- {"[::1]:80", "::1", "80"}, -- {"[::1]", "::1", ""}, -- {"[::1]:", "::1", ""}, -- {"localhost", "localhost", ""}, -- {"localhost:443", "localhost", "443"}, -- {"some.super.long.domain.example.org:8080", "some.super.long.domain.example.org", "8080"}, -- {"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:17000", "2001:0db8:85a3:0000:0000:8a2e:0370:7334", "17000"}, -- {"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", "2001:0db8:85a3:0000:0000:8a2e:0370:7334", ""}, -- -- // Ensure that even when not valid, Host is one of "Hostname", -- // "Hostname:Port", "[Hostname]" or "[Hostname]:Port". -- // See https://golang.org/issue/29098. -- {"[google.com]:80", "google.com", "80"}, -- {"google.com]:80", "google.com]", "80"}, -- {"google.com:80_invalid_port", "google.com:80_invalid_port", ""}, -- {"[::1]extra]:80", "::1]extra", "80"}, -- {"google.com]extra:extra", "google.com]extra:extra", ""}, -+ {"foo.com:80", "foo.com"}, -+ {"foo.com", "foo.com"}, -+ {"FOO.COM", "FOO.COM"}, // no canonicalization (yet?) -+ {"1.2.3.4", "1.2.3.4"}, -+ {"1.2.3.4:80", "1.2.3.4"}, -+ {"[1:2:3:4]", "1:2:3:4"}, -+ {"[1:2:3:4]:80", "1:2:3:4"}, -+ {"[::1]:80", "::1"}, - } - for _, tt := range tests { -- u := &URL{Host: tt.in} -- host, port := u.Hostname(), u.Port() -- if host != tt.host { -- t.Errorf("Hostname for Host %q = %q; want %q", tt.in, host, tt.host) -+ u := &URL{Host: tt.host} -+ got := u.Hostname() -+ if got != tt.want { -+ t.Errorf("Hostname for Host %q = %q; want %q", tt.host, got, tt.want) - } -- if port != tt.port { -- t.Errorf("Port for Host %q = %q; want %q", tt.in, port, tt.port) -+ } -+} -+ -+func TestURLPort(t *testing.T) { -+ tests := []struct { -+ host string // URL.Host field -+ want string -+ }{ -+ {"foo.com", ""}, -+ {"foo.com:80", "80"}, -+ {"1.2.3.4", ""}, -+ {"1.2.3.4:80", "80"}, -+ {"[1:2:3:4]", ""}, -+ {"[1:2:3:4]:80", "80"}, -+ } -+ for _, tt := range tests { -+ u := &URL{Host: tt.host} -+ got := u.Port() -+ if got != tt.want { -+ t.Errorf("Port for Host %q = %q; want %q", tt.host, got, tt.want) - } - } - } --- -2.21.0 - diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/go/go-1.12.17.ebuild b/sdk_container/src/third_party/coreos-overlay/dev-lang/go/go-1.12.17-r1.ebuild similarity index 67% rename from sdk_container/src/third_party/coreos-overlay/dev-lang/go/go-1.12.17.ebuild rename to sdk_container/src/third_party/coreos-overlay/dev-lang/go/go-1.12.17-r1.ebuild index 1c8279bc14..ad9a86ce7d 100644 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/go/go-1.12.17.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/dev-lang/go/go-1.12.17-r1.ebuild @@ -6,7 +6,3 @@ EAPI=6 inherit coreos-go-lang KEYWORDS="-* amd64 arm64" - -PATCHES=( - "${FILESDIR}/${PN}-1.12-revert-url-parsing-change.patch" -) From 5daf5eb1ca8022148e496bd260f4e540a351d281 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 1 Apr 2022 22:13:58 +0200 Subject: [PATCH 2/3] .github: Allow specifying a start number for generated patches When an action generates a couple of patches separately, then it might be a good idea to specify a numbering, so applying the patches is done in the desired order. Without that, all the generated patches would start with "0001-" prefix. --- .../src/third_party/coreos-overlay/.github/workflows/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_container/src/third_party/coreos-overlay/.github/workflows/common.sh b/sdk_container/src/third_party/coreos-overlay/.github/workflows/common.sh index c828700ef8..80b0fb5834 100644 --- a/sdk_container/src/third_party/coreos-overlay/.github/workflows/common.sh +++ b/sdk_container/src/third_party/coreos-overlay/.github/workflows/common.sh @@ -129,7 +129,7 @@ function generate_patches() { git commit -a -m "${CATEGORY_NAME}: Upgrade ${PKGNAME_DESC} ${VERSION_OLD} to ${VERSION_NEW}" # Create a patch for the main ebuilds. - git format-patch -1 HEAD + git format-patch --start-number "${START_NUMBER:-1}" -1 HEAD popd || exit } From 69ef222c6f1e896d6b3c42aeeddb22639d5cecd4 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 9 Aug 2021 17:43:17 +0200 Subject: [PATCH 3/3] .github: Update multiple golang versions Usually last two versions are supported, so make sure we keep them both updated, not only just the latest. But try to also update the newest unsupported version in case there was a window where the update happened and then new major version was released. --- .../.github/workflows/go-apply-patch.sh | 82 +++++++++++++------ .../.github/workflows/go-releases-main.yml | 29 ++++--- 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-apply-patch.sh b/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-apply-patch.sh index 208b399773..fdd17e680d 100755 --- a/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-apply-patch.sh +++ b/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-apply-patch.sh @@ -2,43 +2,77 @@ set -euo pipefail -# trim the 3rd part in the input semver, e.g. from 1.14.3 to 1.14 -VERSION_SHORT=${VERSION_NEW%.*} -UPDATE_NEEDED=1 +function join_by { + local d=${1-} f=${2-} + if shift 2; then + printf '%s' "$f" "${@/#/$d}" + fi +} + +# create a mapping between short version and new version, e.g. 1.16 -> 1.16.3 +declare -A VERSIONS +for version_new in ${VERSIONS_NEW}; do + version_new_trimmed="${version_new%.*}" + if [[ "${version_new_trimmed%.*}" = "${version_new_trimmed}" ]]; then + version_new_trimmed="${version_new}" + fi + VERSIONS["${version_new_trimmed}"]="${version_new}" +done . .github/workflows/common.sh prepare_git_repo -if ! checkout_branches "go-${VERSION_NEW}-${TARGET}"; then - UPDATE_NEEDED=0 +branch_name="go-$(join_by '-and-' ${VERSIONS_NEW})-${TARGET}" + +if ! checkout_branches "${branch_name}"; then exit 0 fi -pushd "${SDK_OUTER_SRCDIR}/third_party/coreos-overlay" >/dev/null || exit - # Parse the Manifest file for already present source files and keep the latest version in the current series # DIST go1.17.src.tar.gz ... => 1.17 # DIST go1.17.1.src.tar.gz ... => 1.17.1 -VERSION_OLD=$(sed -n "s/^DIST go\(${VERSION_SHORT}\.*[0-9]*\)\.src.*/\1/p" dev-lang/go/Manifest | sort -ruV | head -n1) -if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then - echo "already the latest Go, nothing to do" - UPDATE_NEEDED=0 +declare -a UPDATED_VERSIONS_OLD UPDATED_VERSIONS_NEW +any_different=0 +START_NUMBER=1 +for version_short in "${!VERSIONS[@]}"; do + pushd "${SDK_OUTER_SRCDIR}/third_party/coreos-overlay" >/dev/null || exit + VERSION_NEW="${VERSIONS["${version_short}"]}" + VERSION_OLD=$(sed -n "s/^DIST go\(${version_short}\(\.*[0-9]*\)\?\)\.src.*/\1/p" dev-lang/go/Manifest | sort -ruV | head -n1) + if [[ -z "${VERSION_OLD}" ]]; then + echo "${version_short} is not packaged, skipping" + popd >/dev/null || exit + continue + fi + if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "${version_short} is already at the latest (${VERSION_NEW}), skipping" + popd >/dev/null || exit + continue + fi + UPDATED_VERSIONS_OLD+=("${VERSION_OLD}") + UPDATED_VERSIONS_NEW+=("${VERSION_NEW}") + + any_different=1 + EBUILD_FILENAME=$(get_ebuild_filename "dev-lang" "go" "${VERSION_OLD}") + git mv "${EBUILD_FILENAME}" "dev-lang/go/go-${VERSION_NEW}.ebuild" + + popd >/dev/null || exit + + generate_patches dev-lang go Go + ((START_NUMBER++)) +done + +if [[ $any_different -eq 0 ]]; then + echo "go packages were already at the latest versions, nothing to do" exit 0 fi -EBUILD_FILENAME=$(get_ebuild_filename "dev-lang" "go" "${VERSION_OLD}") -git mv "${EBUILD_FILENAME}" "dev-lang/go/go-${VERSION_NEW}.ebuild" - -popd >/dev/null || exit - -URL="https://go.googlesource.com/go/+/refs/tags/go${VERSION_NEW}" - -generate_update_changelog 'Go' "${VERSION_NEW}" "${URL}" 'golang' - -generate_patches dev-lang go Go - apply_patches -echo ::set-output name=VERSION_OLD::"${VERSION_OLD}" -echo ::set-output name=UPDATE_NEEDED::"${UPDATE_NEEDED}" +vo_gh="$(join_by ' and ' "${UPDATED_VERSIONS_OLD[@]}")" +vn_gh="$(join_by ' and ' "${UPDATED_VERSIONS_NEW[@]}")" + +echo ::set-output name=VERSIONS_OLD::"${vo_gh}" +echo ::set-output name=VERSIONS_NEW::"${vn_gh}" +echo ::set-output name=BRANCH_NAME::"${branch_name}" +echo ::set-output name=UPDATE_NEEDED::"1" diff --git a/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-releases-main.yml b/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-releases-main.yml index ee8a5fa3cc..a85de0618b 100644 --- a/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-releases-main.yml +++ b/sdk_container/src/third_party/coreos-overlay/.github/workflows/go-releases-main.yml @@ -5,21 +5,24 @@ on: workflow_dispatch: jobs: - get-go-release: + get-go-releases: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Fetch latest Go release - id: fetch-latest-release + - name: Fetch latest Go releases + id: fetch-latest-releases env: - GO_VERSION: "1.17" + GO_VERSIONS: "1.16 1.17 1.18" run: | git clone --depth=1 --no-checkout https://github.com/golang/go - versionMain=$(git -C go ls-remote --tags origin | cut -f2 | sed -n "/refs\/tags\/go${GO_VERSION}\.[0-9]*$/s/^refs\/tags\/go//p" | egrep -v -e '(beta|rc)' | sort -ruV | head -1) + versionsMain=() + for goversion in ${GO_VERSIONS}; do + versionsMain+=($(git -C go ls-remote --tags origin | cut -f2 | sed -n "/refs\/tags\/go${goversion}\(\.[0-9]*\)\?$/s/^refs\/tags\/go//p" | egrep -v -e '(beta|rc)' | sort -ruV | head -1)) + done rm -rf go - echo ::set-output name=VERSION_MAIN::$(echo ${versionMain}) + echo ::set-output name=VERSIONS_MAIN::$(echo ${versionsMain[*]}) echo ::set-output name=BASE_BRANCH_MAIN::main - name: Set up Flatcar SDK id: setup-flatcar-sdk @@ -28,20 +31,20 @@ jobs: id: apply-patch-main env: TARGET: main - BASE_BRANCH: ${{ steps.fetch-latest-release.outputs.BASE_BRANCH_MAIN }} + BASE_BRANCH: ${{ steps.fetch-latest-releases.outputs.BASE_BRANCH_MAIN }} PATH: ${{ steps.setup-flatcar-sdk.outputs.path }} - VERSION_NEW: ${{ steps.fetch-latest-release.outputs.VERSION_MAIN }} + VERSIONS_NEW: ${{ steps.fetch-latest-releases.outputs.VERSIONS_MAIN }} run: .github/workflows/go-apply-patch.sh - name: Create pull request for main uses: peter-evans/create-pull-request@v3 if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 with: token: ${{ secrets.GITHUB_TOKEN }} - base: ${{ steps.fetch-latest-release.outputs.BASE_BRANCH_MAIN }} - branch: go-${{ steps.fetch-latest-release.outputs.VERSION_MAIN }}-main + base: ${{ steps.fetch-latest-releases.outputs.BASE_BRANCH_MAIN }} + branch: ${{ steps.apply-patch-main.outputs.BRANCH_NAME }} author: Flatcar Buildbot committer: Flatcar Buildbot - title: Upgrade Go in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.fetch-latest-release.outputs.VERSION_MAIN }} - commit-message: Upgrade Go in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.fetch-latest-release.outputs.VERSION_MAIN }} - body: Upgrade Go in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.fetch-latest-release.outputs.VERSION_MAIN }} + title: Upgrade Go from ${{ steps.apply-patch-main.outputs.VERSIONS_OLD }} to ${{ steps.apply-patch-main.outputs.VERSIONS_NEW }} + commit-message: Upgrade Go from ${{ steps.apply-patch-main.outputs.VERSIONS_OLD }} to ${{ steps.apply-patch-main.outputs.VERSIONS_NEW }} + body: Upgrade Go from ${{ steps.apply-patch-main.outputs.VERSIONS_OLD }} to ${{ steps.apply-patch-main.outputs.VERSIONS_NEW }} labels: main