net-misc/curl: Sync with Gentoo

It's from Gentoo commit f5be667ab95a32f51f26a4c49c08596da3b0a076.
This commit is contained in:
Flatcar Buildbot 2024-04-22 07:15:30 +00:00 committed by Krzesimir Nowak
parent 51de874281
commit c8bfcd947f
3 changed files with 578 additions and 0 deletions

View File

@ -0,0 +1,368 @@
# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
EAPI=8
VERIFY_SIG_OPENPGP_KEY_PATH=/usr/share/openpgp-keys/danielstenberg.asc
inherit autotools multilib-minimal multiprocessing prefix toolchain-funcs verify-sig
DESCRIPTION="A Client that groks URLs"
HOMEPAGE="https://curl.se/"
if [[ ${PV} == 9999 ]]; then
inherit git-r3
EGIT_REPO_URI="https://github.com/curl/curl.git"
else
SRC_URI="
https://curl.se/download/${P}.tar.xz
verify-sig? ( https://curl.se/download/${P}.tar.xz.asc )
"
KEYWORDS="~alpha amd64 arm ~arm64 ~hppa ~ia64 ~loong ~m68k ~mips ppc ppc64 ~riscv ~s390 sparc x86 ~amd64-linux ~x86-linux ~arm64-macos ~ppc-macos ~x64-macos ~x64-solaris"
fi
LICENSE="BSD curl ISC test? ( BSD-4 )"
SLOT="0"
IUSE="+adns +alt-svc brotli +ftp gnutls gopher +hsts +http2 idn +imap kerberos ldap mbedtls nghttp3 +openssl +pop3"
IUSE+=" +psl +progress-meter rtmp rustls samba +smtp ssh ssl sslv3 static-libs test telnet +tftp websockets zstd"
# These select the default SSL implementation
IUSE+=" curl_ssl_gnutls curl_ssl_mbedtls +curl_ssl_openssl curl_ssl_rustls"
RESTRICT="!test? ( test )"
# Only one default ssl provider can be enabled
# The default ssl provider needs its USE satisfied
# nghttp3 = https://bugs.gentoo.org/912029
REQUIRED_USE="
ssl? (
^^ (
curl_ssl_gnutls
curl_ssl_mbedtls
curl_ssl_openssl
curl_ssl_rustls
)
)
curl_ssl_gnutls? ( gnutls )
curl_ssl_mbedtls? ( mbedtls )
curl_ssl_openssl? ( openssl )
curl_ssl_rustls? ( rustls )
nghttp3? (
!openssl
alt-svc )
"
# cURL's docs and CI/CD are great resources for confirming supported versions
# particulary for fast-moving targets like HTTP/2 and TCP/2 e.g.:
# - https://github.com/curl/curl/blob/master/docs/INTERNALS.md (core dependencies + minimum versions)
# - https://github.com/curl/curl/blob/master/docs/HTTP3.md (example of a feature that moves quickly)
# - https://github.com/curl/curl/blob/master/.github/workflows/quiche-linux.yml (CI/CD for TCP/2)
# However 'supported' vs 'works' are two entirely different things; be sane but
# don't be afraid to require a later version.
RDEPEND="
>=sys-libs/zlib-1.1.4[${MULTILIB_USEDEP}]
adns? ( net-dns/c-ares:=[${MULTILIB_USEDEP}] )
brotli? ( app-arch/brotli:=[${MULTILIB_USEDEP}] )
http2? ( >=net-libs/nghttp2-1.12.0:=[${MULTILIB_USEDEP}] )
idn? ( net-dns/libidn2:=[static-libs?,${MULTILIB_USEDEP}] )
kerberos? ( >=virtual/krb5-0-r1[${MULTILIB_USEDEP}] )
ldap? ( >=net-nds/openldap-2.0.0:=[static-libs?,${MULTILIB_USEDEP}] )
nghttp3? (
>=net-libs/nghttp3-0.15.0[${MULTILIB_USEDEP}]
>=net-libs/ngtcp2-0.19.1[gnutls,ssl,-openssl,${MULTILIB_USEDEP}]
)
psl? ( net-libs/libpsl[${MULTILIB_USEDEP}] )
rtmp? ( media-video/rtmpdump[${MULTILIB_USEDEP}] )
ssh? ( >=net-libs/libssh2-1.0.0[${MULTILIB_USEDEP}] )
ssl? (
gnutls? (
app-misc/ca-certificates
>=net-libs/gnutls-3.1.10:=[static-libs?,${MULTILIB_USEDEP}]
dev-libs/nettle:=[${MULTILIB_USEDEP}]
)
mbedtls? (
app-misc/ca-certificates
net-libs/mbedtls:=[${MULTILIB_USEDEP}]
)
openssl? (
>=dev-libs/openssl-0.9.7:=[sslv3(-)=,static-libs?,${MULTILIB_USEDEP}]
)
rustls? ( >=net-libs/rustls-ffi-0.12.1:=[${MULTILIB_USEDEP}]
<net-libs/rustls-ffi-0.13.0:=[${MULTILIB_USEDEP}]
)
)
zstd? ( app-arch/zstd:=[${MULTILIB_USEDEP}] )
"
DEPEND="${RDEPEND}"
BDEPEND="
dev-lang/perl
virtual/pkgconfig
test? (
sys-apps/diffutils
http2? ( >=net-libs/nghttp2-1.15.0:=[utils,${MULTILIB_USEDEP}] )
nghttp3? ( net-libs/nghttp2:=[utils,${MULTILIB_USEDEP}] )
)
verify-sig? ( sec-keys/openpgp-keys-danielstenberg )
"
DOCS=( CHANGES README docs/{FEATURES.md,INTERNALS.md,FAQ,BUGS.md,CONTRIBUTE.md} )
MULTILIB_WRAPPED_HEADERS=(
/usr/include/curl/curlbuild.h
)
MULTILIB_CHOST_TOOLS=(
/usr/bin/curl-config
)
QA_CONFIG_IMPL_DECL_SKIP=(
__builtin_available
closesocket
CloseSocket
getpass_r
ioctlsocket
IoctlSocket
mach_absolute_time
setmode
_fseeki64
)
PATCHES=(
"${FILESDIR}"/${PN}-prefix.patch
"${FILESDIR}"/${PN}-respect-cflags-3.patch
"${FILESDIR}"/${PN}-8.7.1-rustls-fixes.patch
"${FILESDIR}"/${P}-chunked-post.patch
"${FILESDIR}"/${P}-fix-compress-option.patch
)
src_prepare() {
default
eprefixify curl-config.in
eautoreconf
}
multilib_src_configure() {
# We make use of the fact that later flags override earlier ones
# So start with all ssl providers off until proven otherwise
# TODO: in the future, we may want to add wolfssl (https://www.wolfssl.com/)
local myconf=()
myconf+=( --without-ca-fallback --with-ca-bundle="${EPREFIX}"/etc/ssl/certs/ca-certificates.crt )
if use ssl; then
myconf+=( --without-gnutls --without-mbedtls --without-rustls )
if use gnutls; then
multilib_is_native_abi && einfo "SSL provided by gnutls"
myconf+=( --with-gnutls )
fi
if use mbedtls; then
multilib_is_native_abi && einfo "SSL provided by mbedtls"
myconf+=( --with-mbedtls )
fi
if use openssl; then
multilib_is_native_abi && einfo "SSL provided by openssl"
myconf+=( --with-ssl --with-ca-path="${EPREFIX}"/etc/ssl/certs )
fi
if use rustls; then
multilib_is_native_abi && einfo "SSL provided by rustls"
myconf+=( --with-rustls )
fi
if use curl_ssl_gnutls; then
multilib_is_native_abi && einfo "Default SSL provided by gnutls"
myconf+=( --with-default-ssl-backend=gnutls )
elif use curl_ssl_mbedtls; then
multilib_is_native_abi && einfo "Default SSL provided by mbedtls"
myconf+=( --with-default-ssl-backend=mbedtls )
elif use curl_ssl_openssl; then
multilib_is_native_abi && einfo "Default SSL provided by openssl"
myconf+=( --with-default-ssl-backend=openssl )
elif use curl_ssl_rustls; then
multilib_is_native_abi && einfo "Default SSL provided by rustls"
myconf+=( --with-default-ssl-backend=rustls )
else
eerror "We can't be here because of REQUIRED_USE."
die "Please file a bug, hit impossible condition w/ USE=ssl handling."
fi
else
myconf+=( --without-ssl )
einfo "SSL disabled"
fi
# These configuration options are organized alphabetically
# within each category. This should make it easier if we
# ever decide to make any of them contingent on USE flags:
# 1) protocols first. To see them all do
# 'grep SUPPORT_PROTOCOLS configure.ac'
# 2) --enable/disable options second.
# 'grep -- --enable configure | grep Check | awk '{ print $4 }' | sort
# 3) --with/without options third.
# grep -- --with configure | grep Check | awk '{ print $4 }' | sort
myconf+=(
$(use_enable alt-svc)
--enable-basic-auth
--enable-bearer-auth
--enable-digest-auth
--enable-kerberos-auth
--enable-negotiate-auth
--enable-aws
--enable-dict
--disable-ech
--enable-file
$(use_enable ftp)
$(use_enable gopher)
$(use_enable hsts)
--enable-http
$(use_enable imap)
$(use_enable ldap)
$(use_enable ldap ldaps)
--enable-ntlm
--disable-ntlm-wb
$(use_enable pop3)
--enable-rt
--enable-rtsp
$(use_enable samba smb)
$(use_with ssh libssh2)
$(use_enable smtp)
$(use_enable telnet)
$(use_enable tftp)
--enable-tls-srp
$(use_enable adns ares)
--enable-cookies
--enable-dateparse
--enable-dnsshuffle
--enable-doh
--enable-symbol-hiding
--enable-http-auth
--enable-ipv6
--enable-largefile
--enable-manual
--enable-mime
--enable-netrc
$(use_enable progress-meter)
--enable-proxy
--enable-socketpair
--disable-sspi
$(use_enable static-libs static)
--enable-pthreads
--enable-threaded-resolver
--disable-versioned-symbols
--without-amissl
--without-bearssl
$(use_with brotli)
--with-fish-functions-dir="${EPREFIX}"/usr/share/fish/vendor_completions.d
$(use_with http2 nghttp2)
--without-hyper
$(use_with idn libidn2)
$(use_with kerberos gssapi "${EPREFIX}"/usr)
--without-libgsasl
$(use_with psl libpsl)
--without-msh3
$(use_with nghttp3)
$(use_with nghttp3 ngtcp2)
--without-quiche
$(use_with rtmp librtmp)
--without-schannel
--without-secure-transport
--without-test-caddy
--without-test-httpd
--without-test-nghttpx
$(use_enable websockets)
--without-winidn
--without-wolfssl
--with-zlib
$(use_with zstd)
--with-zsh-functions-dir="${EPREFIX}"/usr/share/zsh/site-functions
)
if use test && multilib_is_native_abi && ( use http2 || use nghttp3 ); then
myconf+=(
--with-test-nghttpx="${BROOT}/usr/bin/nghttpx"
)
fi
if [[ ${CHOST} == *mingw* ]] ; then
myconf+=(
--disable-pthreads
)
fi
ECONF_SOURCE="${S}" econf "${myconf[@]}"
if ! multilib_is_native_abi; then
# Avoid building the client (we just want libcurl for multilib)
sed -i -e '/SUBDIRS/s:src::' Makefile || die
sed -i -e '/SUBDIRS/s:scripts::' Makefile || die
fi
# Fix up the pkg-config file to be more robust.
# https://github.com/curl/curl/issues/864
local priv=() libs=()
# We always enable zlib.
libs+=( "-lz" )
priv+=( "zlib" )
if use http2; then
libs+=( "-lnghttp2" )
priv+=( "libnghttp2" )
fi
if use nghttp3; then
libs+=( "-lnghttp3" "-lngtcp2" )
priv+=( "libnghttp3" "libngtcp2" )
fi
if use ssl && use curl_ssl_openssl; then
libs+=( "-lssl" "-lcrypto" )
priv+=( "openssl" )
fi
grep -q Requires.private libcurl.pc && die "need to update ebuild"
libs=$(printf '|%s' "${libs[@]}")
sed -i -r \
-e "/^Libs.private/s:(${libs#|})( |$)::g" \
libcurl.pc || die
echo "Requires.private: ${priv[*]}" >> libcurl.pc || die
}
multilib_src_compile() {
default
if multilib_is_native_abi; then
# Shell completions
! tc-is-cross-compiler && emake -C scripts
fi
}
# There is also a pytest harness that tests for bugs in some very specific
# situations; we can rely on upstream for this rather than adding additional test deps.
multilib_src_test() {
# See https://github.com/curl/curl/blob/master/tests/runtests.pl#L5721
# -n: no valgrind (unreliable in sandbox and doesn't work correctly on all arches)
# -v: verbose
# -a: keep going on failure (so we see everything which breaks, not just 1st test)
# -k: keep test files after completion
# -am: automake style TAP output
# -p: print logs if test fails
# Note: if needed, we can skip specific tests. See e.g. Fedora's packaging
# or just read https://github.com/curl/curl/tree/master/tests#run.
# Note: we don't run the testsuite for cross-compilation.
# Upstream recommend 7*nproc as a starting point for parallel tests, but
# this ends up breaking when nproc is huge (like -j80).
# The network sandbox causes tests 241 and 1083 to fail; these are typically skipped
# as most gentoo users don't have an 'ip6-localhost'
multilib_is_native_abi && emake test TFLAGS="-n -v -a -k -am -p -j$((2*$(makeopts_jobs))) !241 !1083"
}
multilib_src_install() {
emake DESTDIR="${D}" install
if multilib_is_native_abi; then
# Shell completions
! tc-is-cross-compiler && emake -C scripts DESTDIR="${D}" install
fi
}
multilib_src_install_all() {
einstalldocs
find "${ED}" -type f -name '*.la' -delete || die
rm -rf "${ED}"/etc/ || die
}

View File

@ -0,0 +1,57 @@
https://github.com/curl/curl/commit/721941aadf4adf4f6aeb3f4c0ab489bb89610c36
From: Stefan Eissing <stefan@eissing.org>
Date: Mon, 1 Apr 2024 15:41:18 +0200
Subject: [PATCH] http: with chunked POST forced, disable length check on read
callback
- when an application forces HTTP/1.1 chunked transfer encoding
by setting the corresponding header and instructs curl to use
the CURLOPT_READFUNCTION, disregard any POST length information.
- this establishes backward compatibility with previous curl versions
Applications are encouraged to not force "chunked", but rather
set length information for a POST. By setting -1, curl will
auto-select chunked on HTTP/1.1 and work properly on other HTTP
versions.
Reported-by: Jeff King
Fixes #13229
Closes #13257
--- a/lib/http.c
+++ b/lib/http.c
@@ -2046,8 +2046,19 @@ static CURLcode set_reader(struct Curl_easy *data, Curl_HttpReq httpreq)
else
result = Curl_creader_set_null(data);
}
- else { /* we read the bytes from the callback */
- result = Curl_creader_set_fread(data, postsize);
+ else {
+ /* we read the bytes from the callback. In case "chunked" encoding
+ * is forced by the application, we disregard `postsize`. This is
+ * a backward compatibility decision to earlier versions where
+ * chunking disregarded this. See issue #13229. */
+ bool chunked = FALSE;
+ char *ptr = Curl_checkheaders(data, STRCONST("Transfer-Encoding"));
+ if(ptr) {
+ /* Some kind of TE is requested, check if 'chunked' is chosen */
+ chunked = Curl_compareheader(ptr, STRCONST("Transfer-Encoding:"),
+ STRCONST("chunked"));
+ }
+ result = Curl_creader_set_fread(data, chunked? -1 : postsize);
}
return result;
@@ -2115,6 +2126,13 @@ CURLcode Curl_http_req_set_reader(struct Curl_easy *data,
data->req.upload_chunky =
Curl_compareheader(ptr,
STRCONST("Transfer-Encoding:"), STRCONST("chunked"));
+ if(data->req.upload_chunky &&
+ Curl_use_http_1_1plus(data, data->conn) &&
+ (data->conn->httpversion >= 20)) {
+ infof(data, "suppressing chunked transfer encoding on connection "
+ "using HTTP version 2 or higher");
+ data->req.upload_chunky = FALSE;
+ }
}
else {
curl_off_t req_clen = Curl_creader_total_length(data);

View File

@ -0,0 +1,153 @@
https://github.com/curl/curl/commit/b30d694a027eb771c02a3db0dee0ca03ccab7377
From: Stefan Eissing <stefan@eissing.org>
Date: Thu, 28 Mar 2024 11:08:15 +0100
Subject: [PATCH] content_encoding: brotli and others, pass through 0-length
writes
- curl's transfer handling may write 0-length chunks at the end of the
download with an EOS flag. (HTTP/2 does this commonly)
- content encoders need to pass-through such a write and not count this
as error in case they are finished decoding
Fixes #13209
Fixes #13212
Closes #13219
--- a/lib/content_encoding.c
+++ b/lib/content_encoding.c
@@ -300,7 +300,7 @@ static CURLcode deflate_do_write(struct Curl_easy *data,
struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
/* Set the compressed input when this function is called */
@@ -457,7 +457,7 @@ static CURLcode gzip_do_write(struct Curl_easy *data,
struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(zp->zlib_init == ZLIB_INIT_GZIP) {
@@ -669,7 +669,7 @@ static CURLcode brotli_do_write(struct Curl_easy *data,
CURLcode result = CURLE_OK;
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!bp->br)
@@ -762,7 +762,7 @@ static CURLcode zstd_do_write(struct Curl_easy *data,
ZSTD_outBuffer out;
size_t errorCode;
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!zp->decomp) {
@@ -916,7 +916,7 @@ static CURLcode error_do_write(struct Curl_easy *data,
(void) buf;
(void) nbytes;
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
failf(data, "Unrecognized content encoding type. "
--- a/tests/http/test_02_download.py
+++ b/tests/http/test_02_download.py
@@ -394,6 +394,19 @@ def test_02_27_paused_no_cl(self, env: Env, httpd, nghttpx, repeat):
r = client.run(args=[url])
r.check_exit_code(0)
+ @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3'])
+ def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, repeat, proto):
+ if proto == 'h3' and not env.have_h3():
+ pytest.skip("h3 not supported")
+ count = 1
+ urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]'
+ curl = CurlClient(env=env)
+ r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
+ '--compressed'
+ ])
+ r.check_exit_code(code=0)
+ r.check_response(count=count, http_status=200)
+
def check_downloads(self, client, srcfile: str, count: int,
complete: bool = True):
for i in range(count):
--- a/tests/http/testenv/env.py
+++ b/tests/http/testenv/env.py
@@ -129,10 +129,11 @@ def __init__(self):
self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs')
self.tld = 'http.curl.se'
self.domain1 = f"one.{self.tld}"
+ self.domain1brotli = f"brotli.one.{self.tld}"
self.domain2 = f"two.{self.tld}"
self.proxy_domain = f"proxy.{self.tld}"
self.cert_specs = [
- CertificateSpec(domains=[self.domain1, 'localhost'], key_type='rsa2048'),
+ CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost'], key_type='rsa2048'),
CertificateSpec(domains=[self.domain2], key_type='rsa2048'),
CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'),
CertificateSpec(name="clientsX", sub_specs=[
@@ -376,6 +377,10 @@ def htdocs_dir(self) -> str:
def domain1(self) -> str:
return self.CONFIG.domain1
+ @property
+ def domain1brotli(self) -> str:
+ return self.CONFIG.domain1brotli
+
@property
def domain2(self) -> str:
return self.CONFIG.domain2
--- a/tests/http/testenv/httpd.py
+++ b/tests/http/testenv/httpd.py
@@ -50,6 +50,7 @@ class Httpd:
'alias', 'env', 'filter', 'headers', 'mime', 'setenvif',
'socache_shmcb',
'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect',
+ 'brotli',
'mpm_event',
]
COMMON_MODULES_DIRS = [
@@ -203,6 +204,7 @@ def _mkpath(self, path):
def _write_config(self):
domain1 = self.env.domain1
+ domain1brotli = self.env.domain1brotli
creds1 = self.env.get_credentials(domain1)
domain2 = self.env.domain2
creds2 = self.env.get_credentials(domain2)
@@ -285,6 +287,24 @@ def _write_config(self):
f'</VirtualHost>',
f'',
])
+ # Alternate to domain1 with BROTLI compression
+ conf.extend([ # https host for domain1, h1 + h2
+ f'<VirtualHost *:{self.env.https_port}>',
+ f' ServerName {domain1brotli}',
+ f' Protocols h2 http/1.1',
+ f' SSLEngine on',
+ f' SSLCertificateFile {creds1.cert_file}',
+ f' SSLCertificateKeyFile {creds1.pkey_file}',
+ f' DocumentRoot "{self._docs_dir}"',
+ f' SetOutputFilter BROTLI_COMPRESS',
+ ])
+ conf.extend(self._curltest_conf(domain1))
+ if domain1 in self._extra_configs:
+ conf.extend(self._extra_configs[domain1])
+ conf.extend([
+ f'</VirtualHost>',
+ f'',
+ ])
conf.extend([ # https host for domain2, no h2
f'<VirtualHost *:{self.env.https_port}>',
f' ServerName {domain2}',