From e1465c1e466d1fe1111731a8444d2ede6ceb0923 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Sun, 9 May 2021 14:41:41 +0200 Subject: [PATCH] REGTESTS: disable inter-thread idle connection sharing on sensitive tests Some regtests involve multiple requests from multiple clients, which can be dispatched as multiple requests to a server. It turns out that the idle connection sharing works so well that very quickly few connections are used, and regularly some of the remaining idle server connections time out at the moment they were going to be reused, causing those random "HTTP header incomplete" traces in the logs that make them fail often. In the end this is only an artefact of the test environment. And indeed, some tests like normalize-uri which perform a lot of reuse fail very often, about 20-30% of the times in the CI, and 100% of the time in local when running 1000 tests in a row. Others like ubase64, sample_fetches or vary_* fail less often but still a lot in tests. This patch addresses this by adding "tune.idle-pool.shared off" to all tests which have at least twice as many requests as clients. It proves very effective as no single error happens on normalize-uri anymore after 10000 tests. Also 100 full runs of all tests yield no error anymore. One test is tricky, http_abortonclose, it used to fail ~10 times per 1000 runs and with this workaround still fails once every 1000 runs. But the test is complex and there's a warning in it mentioning a possible issue when run in parallel due to a port reuse. --- reg-tests/cache/caching_rules.vtc | 6 ++++++ reg-tests/cache/expires.vtc | 6 ++++++ reg-tests/cache/if-modified-since.vtc | 6 ++++++ reg-tests/cache/if-none-match.vtc | 6 ++++++ reg-tests/cache/sample_fetches.vtc | 6 ++++++ reg-tests/cache/vary.vtc | 6 ++++++ reg-tests/cache/vary_accept_encoding.vtc | 6 ++++++ reg-tests/compression/etags_conversion.vtc | 6 ++++++ reg-tests/compression/vary.vtc | 6 ++++++ reg-tests/converter/json_query.vtc | 6 ++++++ reg-tests/converter/secure_memcmp.vtc | 6 ++++++ reg-tests/http-messaging/h1_to_h1.vtc | 6 ++++++ reg-tests/http-messaging/h2_to_h1.vtc | 6 ++++++ reg-tests/http-messaging/http_abortonclose.vtc | 6 ++++++ reg-tests/http-messaging/http_bodyless_response.vtc | 6 ++++++ reg-tests/http-rules/except-forwardfor-originalto.vtc | 6 ++++++ reg-tests/http-rules/http_return.vtc | 6 ++++++ reg-tests/http-rules/normalize_uri.vtc | 4 ++++ reg-tests/lua/set_var.vtc | 6 ++++++ reg-tests/lua/txn_get_priv-thread.vtc | 5 +++++ reg-tests/sample_fetches/ubase64.vtc | 6 ++++++ reg-tests/webstats/webstats-scope-and-post-change.vtc | 5 +++++ 22 files changed, 128 insertions(+) diff --git a/reg-tests/cache/caching_rules.vtc b/reg-tests/cache/caching_rules.vtc index c20a720e3..114b2fd91 100644 --- a/reg-tests/cache/caching_rules.vtc +++ b/reg-tests/cache/caching_rules.vtc @@ -78,6 +78,12 @@ server s2 { } -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/cache/expires.vtc b/reg-tests/cache/expires.vtc index 51c748748..037f09fd0 100644 --- a/reg-tests/cache/expires.vtc +++ b/reg-tests/cache/expires.vtc @@ -40,6 +40,12 @@ server s3 { } -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/cache/if-modified-since.vtc b/reg-tests/cache/if-modified-since.vtc index e491e4660..8ae1cce5d 100644 --- a/reg-tests/cache/if-modified-since.vtc +++ b/reg-tests/cache/if-modified-since.vtc @@ -40,6 +40,12 @@ server s1 { } -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/cache/if-none-match.vtc b/reg-tests/cache/if-none-match.vtc index c5391c90f..ba3336a08 100644 --- a/reg-tests/cache/if-none-match.vtc +++ b/reg-tests/cache/if-none-match.vtc @@ -25,6 +25,12 @@ server s1 { } -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/cache/sample_fetches.vtc b/reg-tests/cache/sample_fetches.vtc index 73e6e1bf6..973b3ad61 100644 --- a/reg-tests/cache/sample_fetches.vtc +++ b/reg-tests/cache/sample_fetches.vtc @@ -44,6 +44,12 @@ server s3 { } -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/cache/vary.vtc b/reg-tests/cache/vary.vtc index f3425d002..0b5d13941 100644 --- a/reg-tests/cache/vary.vtc +++ b/reg-tests/cache/vary.vtc @@ -127,6 +127,12 @@ server s2 { } -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/cache/vary_accept_encoding.vtc b/reg-tests/cache/vary_accept_encoding.vtc index 185ad0502..852ac293e 100644 --- a/reg-tests/cache/vary_accept_encoding.vtc +++ b/reg-tests/cache/vary_accept_encoding.vtc @@ -92,6 +92,12 @@ server s1 { haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/compression/etags_conversion.vtc b/reg-tests/compression/etags_conversion.vtc index 0cae67959..48e7955fc 100644 --- a/reg-tests/compression/etags_conversion.vtc +++ b/reg-tests/compression/etags_conversion.vtc @@ -106,6 +106,12 @@ server s1 { haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/compression/vary.vtc b/reg-tests/compression/vary.vtc index 0a060e4bc..a905f9c94 100644 --- a/reg-tests/compression/vary.vtc +++ b/reg-tests/compression/vary.vtc @@ -61,6 +61,12 @@ server s1 { haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/converter/json_query.vtc b/reg-tests/converter/json_query.vtc index ade7b4ccb..b42094200 100644 --- a/reg-tests/converter/json_query.vtc +++ b/reg-tests/converter/json_query.vtc @@ -9,6 +9,12 @@ server s1 { } -repeat 8 -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http timeout connect 1s diff --git a/reg-tests/converter/secure_memcmp.vtc b/reg-tests/converter/secure_memcmp.vtc index f9341f942..b16e02195 100644 --- a/reg-tests/converter/secure_memcmp.vtc +++ b/reg-tests/converter/secure_memcmp.vtc @@ -16,6 +16,12 @@ server s2 { } -repeat 7 -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http timeout connect 1s diff --git a/reg-tests/http-messaging/h1_to_h1.vtc b/reg-tests/http-messaging/h1_to_h1.vtc index 5b02f1724..c7d00858e 100644 --- a/reg-tests/http-messaging/h1_to_h1.vtc +++ b/reg-tests/http-messaging/h1_to_h1.vtc @@ -115,6 +115,12 @@ server s1 { } -repeat 3 -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/http-messaging/h2_to_h1.vtc b/reg-tests/http-messaging/h2_to_h1.vtc index 481aded12..0d2b1e5f2 100644 --- a/reg-tests/http-messaging/h2_to_h1.vtc +++ b/reg-tests/http-messaging/h2_to_h1.vtc @@ -34,6 +34,12 @@ server s1 { } -repeat 2 -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults #log stdout format raw daemon mode http diff --git a/reg-tests/http-messaging/http_abortonclose.vtc b/reg-tests/http-messaging/http_abortonclose.vtc index c07498257..ae0d237ae 100644 --- a/reg-tests/http-messaging/http_abortonclose.vtc +++ b/reg-tests/http-messaging/http_abortonclose.vtc @@ -46,6 +46,12 @@ syslog S -level info { } -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/http-messaging/http_bodyless_response.vtc b/reg-tests/http-messaging/http_bodyless_response.vtc index f45a7d61b..6f36545f2 100644 --- a/reg-tests/http-messaging/http_bodyless_response.vtc +++ b/reg-tests/http-messaging/http_bodyless_response.vtc @@ -30,6 +30,12 @@ server s1 { } -repeat 2 -start haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http ${no-htx} option http-use-htx diff --git a/reg-tests/http-rules/except-forwardfor-originalto.vtc b/reg-tests/http-rules/except-forwardfor-originalto.vtc index 942c4e2cb..bb346dda7 100644 --- a/reg-tests/http-rules/except-forwardfor-originalto.vtc +++ b/reg-tests/http-rules/except-forwardfor-originalto.vtc @@ -7,6 +7,12 @@ varnishtest "Test IPv4/IPv6 except param for the forwardfor and originalto optio feature ignore_unknown_macro haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http timeout connect 1s diff --git a/reg-tests/http-rules/http_return.vtc b/reg-tests/http-rules/http_return.vtc index a50f1fec8..818902804 100644 --- a/reg-tests/http-rules/http_return.vtc +++ b/reg-tests/http-rules/http_return.vtc @@ -6,6 +6,12 @@ varnishtest "Test the HTTP return action" feature ignore_unknown_macro haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http timeout connect 1s diff --git a/reg-tests/http-rules/normalize_uri.vtc b/reg-tests/http-rules/normalize_uri.vtc index 47032d96c..42c4c428b 100644 --- a/reg-tests/http-rules/normalize_uri.vtc +++ b/reg-tests/http-rules/normalize_uri.vtc @@ -12,6 +12,10 @@ server s1 { haproxy h1 -conf { global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off expose-experimental-directives defaults diff --git a/reg-tests/lua/set_var.vtc b/reg-tests/lua/set_var.vtc index fc21b4133..af4cbf1eb 100644 --- a/reg-tests/lua/set_var.vtc +++ b/reg-tests/lua/set_var.vtc @@ -5,6 +5,12 @@ varnishtest "Lua: set_var" feature ignore_unknown_macro haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + global lua-load ${testdir}/set_var.lua diff --git a/reg-tests/lua/txn_get_priv-thread.vtc b/reg-tests/lua/txn_get_priv-thread.vtc index 77e333dcf..6f5d63269 100644 --- a/reg-tests/lua/txn_get_priv-thread.vtc +++ b/reg-tests/lua/txn_get_priv-thread.vtc @@ -7,6 +7,11 @@ feature ignore_unknown_macro haproxy h1 -conf { global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + lua-load-per-thread ${testdir}/txn_get_priv.lua lua-load-per-thread ${testdir}/txn_get_priv-print_r.lua diff --git a/reg-tests/sample_fetches/ubase64.vtc b/reg-tests/sample_fetches/ubase64.vtc index d6973e3b7..0122bade8 100644 --- a/reg-tests/sample_fetches/ubase64.vtc +++ b/reg-tests/sample_fetches/ubase64.vtc @@ -5,6 +5,12 @@ varnishtest "ub64dec sample fetche Test" feature ignore_unknown_macro haproxy h1 -conf { + global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + defaults mode http timeout connect 1s diff --git a/reg-tests/webstats/webstats-scope-and-post-change.vtc b/reg-tests/webstats/webstats-scope-and-post-change.vtc index a77483b57..fe8b33333 100644 --- a/reg-tests/webstats/webstats-scope-and-post-change.vtc +++ b/reg-tests/webstats/webstats-scope-and-post-change.vtc @@ -8,6 +8,11 @@ server s1 { haproxy h1 -conf { global + # WT: limit false-positives causing "HTTP header incomplete" due to + # idle server connections being randomly used and randomly expiring + # under us. + tune.idle-pool.shared off + stats socket /tmp/haproxy.socket level admin defaults