From e41a4c361aaa59d96716e7ab6199011b57c106bb Mon Sep 17 00:00:00 2001 From: David Cooper Date: Wed, 22 Sep 2021 11:01:33 -0400 Subject: [PATCH 1/5] WIP: Improve check for client authentication As noted in #1709, some servers will only request client authentication if a specific URL is requested. This commit modifies the check for client authentication, in the case that a $URL_PATH is provided, by having testssl.sh perform a GET request on the URL provided on the command line. --- testssl.sh | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/testssl.sh b/testssl.sh index d4f7324..facf0af 100755 --- a/testssl.sh +++ b/testssl.sh @@ -353,6 +353,7 @@ HAS_AES256_GCM=false HAS_ZLIB=false HAS_UDS=false HAS_UDS2=false +HAS_ENABLE_PHA=false HAS_DIG=false HAS_DIG_R=true DIG_R="-r" @@ -2175,6 +2176,10 @@ s_client_options() { # isn't needed for these versions of OpenSSL.) ! "$HAS_NO_SSL2" && options="${options//-no_ssl2/}" + # The -enable_pha option causes the Post-Handshake Authentication extension to be sent. + # It is only supported by OpenSSL 1.1.1 and newer. + ! "$HAS_ENABLE_PHA" && options="${options//-enable_pha/}" + # At least one server will fail under some circumstances if compression methods are offered. # So, only offer compression methods if necessary for the test. In OpenSSL 1.1.0 and # 1.1.1 compression is only offered if the "-comp" option is provided. @@ -19450,6 +19455,7 @@ find_openssl_binary() { HAS_UDS=false HAS_UDS2=false TRUSTED1ST="" + HAS_ENABLE_PHA=false $OPENSSL ciphers -s 2>&1 | grep -aiq "unknown option" || OSSL_CIPHERS_S="-s" @@ -19521,6 +19527,8 @@ find_openssl_binary() { grep -q 'Unix-domain socket' $s_client_has && HAS_UDS=true + grep -q '\-enable_pha' $s_client_has && HAS_ENABLE_PHA=true + # Now check whether the standard $OPENSSL has Unix-domain socket and xmpp-server support. If # not check /usr/bin/openssl -- if available. This is more a kludge which we shouldn't use for # every openssl feature. At some point we need to decide which with openssl version we go. @@ -19872,6 +19880,7 @@ HAS_NNTP: $HAS_NNTP HAS_IRC: $HAS_IRC HAS_UDS: $HAS_UDS HAS_UDS2: $HAS_UDS2 +HAS_ENABLE_PHA: $HAS_ENABLE_PHA HAS_DIG: $HAS_DIG HAS_HOST: $HAS_HOST @@ -21070,7 +21079,16 @@ determine_optimal_proto() { -ssl2) "$HAS_SSL2" || continue ;; *) ;; esac - $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI") $TMPFILE 2>>$ERRFILE + # Only send $GET_REQ11 in case of a non-empty $URL_PATH, as it + # is not needed otherwise. Also, sending $GET_REQ11 may cause + # problems if the server being tested is not an HTTPS server, + # and $GET_REQ11 should be empty for non-HTTPS servers. + if [[ -z "$URL_PATH" ]] || [[ "$URL_PATH" == "/" ]]; then + $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI") $TMPFILE 2>>$ERRFILE + else + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -prexit -enable_pha") $TMPFILE 2>>$ERRFILE + fi + if sclient_auth $? $TMPFILE; then # we use the successful handshake at least to get one valid protocol supported -- it saves us time later if [[ -z "$proto" ]]; then @@ -21188,7 +21206,6 @@ determine_service() { if [[ -z "$1" ]]; then # no STARTTLS. determine_optimal_sockets_params - determine_optimal_proto $SNEAKY && \ ua="$UA_SNEAKY" || \ ua="$UA_STD" @@ -21199,6 +21216,7 @@ determine_service() { reqheader="$(join_by "\r\n" "${REQHEADERS[@]}")\r\n" #Add all required custom http headers to one string with newlines fi GET_REQ11="GET $URL_PATH HTTP/1.1\r\nHost: $NODE\r\nUser-Agent: $ua\r\n${basicauth_header}${reqheader}Accept-Encoding: identity\r\nAccept: text/*\r\nConnection: Close\r\n\r\n" + determine_optimal_proto # returns always 0: service_detection $OPTIMAL_PROTO else # STARTTLS From 7fb688a9d619c31b03f8194e04b3b2f594daf281 Mon Sep 17 00:00:00 2001 From: David Cooper Date: Mon, 3 Jan 2022 15:02:27 -0500 Subject: [PATCH 2/5] Improve check for client authentication Based on initial testing, this commit improves the check for client authentication in the case that the server only requests client authentication for specific URLs. However, it does not work correctly if the server supports TLS 1.3 and $OPENSSL is a version of LibreSSL that supports TLS 1.3 in s_client. The problem is that LibreSSL does not support post-handshake authentication with TLS 1.3 --- testssl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testssl.sh b/testssl.sh index facf0af..bf5dc5f 100755 --- a/testssl.sh +++ b/testssl.sh @@ -21086,7 +21086,7 @@ determine_optimal_proto() { if [[ -z "$URL_PATH" ]] || [[ "$URL_PATH" == "/" ]]; then $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI") $TMPFILE 2>>$ERRFILE else - safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -prexit -enable_pha") $TMPFILE 2>>$ERRFILE + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -enable_pha") >$TMPFILE 2>>$ERRFILE fi if sclient_auth $? $TMPFILE; then From 48819c56e980959a6130a6250282561d6979f366 Mon Sep 17 00:00:00 2001 From: David Cooper Date: Tue, 4 Jan 2022 09:47:14 -0500 Subject: [PATCH 3/5] Improve check for client authentication with LibreSSL Checking for client authentication with TLS 1.3 requires post-handshake authentication, which does not appear to be supported by LibreSSL. This commit improves the check for client authentication when testing a TLS 1.3 server using LibreSSL by having determine_optimal_proto() first test for connectivity with TLS 1.3 without checking for client authentication and then performing a separate check for client authentication using a non-TLS 1.3 protocol. This commit only affects the flow of the program if a $URL_PATH is specified, the server supports TLS 1.3, and $OPENSSL supports TLS 1.3 but not -enable_pha. testss.sh may still provide incorrect information about client authentication if a $URL_PATH is provided, the server is TLS 1.3-only, and LibreSSL is used. --- testssl.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/testssl.sh b/testssl.sh index bf5dc5f..e7eada0 100755 --- a/testssl.sh +++ b/testssl.sh @@ -21082,8 +21082,13 @@ determine_optimal_proto() { # Only send $GET_REQ11 in case of a non-empty $URL_PATH, as it # is not needed otherwise. Also, sending $GET_REQ11 may cause # problems if the server being tested is not an HTTPS server, - # and $GET_REQ11 should be empty for non-HTTPS servers. - if [[ -z "$URL_PATH" ]] || [[ "$URL_PATH" == "/" ]]; then + # and $URL_PATH should be empty for non-HTTPS servers. + # With TLS 1.3 it is only possible to test for client authentication + # if $OPENSSL supports post-handshake authentication. So, don't send try + # to send $GET_REQ11 after a TLS 1.3 ClientHello to a TLS 1.3 server if + # $ENABLE_PHA is false. + if [[ -z "$URL_PATH" ]] || [[ "$URL_PATH" == / ]] || \ + ( "$HAS_TLS13" && ! "$HAS_ENABLE_PHA" && ( [[ -z "$proto" ]] || [[ "$proto" == -tls1_3 ]] ) && [[ $(has_server_protocol "tls1_3") -ne 1 ]] ); then $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI") $TMPFILE 2>>$ERRFILE else safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -enable_pha") >$TMPFILE 2>>$ERRFILE @@ -21105,6 +21110,17 @@ determine_optimal_proto() { OPTIMAL_PROTO="$proto" fi all_failed=false + # If a $URL_PATH is specified and a TLS 1.3 server is being + # tested using an $OPENSSL that supports TLS 1.3 but not + # post-handshake authentication, then test for client + # authentication using a protocol version earlier than + # TLS 1.3 (unless the server only is TLS 1.3-only). + if [[ "$tmp" == tls1_3 ]] && [[ -n "$URL_PATH" ]] && [[ "$URL_PATH" != / ]] && ! "$HAS_ENABLE_PHA" && \ + ( [[ "$(has_server_protocol "tls1_2")" -eq 0 ]] || [[ "$(has_server_protocol "tls1_1")" -eq 0 ]] || \ + [[ "$(has_server_protocol "tls1")" -eq 0 ]] || [[ "$(has_server_protocol "ssl3")" -eq 0 ]] ); then + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -no_tls1_3") >$TEMPDIR/client_auth_test.txt 2>>$ERRFILE + sclient_auth $? $TEMPDIR/client_auth_test.txt + fi break fi done From 50fe6ca96bee0f3f13b6b6ad015bcffb2a80f804 Mon Sep 17 00:00:00 2001 From: David Cooper Date: Tue, 4 Jan 2022 15:38:19 -0500 Subject: [PATCH 4/5] Report if couldn't test for client authentication This commit fixes determine_optimal_proto() and run_server_defaults() so that a "Local problem" is reported if a $URL_PATH is specified, the server is TLS 1.3-only, and $OPENSSL does not support -enable_pha (and the server does not offer client authentication as part of the initial TLS handshake). --- testssl.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/testssl.sh b/testssl.sh index e7eada0..7a30d4e 100755 --- a/testssl.sh +++ b/testssl.sh @@ -10101,9 +10101,13 @@ run_server_defaults() { jsonID="clientAuth" pr_bold " Client Authentication " - outln "$CLIENT_AUTH" + if [[ "$CLIENT_AUTH" == unknown ]]; then + prln_local_problem "$OPENSSL doesn't support \"s_client -enable_pha\"" + else + outln "$CLIENT_AUTH" + fi fileout "$jsonID" "INFO" "$CLIENT_AUTH" - if [[ "$CLIENT_AUTH" != none ]]; then + if [[ "$CLIENT_AUTH" == optional ]] || [[ "$CLIENT_AUTH" == required ]]; then jsonID="clientAuth_CA_list" pr_bold " CA List for Client Auth " out_row_aligned "$CLIENT_AUTH_CA_LIST" " " @@ -21115,11 +21119,14 @@ determine_optimal_proto() { # post-handshake authentication, then test for client # authentication using a protocol version earlier than # TLS 1.3 (unless the server only is TLS 1.3-only). - if [[ "$tmp" == tls1_3 ]] && [[ -n "$URL_PATH" ]] && [[ "$URL_PATH" != / ]] && ! "$HAS_ENABLE_PHA" && \ - ( [[ "$(has_server_protocol "tls1_2")" -eq 0 ]] || [[ "$(has_server_protocol "tls1_1")" -eq 0 ]] || \ - [[ "$(has_server_protocol "tls1")" -eq 0 ]] || [[ "$(has_server_protocol "ssl3")" -eq 0 ]] ); then - safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -no_tls1_3") >$TEMPDIR/client_auth_test.txt 2>>$ERRFILE - sclient_auth $? $TEMPDIR/client_auth_test.txt + if [[ "$tmp" == tls1_3 ]] && [[ -n "$URL_PATH" ]] && [[ "$URL_PATH" != / ]] && ! "$HAS_ENABLE_PHA"; then + if [[ "$(has_server_protocol "tls1_2")" -eq 0 ]] || [[ "$(has_server_protocol "tls1_1")" -eq 0 ]] || \ + [[ "$(has_server_protocol "tls1")" -eq 0 ]] || [[ "$(has_server_protocol "ssl3")" -eq 0 ]]; then + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -no_tls1_3") >$TEMPDIR/client_auth_test.txt 2>>$ERRFILE + sclient_auth $? $TEMPDIR/client_auth_test.txt + elif [[ "$CLIENT_AUTH" == none ]]; then + CLIENT_AUTH="unknown" + fi fi break fi From 680dc9ee01e9e1f57e1e83977993a8c46737e1bb Mon Sep 17 00:00:00 2001 From: David Cooper Date: Wed, 5 Jan 2022 11:19:16 -0500 Subject: [PATCH 5/5] Fix potential stallling in HTTP query In run_http_header() the GET command is first sent over TLS using a background process, and then if that does not hang, it is sent again in the foreground. Similarly, service_detection() runs the command in the background. This commit changes determine_optimal_proto() to follow the example of run_http_header() as protection against the possibility of the HTTP query stalling. --- testssl.sh | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/testssl.sh b/testssl.sh index 7a30d4e..6792007 100755 --- a/testssl.sh +++ b/testssl.sh @@ -21095,7 +21095,18 @@ determine_optimal_proto() { ( "$HAS_TLS13" && ! "$HAS_ENABLE_PHA" && ( [[ -z "$proto" ]] || [[ "$proto" == -tls1_3 ]] ) && [[ $(has_server_protocol "tls1_3") -ne 1 ]] ); then $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI") $TMPFILE 2>>$ERRFILE else - safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -enable_pha") >$TMPFILE 2>>$ERRFILE + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -enable_pha") >$TMPFILE 2>>$ERRFILE & + wait_kill $! $HEADER_MAXSLEEP + if [[ $? -eq 0 ]]; then + # Issue HTTP GET again as it properly finished within $HEADER_MAXSLEEP and didn't hang. + # Doing it again in the foreground to get an accurate return code. + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -enable_pha") >$TMPFILE 2>>$ERRFILE + else + # Issuing HTTP GET caused $OPENSSL to hang, so just try to determine + # protocol support without also trying to collect information about + # client authentication. + $OPENSSL s_client $(s_client_options "$proto $BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI") $TMPFILE 2>>$ERRFILE + fi fi if sclient_auth $? $TMPFILE; then @@ -21122,9 +21133,18 @@ determine_optimal_proto() { if [[ "$tmp" == tls1_3 ]] && [[ -n "$URL_PATH" ]] && [[ "$URL_PATH" != / ]] && ! "$HAS_ENABLE_PHA"; then if [[ "$(has_server_protocol "tls1_2")" -eq 0 ]] || [[ "$(has_server_protocol "tls1_1")" -eq 0 ]] || \ [[ "$(has_server_protocol "tls1")" -eq 0 ]] || [[ "$(has_server_protocol "ssl3")" -eq 0 ]]; then - safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -no_tls1_3") >$TEMPDIR/client_auth_test.txt 2>>$ERRFILE - sclient_auth $? $TEMPDIR/client_auth_test.txt + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -no_tls1_3") >$TEMPDIR/client_auth_test.txt 2>>$ERRFILE & + wait_kill $! $HEADER_MAXSLEEP + # If the HTTP properly finished within $HEADER_MAXSLEEP and didn't hang, then + # do it again in the foreground to get an accurate return code. If it did hang, + # there is no way to test for client authentication, so don't try. + if [[ $? -eq 0 ]]; then + safe_echo "$GET_REQ11" | $OPENSSL s_client $(s_client_options "$BUGS -connect "$NODEIP:$PORT" -msg $PROXY $SNI -ign_eof -no_tls1_3") >$TEMPDIR/client_auth_test.txt 2>>$ERRFILE + sclient_auth $? $TEMPDIR/client_auth_test.txt + fi elif [[ "$CLIENT_AUTH" == none ]]; then + # This is a TLS 1.3-only server and $OPENSSL does not support -enable_pha, so it is not + # possible to test for client authentication. CLIENT_AUTH="unknown" fi fi