From 1ed2c9da2c8ce5d18f5136d63561a71e2ba4afd0 Mon Sep 17 00:00:00 2001
From: William Lallemand <wlallemand@irq6.net>
Date: Tue, 9 Dec 2025 17:18:42 +0100
Subject: [PATCH] REGTESTS: list all skipped tests including 'feature cmd' ones

The script for running regression tests is modified to improve the
visibility of skipped tests.

Previously, the reasons for skipping tests were only visible during the
test discovery phase when grepping the vtc (REQUIRE, EXCLUDE, etc).
But reg-tests skipped by vtest with the 'feature cmd' keywords were not
listed.

This change introduces the following:
  - vtest does not remove the logs itself anymore, because it is not
    able to let the log available when a test is skipped. So the -L
    parameter is now always passed to vtest
  - All skipped tests during the discovery phase are now logged to a
    'skipped.log' file within the test directory
  - The script now parses vtest logs to find tests that were skipped
    due to missing features (via the 'feature cmd' in .vtc files)
    and adds them to the skipped list.
---
 scripts/run-regtests.sh | 72 +++++++++++++++++++++++++++++------------
 1 file changed, 51 insertions(+), 21 deletions(-)

diff --git a/scripts/run-regtests.sh b/scripts/run-regtests.sh
index 912ac94f0..1229649ee 100755
--- a/scripts/run-regtests.sh
+++ b/scripts/run-regtests.sh
@@ -144,7 +144,7 @@ _findtests() {
             regtest_type=default
         fi
         if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
-            echo "  Skip $i because its type '"$regtest_type"' is excluded"
+            echo "  Skipped $i because its type '"$regtest_type"' is excluded" >> "${TESTDIR}/skipped.log"
             skiptest=1
         fi
     fi
@@ -167,7 +167,7 @@ _findtests() {
 
     for excludedtarget in $exclude_targets; do
       if [ "$excludedtarget" = "$TARGET" ]; then
-        echo "  Skip $i because haproxy is compiled for the excluded target $TARGET"
+        echo "  Skipped $i because haproxy is compiled for the excluded target $TARGET" >> "${TESTDIR}/skipped.log"
         skiptest=1
       fi
     done
@@ -181,7 +181,7 @@ _findtests() {
 	fi
       done
       if [ -z $found ]; then
-        echo "  Skip $i because haproxy is not compiled with the required option $requiredoption"
+        echo "  Skipped $i because haproxy is not compiled with the required option $requiredoption"  >> "${TESTDIR}/skipped.log"
         skiptest=1
       fi
     done
@@ -195,7 +195,7 @@ _findtests() {
 	fi
       done
       if [ -z $found ]; then
-        echo "  Skip $i because haproxy is not compiled with the required service $requiredservice"
+        echo "  Skipped $i because haproxy is not compiled with the required service $requiredservice"  >> "${TESTDIR}/skipped.log"
         skiptest=1
       fi
     done
@@ -255,7 +255,7 @@ _process() {
           debug="-v"
           ;;
         --keep-logs)
-          keep_logs="-L"
+          keep_logs=1
           ;;
         --type)
 	      REGTESTS_TYPES="$2"
@@ -302,7 +302,7 @@ LINEFEED="
 jobcount=""
 verbose="-q"
 debug=""
-keep_logs="-l"
+keep_logs=0
 testlist=""
 
 _process "$@";
@@ -376,33 +376,63 @@ if [ -n "$testlist" ]; then
   if [ -n "$jobcount" ]; then
     jobcount="-j $jobcount"
   fi
-  cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t ${VTEST_TIMEOUT} $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
+  cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t ${VTEST_TIMEOUT} -L $verbose $debug $jobcount $vtestparams $testlist"
   eval $cmd
   _vtresult=$?
 else
   echo "No tests found that meet the required criteria"
 fi
 
-
-if [ $_vtresult -eq 0 ]; then
-  # all tests were successful, removing tempdir (the last part.)
-  # ignore errors is the directory is not empty or if it does not exist
-   rmdir "$TESTDIR" 2>/dev/null
-fi
-
 if [ -d "${TESTDIR}" ]; then
-  echo "########################## Gathering results ##########################"
-  export TESTDIR
-  find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
-    if [ ! -e "$i/LOG" ] ; then continue; fi
+  # look for tests skipped by vtest
+  find "${TESTDIR}" -type f -name "LOG" | while read logfile; do
+    REASON=$(grep "SKIPPING test" "$logfile")
+    if [ -n "$REASON" ]; then
+      infofile="$(dirname "$logfile")/INFO"
+      if [ -e "$infofile" ]; then
+        vtc_path=$(sed 's/^Test case: //' "$infofile" )
+        if [ -n "$vtc_path" ]; then
+          echo "  Skipped $vtc_path (feature cmd)" >> "${TESTDIR}/skipped.log"
+        fi
+      fi
+    fi
+  done
 
-    cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
+  if [ $keep_logs -eq 0 ]; then
+    # remove logs for successful tests
+    find "$TESTDIR" -type d -name "vtc.*" | while read vtcdir; do
+      # errors are starting with ----
+      grep -q "^----" ${vtcdir}/LOG || rm -fr "${vtcdir}"
+    done
+  fi
+
+  if [ $_vtresult -eq 0 ]; then
+    # all tests were successful, removing tempdir (the last part.)
+    # ignore errors is the directory is not empty or if it does not exist
+     rmdir "$TESTDIR" 2>/dev/null
+  fi
+
+  # show failed tests
+  if [ -d "${TESTDIR}" ]; then
+    echo "########################## Gathering results ##########################"
+    export TESTDIR
+    find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
+      if [ ! -e "$i/LOG" ] ; then continue; fi
+
+      cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
 $(echo "###### $(cat "$i/INFO") ######")
 $(echo "## test results in: \"$i\"")
 $(echo "## test log file: $i/LOG")
 $(grep -E -- "^(----|\*    diag)" "$i/LOG")
 EOF
-  done' sh {} +
-fi
+    done' sh {} +
+  fi
+
+  echo "########################## Listing skipped tests ####################"
+  count=$(wc -l < "${TESTDIR}/skipped.log")
+  cat "${TESTDIR}/skipped.log" | sort -n
+  echo "Total skipped tests: $count"
+
+fi # if TESTDIR
 
 exit $_vtresult