From 30675ca0b4123753a56fd39a775e486aa2be3f7a Mon Sep 17 00:00:00 2001
From: Dimitrios Mavrommatis <dimitris@opennetworking.org>
Date: Mon, 13 Nov 2017 18:26:53 -0800
Subject: [PATCH] [ONOS-7221] Upgrade state tests

Change-Id: I0e9214ddcb272dc98bb8f54fa600704c38e3562d
---
 tools/test/bin/onos-execute-expect        | 43 ++++++++++++++++-------
 tools/test/scenarios/upgrade-rollback.xml | 35 ++++++++++++++----
 tools/test/scenarios/upgrade.xml          | 34 +++++++++++++++---
 3 files changed, 89 insertions(+), 23 deletions(-)

diff --git a/tools/test/bin/onos-execute-expect b/tools/test/bin/onos-execute-expect
index 4b9270ee9b..0413522597 100755
--- a/tools/test/bin/onos-execute-expect
+++ b/tools/test/bin/onos-execute-expect
@@ -1,27 +1,46 @@
 #!/bin/bash
 # -----------------------------------------------------------------------------
 # Executes a command on the given ONOS instance and matches the output
-# to the passed one.
+# to the passed after X retries.
 # First argument is the IP address of the machine to run the command on,
 # then you pass the command and it's arguments if needed, then --expect and
-# after it the string of what the output should be.
+# after it the string of what the output should be or --retry and the maximum
+# retries with 5 seconds delay between them
 # Example:
-# onos-execute-expect 1.1.1.1 fooCommand fooParamenter --expect fooOutputString
+# onos-execute-expect 1.1.1.1 fooCommand fooParamenter --retry 5 --expect fooOutputString
 # -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults
 
-
 aux=/tmp/stc/stc-$$.log
 trap "rm -f $aux 2>/dev/null" EXIT
-ip=$1
+
+retry=0
 cmd=""
-for a in ${*:2}; do shift; if [ "$a" = "--expect" ]; then break; fi; cmd="$cmd $a"; done
-expect="${@: -1}"
-onos $ip $cmd > $aux
-cat $aux
-grep -q $expect $aux && echo "expected value found" && exit 0
+expct=""
+
+for ((i=1; i<=$#; i++))
+do
+	if [ "${!i}" = "--expect" ]; then
+		next=$((i+1))
+		expct="${!next}"
+		i=$i+1
+	elif [ "${!i}" = "--retry" ]; then
+		next=$((i+1))
+		retry="${!next}"
+		i=$i+1
+	else
+		cmd="$cmd ${!i}";
+	fi
+done
+
+until [ $retry -eq -1 ]
+do
+	onos $cmd > $aux
+	cat $aux
+	grep -q $expct $aux && echo "expected value found" && exit 0
+	retry=$[$retry-1]
+	sleep 5
+done
 exit 1
-
-
diff --git a/tools/test/scenarios/upgrade-rollback.xml b/tools/test/scenarios/upgrade-rollback.xml
index 05841bbe3e..13f4403854 100644
--- a/tools/test/scenarios/upgrade-rollback.xml
+++ b/tools/test/scenarios/upgrade-rollback.xml
@@ -23,6 +23,10 @@
         <step name="Initialize-ECM"
               exec="onos ${OC1} ec-map-test foo put a b"/>
 
+        <step name="Initialize-CM"
+              exec="onos ${OC1} map-test foo put a b"
+              requires="Initialize-ECM"/>
+
         <step name="Initialize-Upgrade"
               exec="onos ${OC1} issu init"
               requires="Initialize-ECM"/>
@@ -65,8 +69,12 @@
                       requires="Phase-One-Wait-for-Start-${#}"/>
 
                 <step name="Phase-One-Check-ECM-${#}"
-                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect b"
+                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect b"
                       requires="Phase-One-Distributed-Primitives-Check-Apps-${#}"/>
+
+                <step name="Phase-One-Check-CM-${#}"
+                      exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect b"
+                      requires="Phase-One-Check-ECM-${#}"/>
             </sequential>
         </group>
 
@@ -74,19 +82,31 @@
             <step name="Change-Majority-ECM"
                   exec="onos ${OCMA1} ec-map-test foo put a d"/>
 
+            <step name="Change-Majority-CM"
+                  exec="onos ${OCMA1} map-test foo put a d"/>
+
             <step name="Change-Minority-ECM"
                   exec="onos ${OCMI1} ec-map-test foo put a c"/>
+
+            <step name="Change-Minority-CM"
+                  exec="onos ${OCMI1} map-test foo put a c"/>
         </group>
 
         <group name="Check-ECM-values" requires="Change-ECM-values">
             <parallel var="${OCMI#}">
                 <step name="Parallel-Check-Minority-ECM-${#}"
-                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect c"/>
+                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect c"/>
+
+                <step name="Parallel-Check-Minority-CM-${#}"
+                      exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect c"/>
             </parallel>
 
             <parallel var="${OCMA#}">
                 <step name="Parallel-Check-Majority-ECM-${#}"
-                      exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --expect d"/>
+                      exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --retry 5 --expect d"/>
+
+                <step name="Parallel-Check-Majority-CM-${#}"
+                      exec="onos-execute-expect ${OCMA#} map-test foo get a --retry 5 --expect d"/>
             </parallel>
         </group>
 
@@ -136,8 +156,12 @@
                       requires="Phase-Two-Wait-for-Start-${#}"/>
 
                 <step name="Phase-Two-Check-ECM-${#}"
-                      exec="onos-execute-expect ${OC#} ec-map-test foo get a --expect d"
+                      exec="onos-execute-expect ${OC#} ec-map-test foo get a --retry 5 --expect d"
                       requires="Phase-Two-Distributed-Primitives-Check-Apps-${#}"/>
+
+                <step name="Phase-Two-Check-CM-${#}"
+                      exec="onos-execute-expect ${OC#} map-test foo get a --retry 5 --expect d"
+                      requires="Phase-Two-Check-ECM-${#}"/>
             </sequential>
         </group>
 
@@ -149,8 +173,7 @@
             <parallel var="${OC#}">
                 <step name="Check-Nodes-${#}"
                       exec="onos-check-nodes ${OC#}"
-                      delay="3"
-                      requires="Verify-EC-Map-${#}"/>
+                      delay="3"/>
 
                 <step name="Check-Components-${#}"
                       exec="onos-check-components ${OC#}"
diff --git a/tools/test/scenarios/upgrade.xml b/tools/test/scenarios/upgrade.xml
index 16fc6271d1..0890dedc9c 100644
--- a/tools/test/scenarios/upgrade.xml
+++ b/tools/test/scenarios/upgrade.xml
@@ -23,9 +23,13 @@
         <step name="Initialize-ECM"
               exec="onos ${OC1} ec-map-test foo put a b"/>
 
+        <step name="Initialize-CM"
+              exec="onos ${OC1} map-test foo put a b"
+              requires="Initialize-ECM"/>
+
         <step name="Initialize-Upgrade"
               exec="onos ${OC1} issu init"
-              requires="Initialize-ECM"/>
+              requires="Initialize-CM"/>
 
         <group name="Phase-1">
             <sequential var="${OCMI#}"
@@ -65,8 +69,12 @@
                       requires="Phase-One-Wait-for-Start-${#}"/>
 
                 <step name="Phase-One-Check-ECM-${#}"
-                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect b"
+                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect b"
                       requires="Phase-One-Distributed-Primitives-Check-Apps-${#}"/>
+
+                <step name="Phase-One-Check-CM-${#}"
+                      exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect b"
+                      requires="Phase-One-Check-ECM-${#}"/>
             </sequential>
         </group>
 
@@ -74,19 +82,31 @@
             <step name="Change-Majority-ECM"
                   exec="onos ${OCMA1} ec-map-test foo put a d"/>
 
+            <step name="Change-Majority-CM"
+                  exec="onos ${OCMA1} map-test foo put a d"/>
+
             <step name="Change-Minority-ECM"
                   exec="onos ${OCMI1} ec-map-test foo put a c"/>
+
+            <step name="Change-Minority-CM"
+                  exec="onos ${OCMI1} map-test foo put a c"/>
         </group>
 
         <group name="Check-ECM-values" requires="Change-ECM-values">
             <parallel var="${OCMI#}">
                 <step name="Parallel-Check-Minority-ECM-${#}"
-                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --expect c"/>
+                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect c"/>
+
+                <step name="Parallel-Check-Minority-CM-${#}"
+                      exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect c"/>
             </parallel>
 
             <parallel var="${OCMA#}">
                 <step name="Parallel-Check-Majority-ECM-${#}"
-                      exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --expect d"/>
+                      exec="onos-execute-expect ${OCMA#} ec-map-test foo get a --retry 5 --expect d"/>
+
+                <step name="Parallel-Check-Majority-CM-${#}"
+                      exec="onos-execute-expect ${OCMA#} map-test foo get a --retry 5 --expect d"/>
             </parallel>
         </group>
 
@@ -132,8 +152,12 @@
                       requires="Phase-Two-Wait-for-Start-${#}"/>
 
                 <step name="Phase-Two-Check-ECM-${#}"
-                      exec="onos-execute-expect ${OC#} ec-map-test foo get a --expect c"
+                      exec="onos-execute-expect ${OC#} ec-map-test foo get a --retry 5 --expect c"
                       requires="Phase-Two-Distributed-Primitives-Check-Apps-${#}"/>
+
+                <step name="Phase-Two-Check-CM-${#}"
+                      exec="onos-execute-expect ${OC#} map-test foo get a --retry 5 --expect c"
+                      requires="Phase-Two-Check-ECM-${#}"/>
             </sequential>
         </group>