From 3c65d1bb24bb25c6ed715fc9eaddf527ac97f209 Mon Sep 17 00:00:00 2001
From: Jordan Halterman <jordan@opennetworking.org>
Date: Tue, 9 Jan 2018 13:01:03 -0800
Subject: [PATCH] [ONOS-7350] Add ISSU HA/rollback test scenario

Change-Id: I8c1952480cd429d40c77db2e66d16ace27948148
---
 .../upgrade/impl/UpgradeManager.java          |   1 +
 tools/test/bin/onos-check-issu-status         |  27 +++
 tools/test/scenarios/ha-upgrade.xml           | 165 ++++++++++++++++++
 3 files changed, 193 insertions(+)
 create mode 100755 tools/test/bin/onos-check-issu-status
 create mode 100644 tools/test/scenarios/ha-upgrade.xml

diff --git a/core/net/src/main/java/org/onosproject/upgrade/impl/UpgradeManager.java b/core/net/src/main/java/org/onosproject/upgrade/impl/UpgradeManager.java
index ab1b1c3e92..218a357589 100644
--- a/core/net/src/main/java/org/onosproject/upgrade/impl/UpgradeManager.java
+++ b/core/net/src/main/java/org/onosproject/upgrade/impl/UpgradeManager.java
@@ -385,6 +385,7 @@ public class UpgradeManager
                         .filter(id -> clusterService.getVersion(id).equals(upgrade.target()))
                         .collect(Collectors.toSet());
                 if (upgradedNodes.contains(event.subject().id())) {
+                    log.warn("Upgrade failure detected: rolling back upgrade");
                     rollback();
                 }
             }
diff --git a/tools/test/bin/onos-check-issu-status b/tools/test/bin/onos-check-issu-status
new file mode 100755
index 0000000000..a0aa52bc7b
--- /dev/null
+++ b/tools/test/bin/onos-check-issu-status
@@ -0,0 +1,27 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Checks whether the specified ONOS cluster node has the desired state.
+# -----------------------------------------------------------------------------
+
+aux=/tmp/stc/stc-$$.log
+trap "rm -f $aux 2>/dev/null" EXIT
+
+function onos_nodes() {
+    echo $(env | sort | egrep "^OC[0-9]+" | cut -d= -f2)
+}
+
+nodes=$(onos_nodes)
+
+for attempt in {1..5}; do
+    for node in $nodes; do
+        onos $node issu status > $aux
+        cat $aux
+
+        status=$(cat $aux)
+
+        [ "$status" = "${1}" ] && exit 0
+        sleep 1
+    done
+done
+
+exit 1
\ No newline at end of file
diff --git a/tools/test/scenarios/ha-upgrade.xml b/tools/test/scenarios/ha-upgrade.xml
new file mode 100644
index 0000000000..0f832a9634
--- /dev/null
+++ b/tools/test/scenarios/ha-upgrade.xml
@@ -0,0 +1,165 @@
+<!--
+  ~ Copyright 2017-present Open Networking Foundation
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<scenario name="ha-upgrade" description="ONOS cluster upgrade fault tolerance">
+    <import file="${ONOS_SCENARIOS}/dist-setup.xml"/>
+    <dependency name="Distributed-Primitives-Setup"/>
+
+    <group name="HA-Upgrade" requires="Distributed-Primitives-Setup">
+        <step name="Push-Bits" exec="onos-push-bits-through-proxy" if="${OCT}"/>
+
+        <step name="Initialize-ECM"
+              exec="onos ${OC1} ec-map-test foo put a b"/>
+
+        <step name="Initialize-CM"
+              exec="onos ${OC1} map-test foo put a b"
+              requires="Initialize-ECM"/>
+
+        <step name="Initialize-Upgrade"
+              exec="onos ${OC1} issu init"
+              requires="Initialize-CM"/>
+
+        <group name="Phase-1">
+            <sequential var="${OCMI#}"
+                        starts="Phase-One-Stop-Service-${#}"
+                        ends="Phase-One-Wait-for-Start-${#-1}">
+                <step name="Phase-One-Stop-Service-${#}"
+                      exec="onos-service ${OCMI#} stop"
+                      requires="Initialize-Upgrade"/>
+
+                <step name="Phase-One-Wait-for-Stop-${#}"
+                      exec="onos-wait-for-stop ${OCMI#}"
+                      requires="~Phase-One-Stop-Service-${#}"/>
+
+                <step name="Phase-One-Uninstall-${#}"
+                      exec="onos-uninstall ${OCMI#}"
+                      requires="~Phase-One-Wait-for-Stop-${#}"/>
+
+                <step name="Phase-One-Push-Bits-${#}"
+                      exec="onos-push-bits ${OCMI#}"
+                      unless="${OCT}"
+                      requires="~Phase-One-Stop-Service-${#}"/>
+
+                <step name="Phase-One-Install-Upgrade-${#}"
+                      exec="onos-install -v ${OCMI#}"
+                      requires="Phase-One-Push-Bits-${#},Push-Bits,Phase-One-Uninstall-${#}"/>
+
+                <step name="Phase-One-Secure-SSH-${#}"
+                      exec="onos-secure-ssh -u ${ONOS_WEB_USER} -p ${ONOS_WEB_PASS} ${OCMI#}"
+                      requires="~Phase-One-Install-Upgrade-${#}"/>
+
+                <step name="Phase-One-Wait-for-Start-${#}"
+                      exec="onos-wait-for-start ${OCMI#}"
+                      requires="Phase-One-Secure-SSH-${#}"/>
+
+                <step name="Phase-One-Distributed-Primitives-Check-Apps-${#}"
+                      exec="onos-check-apps ${OCMI#} distributedprimitives includes"
+                      requires="Phase-One-Wait-for-Start-${#}"/>
+
+                <step name="Phase-One-Check-ECM-${#}"
+                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect b"
+                      requires="Phase-One-Distributed-Primitives-Check-Apps-${#}"/>
+
+                <step name="Phase-One-Check-CM-${#}"
+                      exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect b"
+                      requires="Phase-One-Check-ECM-${#}"/>
+            </sequential>
+        </group>
+
+        <step name="Run-Upgrade"
+              exec="onos ${OC1} issu upgrade"
+              requires="Phase-1"/>
+
+        <step name="Kill-Upgraded-Node"
+              exec="onos-power ${OCMI1} off"
+              requires="Run-Upgrade"/>
+
+        <step name="Wait-For-Downgrade"
+              exec="onos-check-issu-status ROLLED_BACK"
+              requires="Kill-Upgraded-Node"/>
+
+        <group name="Phase-2">
+            <sequential var="${OCMI#}"
+                        starts="Phase-Two-Stop-Service-${#}"
+                        ends="Phase-Two-Wait-for-Start-${#-1}">
+                <step name="Phase-Two-Stop-Service-${#}"
+                      exec="onos-service ${OCMI#} stop"
+                      requires="Wait-For-Downgrade"/>
+
+                <step name="Phase-Two-Wait-for-Stop-${#}"
+                      exec="onos-wait-for-stop ${OCMI#}"
+                      requires="~Phase-Two-Stop-Service-${#}"/>
+
+                <step name="Phase-Two-Uninstall-${#}"
+                      exec="onos-uninstall ${OCMI#}"
+                      requires="~Phase-Two-Wait-for-Stop-${#}"/>
+
+                <step name="Phase-Two-Push-Bits-${#}"
+                      exec="onos-push-bits ${OCMI#}"
+                      unless="${OCT}"
+                      requires="~Phase-Two-Stop-Service-${#}"/>
+
+                <step name="Phase-Two-Install-Downgrade-${#}"
+                      exec="onos-install ${OCMI#}"
+                      requires="Phase-Two-Push-Bits-${#},Push-Bits,Phase-Two-Uninstall-${#}"/>
+
+                <step name="Phase-Two-Secure-SSH-${#}"
+                      exec="onos-secure-ssh -u ${ONOS_WEB_USER} -p ${ONOS_WEB_PASS} ${OCMI#}"
+                      requires="~Phase-Two-Install-Downgrade-${#}"/>
+
+                <step name="Phase-Two-Wait-for-Start-${#}"
+                      exec="onos-wait-for-start ${OCMI#}"
+                      requires="Phase-Two-Secure-SSH-${#}"/>
+
+                <step name="Phase-Two-Distributed-Primitives-Check-Apps-${#}"
+                      exec="onos-check-apps ${OCMI#} distributedprimitives includes"
+                      requires="Phase-Two-Wait-for-Start-${#}"/>
+
+                <step name="Phase-Two-Check-ECM-${#}"
+                      exec="onos-execute-expect ${OCMI#} ec-map-test foo get a --retry 5 --expect b"
+                      requires="Phase-Two-Distributed-Primitives-Check-Apps-${#}"/>
+
+                <step name="Phase-Two-Check-CM-${#}"
+                      exec="onos-execute-expect ${OCMI#} map-test foo get a --retry 5 --expect b"
+                      requires="Phase-Two-Check-ECM-${#}"/>
+            </sequential>
+        </group>
+
+        <step name="Reset-Downgrade"
+              exec="onos ${OC1} issu reset"
+              requires="Phase-2"/>
+
+        <group name="Verify-Upgrade" requires="Reset-Downgrade">
+            <parallel var="${OC#}">
+                <step name="Check-Nodes-${#}"
+                      exec="onos-check-nodes ${OC#}"
+                      delay="3"/>
+
+                <step name="Check-Components-${#}"
+                      exec="onos-check-components ${OC#}"
+                      delay="5"
+                      requires="~Check-Nodes-${#}"/>
+
+                <step name="Check-Logs-${#}"
+                      exec="onos-check-logs ${OC#}"
+                      requires="~Check-Components-${#}"/>
+
+                <step name="Check-Apps-${#}"
+                      exec="onos-check-apps ${OC#} ${ONOS_APPS},distributedprimitives includes"
+                      requires="~Check-Components-${#}"/>
+            </parallel>
+        </group>
+    </group>
+</scenario>