Enhancing diagnostic and test facilities for troubleshooting

- added new CLI to list driver providers
- added ability to power on/off borrowed cells
- enabled ONOS service to restart on boot/death

Change-Id: Ifc889cdbc0740e5d3286c9836dfecc38d458bbc5

cli/src/main/java/org/onosproject/cli/net/DriverProvidersListCommand.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cli.net;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.net.driver.Driver;
+import org.onosproject.net.driver.DriverAdminService;
+import org.onosproject.net.driver.DriverProvider;
+
+import java.util.stream.Collectors;
+
+/**
+ * Lists device drivers.
+ */
+@Command(scope = "onos", name = "driver-providers",
+        description = "Lists device driver providers")
+public class DriverProvidersListCommand extends AbstractShellCommand {
+
+    private static final String FMT = "provider=%s, drivers=%s";
+
+    @Override
+    protected void execute() {
+        DriverAdminService service = get(DriverAdminService.class);
+        service.getProviders().forEach(this::printDriverProvider);
+    }
+
+    private void printDriverProvider(DriverProvider provider) {
+        print(FMT, provider.getClass().getName(),
+              provider.getDrivers().stream()
+                      .map(Driver::name)
+                      .collect(Collectors.toSet()));
+    }
+
+}

cli/src/main/resources/OSGI-INF/blueprint/shell-config.xml

@@ -128,6 +128,9 @@
             <action class="org.onosproject.cli.net.McastDeleteCommand"/>
         </command>
 
+        <command>
+            <action class="org.onosproject.cli.net.DriverProvidersListCommand"/>
+        </command>
         <command>
             <action class="org.onosproject.cli.net.DriversListCommand"/>
             <completers>

tools/test/bin/onos-install

@@ -101,8 +101,8 @@ ssh -tt $remote "
     echo 'log4j.appender.out.layout.ConversionPattern=%d{ISO8601} | %-5.5p | %-16.16t | %-32.32c{1} | %X{bundle.id} | %m%n' \
         >> $ONOS_INSTALL_DIR/$KARAF_DIST/etc/org.ops4j.pax.logging.cfg
 
-    # Set up the ONOS service on systemd-based systems
-    sudo systemctl daemon-reload || true
+    # Set up and enable the ONOS service on systemd-based systems
+    sudo systemctl daemon-reload && sudo systemctl enable onos.service || true
 
 "
 

tools/test/bin/onos-power

@@ -0,0 +1,34 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Provides a unioform way to simulate power on/off control over an ONOS node.
+# If environment variable HARD_POWER_OFF is set and the ONOS_CELL value
+# indicates that the cell is a borrowed shared cell LXC machine, it will use
+# a REST call to warden, which then uses lxc-stop/lxc-start command to
+# simulate a power control operations. Otherwise, it will just use the soft
+# 'onos-die' command.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+node=${1:-$OCI}
+cmd=${2:-on}
+
+if [ "$cmd" = "on" ]; then
+    if [ -n "$HARD_POWER_OFF" -a $ONOS_CELL = "borrow" ]; then
+        curl -sS -X PUT "http://$CELL_WARDEN:4321/?cmd=powerOn&nodeIp=$node&user=$(id -un)"
+    else
+        onos-service $node start
+    fi
+
+elif [ "$cmd" = "off" ]; then
+    if [ -n "$HARD_POWER_OFF" -a $ONOS_CELL = "borrow" ]; then
+        curl -sS -X PUT "http://$CELL_WARDEN:4321/?cmd=powerOff&nodeIp=$node&user=$(id -un)"
+    else
+        onos-die $node start
+    fi
+
+else
+    echo "usage: $(basename $0) node-ip {on|off}" >&2 && exit 1
+fi
+

tools/test/scenarios/ha-single-node.xml

@@ -17,8 +17,7 @@
 <scenario name="ha-single-node"
           description="ONOS cluster single node failure and recovery">
     <group name="Single-Node-Failure" if="${OC2}">
-        <step name="Node-Death" exec="onos-die ${OC1}"/>
-
+        <step name="Node-Death" exec="onos-power ${OC1} off"/>
         <step name="Validate-Death" requires="Node-Death"
               exec="onos-check-node-status ${OC2} ${OC1} INACTIVE"/>
 
@@ -36,10 +35,8 @@
                   exec="onos-check-flows ${OC2}"/>
         </group>
 
-        <group name="Recover" requires="Validate-Normal-Operation">
-            <step name="Node-Start"
-                  exec="onos-service ${OC1} start"/>
-
+        <group name="Recover" requires="~Validate-Normal-Operation">
+            <step name="Node-Start" exec="onos-power ${OC1} on"/>
             <step name="Wait-for-Start" requires="~Node-Start"
                   exec="onos-wait-for-start ${OC1}"/>
         </group>