Enhancing onos-check-logs to allow ignoring store exceptions

Change-Id: I60dae339e987a15d0fa3d71967e13f3ce09434bc

tools/test/bin/onos-check-logs

@@ -5,14 +5,16 @@
 
 function __usage() {
 cat << _EOM_
-
-usage:
- $(basename $0) [node] ['old']
+usage: $(basename $0) [--old|--ignore-store-exceptions] [node]
 
 options:
- - [node]  : The node whose logs to inspect. The default is \$OCI.
- - ['old'] : If 'old' is specified, the logs are simply searched for errors
-             and exceptions, and they are displayed.
+ --ignore-store-exceptions
+        if specified, any store service exceptions are ignored
+ --old
+        if specified, the entire logs are searched for errors and exceptions;
+        otherwise logs are scanned only from the last server start-up
+
+ node   the cluster node whose logs to inspect; default is \$OCI
 
 summary:
  Checks the logs of the remote ONOS instance and makes sure they are clean.
@@ -20,24 +22,33 @@ summary:
 _EOM_
 }
 
-[ "$1" = "-h" ] && __usage && exit 0
-
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults
 
+# Process options
+while [ -z $opts ] ; do
+    case "$1" in
+    --ignore-store-exceptions) export ise=1; shift;;
+    --old) export allLogs=1; shift;;
+    --*) __usage && exit 1;;
+    -h)  __usage && exit 0;;
+    *) export opts=1;;
+    esac
+done
+
 remote=$ONOS_USER@${1:-$OCI}
 
 LOG=$ONOS_INSTALL_DIR/log/karaf.log*
 
 aux=/tmp/log.$$
 
-if [ "$2" = "old" ]; then
+if [ -n "$allLogs" ]; then
     ssh $remote "egrep 'ERROR|Exception|Error' $LOG"
 
 else
     ssh $remote "
         [ "'`uname`'" != "'"Linux"'" ] && alias tac='tail -r'
-        tac $LOG | tr -d '\000' | awk '
+        tac $LOG | tr -d '\000' | awk -v ignoreStoreExceptions=${ise:-0} '
             BEGIN { off = 0; fail = 0; exclusion = 0; trace = \"\";}
             / org.apache.karaf.main.lock.SimpleFileLock lock/ {
                 exit fail;
@@ -63,6 +74,9 @@ else
                 exclusion = 0;
             }
 
+            # Sanctioned exclusions for exceptions in the distributed stores; one pattern per exclusion
+            /org\.onosproject\.store\.service\..*Exception/ { exclusion = ignoreStoreExceptions;  }
+
             /^[a-zA-Z0-9.]*(Exception|Error)/ {
                 if (!exclusion) {
                     print trace;

tools/test/scenarios/ha-single-node.xml

@@ -53,7 +53,7 @@
                       exec="onos-check-components ${OC#}"/>
 
                 <step name="Check-Logs-${#}" requires="~Check-Nodes-${#}"
-                      exec="onos-check-logs ${OC#}"/>
+                      exec="onos-check-logs --ignore-store-exceptions ${OC#}"/>
                 <step name="Check-Apps-${#}" requires="~Check-Nodes-${#}"
                       exec="onos-check-apps ${OC#} ${ONOS_APPS} includes"/>