diff --git a/tools/dev/mininet/examples/ha.py b/tools/dev/mininet/examples/ha.py new file mode 100755 index 0000000000..18f12226da --- /dev/null +++ b/tools/dev/mininet/examples/ha.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python + +""" +NOTES + +To change onos log levels before start you can add something similar to +onos.py's ONOSNode.start method before starting onos service: + # Change log levels + self.ucmd( 'echo "log4j.logger.io.atomix= DEBUG" >> $ONOS_HOME/apache-karaf-*/etc/org.ops4j.pax.logging.cfg' ) + +""" + +import argparse +from mininet.log import output, info, warn, error, debug, setLogLevel +from mininet.cli import CLI as origCLI +from mininet.net import Mininet +from mininet.topo import SingleSwitchTopo, Topo +from mininet.node import Host +from os.path import join +from glob import glob +import re +import json +from collections import deque +import hashlib +import onos # onos.py + +# Utility functions + +def pause( net, msg, hint=False): + """Reenter the CLI. Note that we use the mn base CLI class to allow + extensibility and combination of custom files""" + + info( msg ) + + if hint: + help_msg = "Currently in the root Mininet net namespace...\n" + help_msg += "To access control net functions use:\n" + help_msg += "\tpx cnet=net.controllers[0].net\n" + help_msg += "\tpy cnet.METHOD\n" + help_msg += "To send commands to each onos node, use: onos_all CMD\n" + help_msg += "\nBy default, ONOS nodes are running on the 192.168.123.X network\n" + info( "%s\n" % help_msg ) + # NOTE: If we use onos.py as a custom file and as an imported module, + # we get two different sets of ONOS* classes. They don't play + # together and things don't work properly. Specifically the + # isinstance calls fail. This is due to import vs. exec calls + onos.ONOSCLI( net ) + +def cprint( msg, color="default"): + color=color.lower() + colors = { 'cyan': '\033[96m', 'purple': '\033[95m', + 'blue': '\033[94m', 'green': '\033[92m', + 'yellow': '\033[93m', 'red': '\033[91m', + 'end': '\033[0m' } + pre = colors.get( color, '' ) + output = pre + msg + colors['end'] + print( output ) + +def getNode( net, nodeId=0 ): + "Helper function: return ONOS node, defaults to the first node" + return net.controllers[ 0 ].nodes()[ nodeId ] + +def onos_cli( net, line, nodeId=0 ): + "Send command to ONOS CLI" + c0 = net.controllers[ 0 ] + # FIXME add this back after import onos.py works + if isinstance( c0, onos.ONOSCluster ): + # cmdLoop strips off command name 'onos' + if line.startswith( ':' ): + line = 'onos' + line + node = getNode( net, nodeId ) + if line: + line = '"%s"' % line + cmd = 'client -h %s %s' % ( node.IP(), line ) + #node.cmdPrint( cmd ) + output = node.cmd( cmd ) + info( line ) + # Remove verbose spam from output + m = re.search( "unverified \{\} key: \{\}", output ) + if m: + info( output[m.end():] ) + else: + info( output ) + +def onos_all( net, line ): + onosNodes = [ n for cluster in net.controllers for n in cluster.nodes() ] + for node in range( len( onosNodes ) ): + cprint( "*" * 53, "red" ) + cprint( "onos%s: %s" % ( str( node + 1 ), repr( onosNodes[ node ] ) ), + "red" ) + cprint( "*" * 53, "red" ) + onos_cli( net, line, node ) + +# FIXME This needs a better name +def do_onos_all( self, line ): + onos_all( self.mn, line) + +# Add custom cli commands +# NOTE: This is so we can keep ONOSCLI and also add commands to it! +origCLI.do_onos_all = do_onos_all + +# Test cases + +def Partition( net ): + # Controller net instance + cnet = net.controllers[0].net + + info( "ONOS control network partition test\n") + net.pingAll() + if args.interactive: + pause( net, "~~~ Dropping into cli... Exit cli to continue test\n", True ) + + onos_all( net, "nodes;partitions;partitions -c") + info( "~~~ Right before the partitioned\n" ) + if args.interactive: + pause( net, "Dropping into cli... Exit cli to continue test\n" ) + + cs1 = cnet.switches[0] + cs2 = cnet.switches[1] + + # PARTITION sub-clusters + + # we need to use names here + cnet.configLinkStatus( cs1.name, cs2.name, "down" ) + onos_all( net, "nodes;partitions;partitions -c") + info( "~~~ Right after cluster is partitioned. Next step is to heal the partition\n" ) + if args.interactive: + pause( net, "Dropping into cli... Exit cli to continue test\n" ) + + cnet.configLinkStatus( cs1.name, cs2.name, "up" ) + onos_all( net, "nodes;partitions;partitions -c") + info( "~~~ Right after the partition is healed \n" ) + if args.interactive: + pause( net, "Test is finished! Exit cli to exit test.\n" ) + +def Scaling( net ): + + def startNodes( net, nodes ): + "start multiple ONOS nodes" + cluster = net.controllers[0] + cluster.activeNodes.extend( nodes ) + cluster.activeNodes = sorted( set( cluster.activeNodes ) ) + for node in nodes: + node.shouldStart = True + node.start( cluster.env, cluster.activeNodes ) + for node in nodes: + node.waitStarted() + + # control net objects + cluster = net.controllers[0] + cnet = cluster.net + cs1 = cnet.switches[0] + + info( "ONOS dynamic clustering scaling test\n") + # Start the first node + cluster.activeNodes.append( cnet.hosts[0] ) + cluster.activeNodes = sorted( set( cluster.activeNodes ) ) + startNodes( net, cluster.activeNodes ) + + onos_all( net, "nodes;partitions;partitions -c") + if args.interactive: + pause( net, "Dropping into cli... Exit cli to continue test\n" ) + + # Scale up by two + while True: + new = [ n for c in net.controllers for n in c.net.hosts if isinstance( n, DynamicONOSNode) and not n.started ][:2] + if not new: + break + startNodes( net, new ) + onos_all( net, "nodes;partitions;partitions -c") + if args.interactive: + pause( net, "Dropping into cli... Exit cli to continue test\n" ) + + # Scale down + for i in range( len( cluster.activeNodes ) - 1 ): + node = cluster.activeNodes.pop() + node.genPartitions( cluster.activeNodes, node.metadata ) + onos_all( net, "nodes;partitions;partitions -c") + if args.interactive: + pause( net, "Dropping into cli... Exit cli to continue test\n" ) + if args.interactive: + pause( net, "Test is finished! Exit cli to exit test.\n" ) + + + +# Mininet object subclasses + +class HTTP( Host ): + def __init__( self, *args, **kwargs ): + super( HTTP, self).__init__( *args, **kwargs ) + self.dir = '/tmp/%s' % self.name + self.cmd( 'rm -rf', self.dir ) + self.cmd( 'mkdir', self.dir ) + self.cmd( 'cd', self.dir ) + + def start( self ): + output( "(starting HTTP Server)" ) + # start python web server as a bg process + self.cmd( 'python -m SimpleHTTPServer &> web.log &' ) + + def stop( self ): + # XXX is this ever called? + print "Stopping HTTP Server..." + print self.cmd( 'fg' ) + print self.cmd( '\x03' ) # ctrl-c + + +class DynamicONOSNode( onos.ONOSNode ): + def __init__( self, *args, **kwargs ): + self.shouldStart = False + self.started = False + self.metadata = '/tmp/cluster.json' + super( DynamicONOSNode, self ).__init__( *args, **kwargs ) + # XXX HACK, need to get this passed in correctly + self.alertAction = 'warn' + + def start( self, env, nodes=()): + if not self.shouldStart: + return + elif self.started: + return + else: + ##### Modified from base class + env = dict( env ) + env.update( ONOS_HOME=self.ONOS_HOME ) + if self.remote: + # Point onos to rewmote cluster metadata file + ip = self.remote.get( 'ip', '127.0.0.1' ) + port = self.remote.get( 'port', '8000' ) + filename = self.remote.get( 'filename', 'cluster.json' ) + remote = 'http://%s:%s/%s' % ( ip, port, filename ) + uri = '-Donos.cluster.metadata.uri=%s' % remote + prev = env.get( 'JAVA_OPTS', False ) + if prev: + jarg = ':'.join( [prev, uri] ) + else: + jarg = uri + env.update( JAVA_OPTS=jarg ) + self.updateEnv( env ) + karafbin = glob( '%s/apache*/bin' % self.ONOS_HOME )[ 0 ] + onosbin = join( self.ONOS_ROOT, 'tools/test/bin' ) + self.cmd( 'export PATH=%s:%s:$PATH' % ( onosbin, karafbin ) ) + self.cmd( 'cd', self.ONOS_HOME ) + self.ucmd( 'mkdir -p config ' ) + self.genPartitions( nodes, self.metadata ) + info( '(starting %s)' % self ) + service = join( self.ONOS_HOME, 'bin/onos-service' ) + self.ucmd( service, 'server 1>../onos.log 2>../onos.log' + ' & echo $! > onos.pid; ln -s `pwd`/onos.pid ..' ) + self.onosPid = int( self.cmd( 'cat onos.pid' ).strip() ) + self.warningCount = 0 + #### + self.started=True + + def sanityCheck( self, lowMem=100000 ): + if self.started: + super( DynamicONOSNode, self ).sanityCheck( lowMem ) + + def waitStarted( self ): + if self.started: + super( DynamicONOSNode, self ).waitStarted() + + def genPartitions( self, nodes, location='/tmp/cluster.json' ): + """ + Generate a cluster metadata file for dynamic clustering. + Note: name should be the same in different versions of the file as + well as the number of partitions. + """ + def genParts( nodes, k, parts=3): + l = deque( nodes ) + perms = [] + for i in range( 1, parts + 1 ): + part = { + 'id': i, + 'members': list(l)[:k] + } + perms.append( part ) + l.rotate( -1 ) + return perms + + print "Generating %s with %s" % ( location, str(nodes) ) + port = 9876 + ips = [ node.IP() for node in nodes ] + node = lambda k: { 'id': k, 'ip': k, 'port': port } + m = hashlib.sha256( "Mininet based ONOS test" ) + name = int(m.hexdigest()[:8], base=16 ) + partitions = genParts( ips, 3 ) + data = { + 'name': name, + 'nodes': [ node(v) for v in ips ], + 'partitions': partitions + } + output = json.dumps( data, indent=4 ) + with open( location, 'w' ) as f: + f.write( output ) + cprint( output, "yellow" ) + + +class DynamicONOSCluster( onos.ONOSCluster ): + def __init__( self, *args, **kwargs ): + self.activeNodes = [] + # TODO: can we get super to use super's nodes()? + super( DynamicONOSCluster, self ).__init__( *args, **kwargs ) + self.activeNodes = [ h for h in self.net.hosts if onos.isONOSNode( h ) ] + onos.updateNodeIPs( self.env, self.nodes() ) + self.activeNodes = [] + + def start( self ): + "Start up ONOS control network" + info( '*** ONOS_APPS = %s\n' % onos.ONOS_APPS ) + self.net.start() + for node in self.net.hosts: + if onos.isONOSNode( node ): + node.start( self.env, self.nodes() ) + else: + try: + node.start() + except AttributeError: + # NAT doesn't have start? + pass + info( '\n' ) + self.configPortForwarding( ports=self.forward, action='A' ) + self.waitStarted() + return + + def nodes( self ): + "Return list of ONOS nodes that should be running" + return self.activeNodes + +class HATopo( Topo ): + def build( self, partitions=[], serverCount=1, dynamic=False, **kwargs ): + """ + partitions = a list of strings specifing the assignment of onos nodes + to regions. ['1', '2,3'] designates two regions, with + ONOS 1 in the first and ONOS 2 and 3 in the second. + serverCount = If partitions is not given, then the number of ONOS + nodes to create + dynamic = A boolean indicating dynamic ONOS clustering + """ + self.switchNum = 1 + if dynamic: + cls = DynamicONOSNode + else: + cls = onos.ONOSNode + if partitions: + prev = None + for partition in partitions: + # Create a region of ONOS nodes connected to a switch + # FIXME Check for nodes that are not assigned to a partition? + cur = self.addRegion( partition, cls ) + + # Connect switch to previous switch + if prev: + self.addLink( prev, cur ) + prev = cur + else: + partition = ','.join( [ str( x ) for x in range( 1, serverCount + 1 ) ] ) + cs1 = self.addRegion( partition, cls ) + if dynamic: + # TODO Pass these in + scale = 2 + new = ','.join( [ str( x + 1 ) for x in range( serverCount , serverCount + scale ) ] ) + cs2 = self.addRegion( new, cls ) + self.addLink( cs1, cs2 ) + server = self.addHost( "server", cls=HTTP ) + for switch in self.switches(): + self.addLink( server, switch ) + + def addRegion( self, partition, cls=onos.ONOSNode ): + switch = self.addSwitch( 'cs%s' % self.switchNum ) + self.switchNum += 1 + for n in partition.split( ',' ): + node = self.addHost( "onos" + str( n ), cls=cls ) + self.addLink( switch, node ) + return switch + + +CLI = onos.ONOSCLI + +# The main runner +def runTest( args ): + test = None + if args.test == "partition": + test=Partition + serverCount = args.nodes + # NOTE we are ignoring serverCount for this test, using partition assignment instead. + topo = HATopo( partitions=args.partition ) + # FIXME Configurable dataplane topology + net = Mininet( topo=SingleSwitchTopo( 3 ), + controller=[ onos.ONOSCluster( 'c0', topo=topo, alertAction='warn' ) ], + switch=onos.ONOSOVSSwitch ) + elif args.test == "scaling": + test=Scaling + serverCount = args.nodes + topo = HATopo( serverCount=serverCount, dynamic=True ) + net = Mininet( topo=SingleSwitchTopo( 3 ), + controller=[ DynamicONOSCluster( 'c0', topo=topo, alertAction='warn' ) ], + switch=onos.ONOSOVSSwitch ) + cluster = net.controllers[0] + cnet = cluster.net + server = cnet.get( 'server' ) + remote = { 'ip': server.IP(), + 'port': '8000', + 'filename':'cluster.json' } + for node in cnet.hosts: + if isinstance( node, DynamicONOSNode ): + node.metadata = '%s/cluster.json' % server.dir + node.remote = remote + ips = [] + cluster.activeNodes = [ cnet.get( "onos%s" % ( i + 1 ) ) for i in range( serverCount ) ] + for node in cluster.activeNodes: + node.shouldStart = True + else: + print "Incorrect test" + return + net.start() + if args.interactive: + CLI( net ) + test(net) + CLI( net ) + net.stop() + + +if __name__ == '__main__': + setLogLevel( 'info' ) + # Base parser + parser= argparse.ArgumentParser( + description='Mininet based HA tests for ONOS. For more detailed help on a test include the test option' ) + parser.add_argument( + '-n', '--nodes', metavar="NODES", type=int, default=1, + help="Number of nodes in the ONOS cluster" ) + parser.add_argument( + '-i', '--interactive',# type=bool, + default=False, action="store_true", + help="Pause the test in between steps" ) + test_parsers=parser.add_subparsers( title="Tests", help="Types of HA tests", dest="test" ) + + # Partition test parser + partition_help = 'Network partition test. Each set of ONOS nodes is connected to their own switch in the control network. Partitions are introduced by removing links between control network switches.' + partition_parser = test_parsers.add_parser( + "partition", description=partition_help ) + partition_parser.add_argument( + '-p', '--partition', metavar='Partition', required=True, + type=str, nargs=2, + help='Specify the membership for two partitions by node id. Nodes are comma separated and node count begins at 1. E.g. "1,3 2" will create a network with 3 ONOS nodes and two connected switches. Switch 1 will be connected to ONOS1 and ONOS3 while switch 2 will be connected to ONOS2. A partition will be created by disconnecting the two switches. All ONOS nodes will still be connected to the dataplane.' ) + + # Dynamic scaling test parser + # FIXME Replace with real values + scaling_parser = test_parsers.add_parser( "scaling" ) + + args = parser.parse_args() + runTest( args ) diff --git a/tools/dev/mininet/onos.py b/tools/dev/mininet/onos.py index 1f69e0f44b..051b3b3fb2 100755 --- a/tools/dev/mininet/onos.py +++ b/tools/dev/mininet/onos.py @@ -117,7 +117,8 @@ def updateNodeIPs( env, nodes ): for index, node in enumerate( nodes, 1 ): var = 'OC%d' % index env[ var ] = node.IP() - env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ] + if nodes: + env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ] env[ 'ONOS_INSTANCES' ] = '\n'.join( node.IP() for node in nodes ) environ.update( env ) @@ -242,6 +243,7 @@ class ONOSNode( Controller ): self.ONOS_HOME = '/tmp' self.cmd( 'rm -rf', self.dir ) self.ONOS_HOME = unpackONOS( self.dir, run=self.ucmd ) + self.ONOS_ROOT = ONOS_ROOT # pylint: disable=arguments-differ