#!/usr/bin/env python """ NOTES To change onos log levels before start you can add something similar to onos.py's ONOSNode.start method before starting onos service: # Change log levels self.ucmd( 'echo "log4j.logger.io.atomix= DEBUG" >> $ONOS_HOME/apache-karaf-*/etc/org.ops4j.pax.logging.cfg' ) """ import argparse from mininet.log import output, info, warn, error, debug, setLogLevel from mininet.cli import CLI as origCLI from mininet.net import Mininet from mininet.topo import SingleSwitchTopo, Topo from mininet.node import Host from os.path import join from glob import glob import re import json from collections import deque import hashlib import onos # onos.py # Utility functions def pause( net, msg, hint=False): """Reenter the CLI. Note that we use the mn base CLI class to allow extensibility and combination of custom files""" info( msg ) if hint: help_msg = "Currently in the root Mininet net namespace...\n" help_msg += "To access control net functions use:\n" help_msg += "\tpx cnet=net.controllers[0].net\n" help_msg += "\tpy cnet.METHOD\n" help_msg += "To send commands to each onos node, use: onos_all CMD\n" help_msg += "\nBy default, ONOS nodes are running on the 192.168.123.X network\n" info( "%s\n" % help_msg ) # NOTE: If we use onos.py as a custom file and as an imported module, # we get two different sets of ONOS* classes. They don't play # together and things don't work properly. Specifically the # isinstance calls fail. This is due to import vs. exec calls onos.ONOSCLI( net ) def cprint( msg, color="default"): color=color.lower() colors = { 'cyan': '\033[96m', 'purple': '\033[95m', 'blue': '\033[94m', 'green': '\033[92m', 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' } pre = colors.get( color, '' ) output = pre + msg + colors['end'] print( output ) def getNode( net, nodeId=0 ): "Helper function: return ONOS node, defaults to the first node" return net.controllers[ 0 ].nodes()[ nodeId ] def onos_cli( net, line, nodeId=0 ): "Send command to ONOS CLI" c0 = net.controllers[ 0 ] # FIXME add this back after import onos.py works if isinstance( c0, onos.ONOSCluster ): # cmdLoop strips off command name 'onos' if line.startswith( ':' ): line = 'onos' + line node = getNode( net, nodeId ) if line: line = '"%s"' % line cmd = 'client -h %s %s' % ( node.IP(), line ) #node.cmdPrint( cmd ) output = node.cmd( cmd ) info( line ) # Remove verbose spam from output m = re.search( "unverified \{\} key: \{\}", output ) if m: info( output[m.end():] ) else: info( output ) def onos_all( net, line ): onosNodes = [ n for cluster in net.controllers for n in cluster.nodes() ] for node in range( len( onosNodes ) ): cprint( "*" * 53, "red" ) cprint( "onos%s: %s" % ( str( node + 1 ), repr( onosNodes[ node ] ) ), "red" ) cprint( "*" * 53, "red" ) onos_cli( net, line, node ) # FIXME This needs a better name def do_onos_all( self, line ): onos_all( self.mn, line) # Add custom cli commands # NOTE: This is so we can keep ONOSCLI and also add commands to it! origCLI.do_onos_all = do_onos_all # Test cases def Partition( net ): # Controller net instance cnet = net.controllers[0].net info( "ONOS control network partition test\n") net.pingAll() if args.interactive: pause( net, "~~~ Dropping into cli... Exit cli to continue test\n", True ) onos_all( net, "nodes;partitions;partitions -c") info( "~~~ Right before the partitioned\n" ) if args.interactive: pause( net, "Dropping into cli... Exit cli to continue test\n" ) cs1 = cnet.switches[0] cs2 = cnet.switches[1] # PARTITION sub-clusters # we need to use names here cnet.configLinkStatus( cs1.name, cs2.name, "down" ) onos_all( net, "nodes;partitions;partitions -c") info( "~~~ Right after cluster is partitioned. Next step is to heal the partition\n" ) if args.interactive: pause( net, "Dropping into cli... Exit cli to continue test\n" ) cnet.configLinkStatus( cs1.name, cs2.name, "up" ) onos_all( net, "nodes;partitions;partitions -c") info( "~~~ Right after the partition is healed \n" ) if args.interactive: pause( net, "Test is finished! Exit cli to exit test.\n" ) def Scaling( net ): def startNodes( net, nodes ): "start multiple ONOS nodes" cluster = net.controllers[0] cluster.activeNodes.extend( nodes ) cluster.activeNodes = sorted( set( cluster.activeNodes ) ) for node in nodes: node.shouldStart = True node.start( cluster.env, cluster.activeNodes ) for node in nodes: node.waitStarted() # control net objects cluster = net.controllers[0] cnet = cluster.net cs1 = cnet.switches[0] info( "ONOS dynamic clustering scaling test\n") # Start the first node cluster.activeNodes.append( cnet.hosts[0] ) cluster.activeNodes = sorted( set( cluster.activeNodes ) ) startNodes( net, cluster.activeNodes ) onos_all( net, "nodes;partitions;partitions -c") if args.interactive: pause( net, "Dropping into cli... Exit cli to continue test\n" ) # Scale up by two while True: new = [ n for c in net.controllers for n in c.net.hosts if isinstance( n, DynamicONOSNode) and not n.started ][:2] if not new: break startNodes( net, new ) onos_all( net, "nodes;partitions;partitions -c") if args.interactive: pause( net, "Dropping into cli... Exit cli to continue test\n" ) # Scale down for i in range( len( cluster.activeNodes ) - 1 ): node = cluster.activeNodes.pop() node.genPartitions( cluster.activeNodes, node.metadata ) onos_all( net, "nodes;partitions;partitions -c") if args.interactive: pause( net, "Dropping into cli... Exit cli to continue test\n" ) if args.interactive: pause( net, "Test is finished! Exit cli to exit test.\n" ) # Mininet object subclasses class HTTP( Host ): def __init__( self, *args, **kwargs ): super( HTTP, self).__init__( *args, **kwargs ) self.dir = '/tmp/%s' % self.name self.cmd( 'rm -rf', self.dir ) self.cmd( 'mkdir', self.dir ) self.cmd( 'cd', self.dir ) def start( self ): output( "(starting HTTP Server)" ) # start python web server as a bg process self.cmd( 'python -m SimpleHTTPServer &> web.log &' ) def stop( self ): # XXX is this ever called? print "Stopping HTTP Server..." print self.cmd( 'fg' ) print self.cmd( '\x03' ) # ctrl-c class DynamicONOSNode( onos.ONOSNode ): def __init__( self, *args, **kwargs ): self.shouldStart = False self.started = False self.metadata = '/tmp/cluster.json' super( DynamicONOSNode, self ).__init__( *args, **kwargs ) # XXX HACK, need to get this passed in correctly self.alertAction = 'warn' def start( self, env, nodes=()): if not self.shouldStart: return elif self.started: return else: ##### Modified from base class env = dict( env ) env.update( ONOS_HOME=self.ONOS_HOME ) if self.remote: # Point onos to rewmote cluster metadata file ip = self.remote.get( 'ip', '127.0.0.1' ) port = self.remote.get( 'port', '8000' ) filename = self.remote.get( 'filename', 'cluster.json' ) remote = 'http://%s:%s/%s' % ( ip, port, filename ) uri = '-Donos.cluster.metadata.uri=%s' % remote prev = env.get( 'JAVA_OPTS', False ) if prev: jarg = ':'.join( [prev, uri] ) else: jarg = uri env.update( JAVA_OPTS=jarg ) self.updateEnv( env ) karafbin = glob( '%s/apache*/bin' % self.ONOS_HOME )[ 0 ] onosbin = join( self.ONOS_ROOT, 'tools/test/bin' ) self.cmd( 'export PATH=%s:%s:$PATH' % ( onosbin, karafbin ) ) self.cmd( 'cd', self.ONOS_HOME ) self.ucmd( 'mkdir -p config ' ) self.genPartitions( nodes, self.metadata ) info( '(starting %s)' % self ) service = join( self.ONOS_HOME, 'bin/onos-service' ) self.ucmd( service, 'server 1>../onos.log 2>../onos.log' ' & echo $! > onos.pid; ln -s `pwd`/onos.pid ..' ) self.onosPid = int( self.cmd( 'cat onos.pid' ).strip() ) self.warningCount = 0 #### self.started=True def sanityCheck( self, lowMem=100000 ): if self.started: super( DynamicONOSNode, self ).sanityCheck( lowMem ) def waitStarted( self ): if self.started: super( DynamicONOSNode, self ).waitStarted() def genPartitions( self, nodes, location='/tmp/cluster.json' ): """ Generate a cluster metadata file for dynamic clustering. Note: name should be the same in different versions of the file as well as the number of partitions. """ def genParts( nodes, k, parts=3): l = deque( nodes ) perms = [] for i in range( 1, parts + 1 ): part = { 'id': i, 'members': list(l)[:k] } perms.append( part ) l.rotate( -1 ) return perms print "Generating %s with %s" % ( location, str(nodes) ) port = 9876 ips = [ node.IP() for node in nodes ] node = lambda k: { 'id': k, 'ip': k, 'port': port } m = hashlib.sha256( "Mininet based ONOS test" ) name = int(m.hexdigest()[:8], base=16 ) partitions = genParts( ips, 3 ) data = { 'name': name, 'nodes': [ node(v) for v in ips ], 'partitions': partitions } output = json.dumps( data, indent=4 ) with open( location, 'w' ) as f: f.write( output ) cprint( output, "yellow" ) class DynamicONOSCluster( onos.ONOSCluster ): def __init__( self, *args, **kwargs ): self.activeNodes = [] # TODO: can we get super to use super's nodes()? super( DynamicONOSCluster, self ).__init__( *args, **kwargs ) self.activeNodes = [ h for h in self.net.hosts if onos.isONOSNode( h ) ] onos.updateNodeIPs( self.env, self.nodes() ) self.activeNodes = [] def start( self ): "Start up ONOS control network" info( '*** ONOS_APPS = %s\n' % onos.ONOS_APPS ) self.net.start() for node in self.net.hosts: if onos.isONOSNode( node ): node.start( self.env, self.nodes() ) else: try: node.start() except AttributeError: # NAT doesn't have start? pass info( '\n' ) self.configPortForwarding( ports=self.forward, action='A' ) self.waitStarted() return def nodes( self ): "Return list of ONOS nodes that should be running" return self.activeNodes class HATopo( Topo ): def build( self, partitions=[], serverCount=1, dynamic=False, **kwargs ): """ partitions = a list of strings specifing the assignment of onos nodes to regions. ['1', '2,3'] designates two regions, with ONOS 1 in the first and ONOS 2 and 3 in the second. serverCount = If partitions is not given, then the number of ONOS nodes to create dynamic = A boolean indicating dynamic ONOS clustering """ self.switchNum = 1 if dynamic: cls = DynamicONOSNode else: cls = onos.ONOSNode if partitions: prev = None for partition in partitions: # Create a region of ONOS nodes connected to a switch # FIXME Check for nodes that are not assigned to a partition? cur = self.addRegion( partition, cls ) # Connect switch to previous switch if prev: self.addLink( prev, cur ) prev = cur else: partition = ','.join( [ str( x ) for x in range( 1, serverCount + 1 ) ] ) cs1 = self.addRegion( partition, cls ) if dynamic: # TODO Pass these in scale = 2 new = ','.join( [ str( x + 1 ) for x in range( serverCount , serverCount + scale ) ] ) cs2 = self.addRegion( new, cls ) self.addLink( cs1, cs2 ) server = self.addHost( "server", cls=HTTP ) for switch in self.switches(): self.addLink( server, switch ) def addRegion( self, partition, cls=onos.ONOSNode ): switch = self.addSwitch( 'cs%s' % self.switchNum ) self.switchNum += 1 for n in partition.split( ',' ): node = self.addHost( "onos" + str( n ), cls=cls ) self.addLink( switch, node ) return switch CLI = onos.ONOSCLI # The main runner def runTest( args ): test = None if args.test == "partition": test=Partition serverCount = args.nodes # NOTE we are ignoring serverCount for this test, using partition assignment instead. topo = HATopo( partitions=args.partition ) # FIXME Configurable dataplane topology net = Mininet( topo=SingleSwitchTopo( 3 ), controller=[ onos.ONOSCluster( 'c0', topo=topo, alertAction='warn' ) ], switch=onos.ONOSOVSSwitch ) elif args.test == "scaling": test=Scaling serverCount = args.nodes topo = HATopo( serverCount=serverCount, dynamic=True ) net = Mininet( topo=SingleSwitchTopo( 3 ), controller=[ DynamicONOSCluster( 'c0', topo=topo, alertAction='warn' ) ], switch=onos.ONOSOVSSwitch ) cluster = net.controllers[0] cnet = cluster.net server = cnet.get( 'server' ) remote = { 'ip': server.IP(), 'port': '8000', 'filename':'cluster.json' } for node in cnet.hosts: if isinstance( node, DynamicONOSNode ): node.metadata = '%s/cluster.json' % server.dir node.remote = remote ips = [] cluster.activeNodes = [ cnet.get( "onos%s" % ( i + 1 ) ) for i in range( serverCount ) ] for node in cluster.activeNodes: node.shouldStart = True else: print "Incorrect test" return net.start() if args.interactive: CLI( net ) test(net) CLI( net ) net.stop() if __name__ == '__main__': setLogLevel( 'info' ) # Base parser parser= argparse.ArgumentParser( description='Mininet based HA tests for ONOS. For more detailed help on a test include the test option' ) parser.add_argument( '-n', '--nodes', metavar="NODES", type=int, default=1, help="Number of nodes in the ONOS cluster" ) parser.add_argument( '-i', '--interactive',# type=bool, default=False, action="store_true", help="Pause the test in between steps" ) test_parsers=parser.add_subparsers( title="Tests", help="Types of HA tests", dest="test" ) # Partition test parser partition_help = 'Network partition test. Each set of ONOS nodes is connected to their own switch in the control network. Partitions are introduced by removing links between control network switches.' partition_parser = test_parsers.add_parser( "partition", description=partition_help ) partition_parser.add_argument( '-p', '--partition', metavar='Partition', required=True, type=str, nargs=2, help='Specify the membership for two partitions by node id. Nodes are comma separated and node count begins at 1. E.g. "1,3 2" will create a network with 3 ONOS nodes and two connected switches. Switch 1 will be connected to ONOS1 and ONOS3 while switch 2 will be connected to ONOS2. A partition will be created by disconnecting the two switches. All ONOS nodes will still be connected to the dataplane.' ) # Dynamic scaling test parser # FIXME Replace with real values scaling_parser = test_parsers.add_parser( "scaling" ) args = parser.parse_args() runTest( args )