mirror of
https://github.com/opennetworkinglab/onos.git
synced 2026-05-05 04:06:49 +02:00
Add HA.py
A set of mininet based HA tests based on onos.py
Currently includes the following tests:
- a control network partitioning test
- A dynamic cluster scaling test
Change-Id: I9a8e1019dcc51666fee1d933afd66ff390592525
This commit is contained in:
Jon Hall
Gerrit Code Review
parent
37a396ac37
commit
2548559590
452
tools/dev/mininet/examples/ha.py
Executable file
452
tools/dev/mininet/examples/ha.py
Executable file
@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
NOTES
|
||||
|
||||
To change onos log levels before start you can add something similar to
|
||||
onos.py's ONOSNode.start method before starting onos service:
|
||||
# Change log levels
|
||||
self.ucmd( 'echo "log4j.logger.io.atomix= DEBUG" >> $ONOS_HOME/apache-karaf-*/etc/org.ops4j.pax.logging.cfg' )
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from mininet.log import output, info, warn, error, debug, setLogLevel
|
||||
from mininet.cli import CLI as origCLI
|
||||
from mininet.net import Mininet
|
||||
from mininet.topo import SingleSwitchTopo, Topo
|
||||
from mininet.node import Host
|
||||
from os.path import join
|
||||
from glob import glob
|
||||
import re
|
||||
import json
|
||||
from collections import deque
|
||||
import hashlib
|
||||
import onos # onos.py
|
||||
|
||||
# Utility functions
|
||||
|
||||
def pause( net, msg, hint=False):
|
||||
"""Reenter the CLI. Note that we use the mn base CLI class to allow
|
||||
extensibility and combination of custom files"""
|
||||
|
||||
info( msg )
|
||||
|
||||
if hint:
|
||||
help_msg = "Currently in the root Mininet net namespace...\n"
|
||||
help_msg += "To access control net functions use:\n"
|
||||
help_msg += "\tpx cnet=net.controllers[0].net\n"
|
||||
help_msg += "\tpy cnet.METHOD\n"
|
||||
help_msg += "To send commands to each onos node, use: onos_all CMD\n"
|
||||
help_msg += "\nBy default, ONOS nodes are running on the 192.168.123.X network\n"
|
||||
info( "%s\n" % help_msg )
|
||||
# NOTE: If we use onos.py as a custom file and as an imported module,
|
||||
# we get two different sets of ONOS* classes. They don't play
|
||||
# together and things don't work properly. Specifically the
|
||||
# isinstance calls fail. This is due to import vs. exec calls
|
||||
onos.ONOSCLI( net )
|
||||
|
||||
def cprint( msg, color="default"):
|
||||
color=color.lower()
|
||||
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
|
||||
'blue': '\033[94m', 'green': '\033[92m',
|
||||
'yellow': '\033[93m', 'red': '\033[91m',
|
||||
'end': '\033[0m' }
|
||||
pre = colors.get( color, '' )
|
||||
output = pre + msg + colors['end']
|
||||
print( output )
|
||||
|
||||
def getNode( net, nodeId=0 ):
|
||||
"Helper function: return ONOS node, defaults to the first node"
|
||||
return net.controllers[ 0 ].nodes()[ nodeId ]
|
||||
|
||||
def onos_cli( net, line, nodeId=0 ):
|
||||
"Send command to ONOS CLI"
|
||||
c0 = net.controllers[ 0 ]
|
||||
# FIXME add this back after import onos.py works
|
||||
if isinstance( c0, onos.ONOSCluster ):
|
||||
# cmdLoop strips off command name 'onos'
|
||||
if line.startswith( ':' ):
|
||||
line = 'onos' + line
|
||||
node = getNode( net, nodeId )
|
||||
if line:
|
||||
line = '"%s"' % line
|
||||
cmd = 'client -h %s %s' % ( node.IP(), line )
|
||||
#node.cmdPrint( cmd )
|
||||
output = node.cmd( cmd )
|
||||
info( line )
|
||||
# Remove verbose spam from output
|
||||
m = re.search( "unverified \{\} key: \{\}", output )
|
||||
if m:
|
||||
info( output[m.end():] )
|
||||
else:
|
||||
info( output )
|
||||
|
||||
def onos_all( net, line ):
|
||||
onosNodes = [ n for cluster in net.controllers for n in cluster.nodes() ]
|
||||
for node in range( len( onosNodes ) ):
|
||||
cprint( "*" * 53, "red" )
|
||||
cprint( "onos%s: %s" % ( str( node + 1 ), repr( onosNodes[ node ] ) ),
|
||||
"red" )
|
||||
cprint( "*" * 53, "red" )
|
||||
onos_cli( net, line, node )
|
||||
|
||||
# FIXME This needs a better name
|
||||
def do_onos_all( self, line ):
|
||||
onos_all( self.mn, line)
|
||||
|
||||
# Add custom cli commands
|
||||
# NOTE: This is so we can keep ONOSCLI and also add commands to it!
|
||||
origCLI.do_onos_all = do_onos_all
|
||||
|
||||
# Test cases
|
||||
|
||||
def Partition( net ):
|
||||
# Controller net instance
|
||||
cnet = net.controllers[0].net
|
||||
|
||||
info( "ONOS control network partition test\n")
|
||||
net.pingAll()
|
||||
if args.interactive:
|
||||
pause( net, "~~~ Dropping into cli... Exit cli to continue test\n", True )
|
||||
|
||||
onos_all( net, "nodes;partitions;partitions -c")
|
||||
info( "~~~ Right before the partitioned\n" )
|
||||
if args.interactive:
|
||||
pause( net, "Dropping into cli... Exit cli to continue test\n" )
|
||||
|
||||
cs1 = cnet.switches[0]
|
||||
cs2 = cnet.switches[1]
|
||||
|
||||
# PARTITION sub-clusters
|
||||
|
||||
# we need to use names here
|
||||
cnet.configLinkStatus( cs1.name, cs2.name, "down" )
|
||||
onos_all( net, "nodes;partitions;partitions -c")
|
||||
info( "~~~ Right after cluster is partitioned. Next step is to heal the partition\n" )
|
||||
if args.interactive:
|
||||
pause( net, "Dropping into cli... Exit cli to continue test\n" )
|
||||
|
||||
cnet.configLinkStatus( cs1.name, cs2.name, "up" )
|
||||
onos_all( net, "nodes;partitions;partitions -c")
|
||||
info( "~~~ Right after the partition is healed \n" )
|
||||
if args.interactive:
|
||||
pause( net, "Test is finished! Exit cli to exit test.\n" )
|
||||
|
||||
def Scaling( net ):
|
||||
|
||||
def startNodes( net, nodes ):
|
||||
"start multiple ONOS nodes"
|
||||
cluster = net.controllers[0]
|
||||
cluster.activeNodes.extend( nodes )
|
||||
cluster.activeNodes = sorted( set( cluster.activeNodes ) )
|
||||
for node in nodes:
|
||||
node.shouldStart = True
|
||||
node.start( cluster.env, cluster.activeNodes )
|
||||
for node in nodes:
|
||||
node.waitStarted()
|
||||
|
||||
# control net objects
|
||||
cluster = net.controllers[0]
|
||||
cnet = cluster.net
|
||||
cs1 = cnet.switches[0]
|
||||
|
||||
info( "ONOS dynamic clustering scaling test\n")
|
||||
# Start the first node
|
||||
cluster.activeNodes.append( cnet.hosts[0] )
|
||||
cluster.activeNodes = sorted( set( cluster.activeNodes ) )
|
||||
startNodes( net, cluster.activeNodes )
|
||||
|
||||
onos_all( net, "nodes;partitions;partitions -c")
|
||||
if args.interactive:
|
||||
pause( net, "Dropping into cli... Exit cli to continue test\n" )
|
||||
|
||||
# Scale up by two
|
||||
while True:
|
||||
new = [ n for c in net.controllers for n in c.net.hosts if isinstance( n, DynamicONOSNode) and not n.started ][:2]
|
||||
if not new:
|
||||
break
|
||||
startNodes( net, new )
|
||||
onos_all( net, "nodes;partitions;partitions -c")
|
||||
if args.interactive:
|
||||
pause( net, "Dropping into cli... Exit cli to continue test\n" )
|
||||
|
||||
# Scale down
|
||||
for i in range( len( cluster.activeNodes ) - 1 ):
|
||||
node = cluster.activeNodes.pop()
|
||||
node.genPartitions( cluster.activeNodes, node.metadata )
|
||||
onos_all( net, "nodes;partitions;partitions -c")
|
||||
if args.interactive:
|
||||
pause( net, "Dropping into cli... Exit cli to continue test\n" )
|
||||
if args.interactive:
|
||||
pause( net, "Test is finished! Exit cli to exit test.\n" )
|
||||
|
||||
|
||||
|
||||
# Mininet object subclasses
|
||||
|
||||
class HTTP( Host ):
|
||||
def __init__( self, *args, **kwargs ):
|
||||
super( HTTP, self).__init__( *args, **kwargs )
|
||||
self.dir = '/tmp/%s' % self.name
|
||||
self.cmd( 'rm -rf', self.dir )
|
||||
self.cmd( 'mkdir', self.dir )
|
||||
self.cmd( 'cd', self.dir )
|
||||
|
||||
def start( self ):
|
||||
output( "(starting HTTP Server)" )
|
||||
# start python web server as a bg process
|
||||
self.cmd( 'python -m SimpleHTTPServer &> web.log &' )
|
||||
|
||||
def stop( self ):
|
||||
# XXX is this ever called?
|
||||
print "Stopping HTTP Server..."
|
||||
print self.cmd( 'fg' )
|
||||
print self.cmd( '\x03' ) # ctrl-c
|
||||
|
||||
|
||||
class DynamicONOSNode( onos.ONOSNode ):
|
||||
def __init__( self, *args, **kwargs ):
|
||||
self.shouldStart = False
|
||||
self.started = False
|
||||
self.metadata = '/tmp/cluster.json'
|
||||
super( DynamicONOSNode, self ).__init__( *args, **kwargs )
|
||||
# XXX HACK, need to get this passed in correctly
|
||||
self.alertAction = 'warn'
|
||||
|
||||
def start( self, env, nodes=()):
|
||||
if not self.shouldStart:
|
||||
return
|
||||
elif self.started:
|
||||
return
|
||||
else:
|
||||
##### Modified from base class
|
||||
env = dict( env )
|
||||
env.update( ONOS_HOME=self.ONOS_HOME )
|
||||
if self.remote:
|
||||
# Point onos to rewmote cluster metadata file
|
||||
ip = self.remote.get( 'ip', '127.0.0.1' )
|
||||
port = self.remote.get( 'port', '8000' )
|
||||
filename = self.remote.get( 'filename', 'cluster.json' )
|
||||
remote = 'http://%s:%s/%s' % ( ip, port, filename )
|
||||
uri = '-Donos.cluster.metadata.uri=%s' % remote
|
||||
prev = env.get( 'JAVA_OPTS', False )
|
||||
if prev:
|
||||
jarg = ':'.join( [prev, uri] )
|
||||
else:
|
||||
jarg = uri
|
||||
env.update( JAVA_OPTS=jarg )
|
||||
self.updateEnv( env )
|
||||
karafbin = glob( '%s/apache*/bin' % self.ONOS_HOME )[ 0 ]
|
||||
onosbin = join( self.ONOS_ROOT, 'tools/test/bin' )
|
||||
self.cmd( 'export PATH=%s:%s:$PATH' % ( onosbin, karafbin ) )
|
||||
self.cmd( 'cd', self.ONOS_HOME )
|
||||
self.ucmd( 'mkdir -p config ' )
|
||||
self.genPartitions( nodes, self.metadata )
|
||||
info( '(starting %s)' % self )
|
||||
service = join( self.ONOS_HOME, 'bin/onos-service' )
|
||||
self.ucmd( service, 'server 1>../onos.log 2>../onos.log'
|
||||
' & echo $! > onos.pid; ln -s `pwd`/onos.pid ..' )
|
||||
self.onosPid = int( self.cmd( 'cat onos.pid' ).strip() )
|
||||
self.warningCount = 0
|
||||
####
|
||||
self.started=True
|
||||
|
||||
def sanityCheck( self, lowMem=100000 ):
|
||||
if self.started:
|
||||
super( DynamicONOSNode, self ).sanityCheck( lowMem )
|
||||
|
||||
def waitStarted( self ):
|
||||
if self.started:
|
||||
super( DynamicONOSNode, self ).waitStarted()
|
||||
|
||||
def genPartitions( self, nodes, location='/tmp/cluster.json' ):
|
||||
"""
|
||||
Generate a cluster metadata file for dynamic clustering.
|
||||
Note: name should be the same in different versions of the file as
|
||||
well as the number of partitions.
|
||||
"""
|
||||
def genParts( nodes, k, parts=3):
|
||||
l = deque( nodes )
|
||||
perms = []
|
||||
for i in range( 1, parts + 1 ):
|
||||
part = {
|
||||
'id': i,
|
||||
'members': list(l)[:k]
|
||||
}
|
||||
perms.append( part )
|
||||
l.rotate( -1 )
|
||||
return perms
|
||||
|
||||
print "Generating %s with %s" % ( location, str(nodes) )
|
||||
port = 9876
|
||||
ips = [ node.IP() for node in nodes ]
|
||||
node = lambda k: { 'id': k, 'ip': k, 'port': port }
|
||||
m = hashlib.sha256( "Mininet based ONOS test" )
|
||||
name = int(m.hexdigest()[:8], base=16 )
|
||||
partitions = genParts( ips, 3 )
|
||||
data = {
|
||||
'name': name,
|
||||
'nodes': [ node(v) for v in ips ],
|
||||
'partitions': partitions
|
||||
}
|
||||
output = json.dumps( data, indent=4 )
|
||||
with open( location, 'w' ) as f:
|
||||
f.write( output )
|
||||
cprint( output, "yellow" )
|
||||
|
||||
|
||||
class DynamicONOSCluster( onos.ONOSCluster ):
|
||||
def __init__( self, *args, **kwargs ):
|
||||
self.activeNodes = []
|
||||
# TODO: can we get super to use super's nodes()?
|
||||
super( DynamicONOSCluster, self ).__init__( *args, **kwargs )
|
||||
self.activeNodes = [ h for h in self.net.hosts if onos.isONOSNode( h ) ]
|
||||
onos.updateNodeIPs( self.env, self.nodes() )
|
||||
self.activeNodes = []
|
||||
|
||||
def start( self ):
|
||||
"Start up ONOS control network"
|
||||
info( '*** ONOS_APPS = %s\n' % onos.ONOS_APPS )
|
||||
self.net.start()
|
||||
for node in self.net.hosts:
|
||||
if onos.isONOSNode( node ):
|
||||
node.start( self.env, self.nodes() )
|
||||
else:
|
||||
try:
|
||||
node.start()
|
||||
except AttributeError:
|
||||
# NAT doesn't have start?
|
||||
pass
|
||||
info( '\n' )
|
||||
self.configPortForwarding( ports=self.forward, action='A' )
|
||||
self.waitStarted()
|
||||
return
|
||||
|
||||
def nodes( self ):
|
||||
"Return list of ONOS nodes that should be running"
|
||||
return self.activeNodes
|
||||
|
||||
class HATopo( Topo ):
|
||||
def build( self, partitions=[], serverCount=1, dynamic=False, **kwargs ):
|
||||
"""
|
||||
partitions = a list of strings specifing the assignment of onos nodes
|
||||
to regions. ['1', '2,3'] designates two regions, with
|
||||
ONOS 1 in the first and ONOS 2 and 3 in the second.
|
||||
serverCount = If partitions is not given, then the number of ONOS
|
||||
nodes to create
|
||||
dynamic = A boolean indicating dynamic ONOS clustering
|
||||
"""
|
||||
self.switchNum = 1
|
||||
if dynamic:
|
||||
cls = DynamicONOSNode
|
||||
else:
|
||||
cls = onos.ONOSNode
|
||||
if partitions:
|
||||
prev = None
|
||||
for partition in partitions:
|
||||
# Create a region of ONOS nodes connected to a switch
|
||||
# FIXME Check for nodes that are not assigned to a partition?
|
||||
cur = self.addRegion( partition, cls )
|
||||
|
||||
# Connect switch to previous switch
|
||||
if prev:
|
||||
self.addLink( prev, cur )
|
||||
prev = cur
|
||||
else:
|
||||
partition = ','.join( [ str( x ) for x in range( 1, serverCount + 1 ) ] )
|
||||
cs1 = self.addRegion( partition, cls )
|
||||
if dynamic:
|
||||
# TODO Pass these in
|
||||
scale = 2
|
||||
new = ','.join( [ str( x + 1 ) for x in range( serverCount , serverCount + scale ) ] )
|
||||
cs2 = self.addRegion( new, cls )
|
||||
self.addLink( cs1, cs2 )
|
||||
server = self.addHost( "server", cls=HTTP )
|
||||
for switch in self.switches():
|
||||
self.addLink( server, switch )
|
||||
|
||||
def addRegion( self, partition, cls=onos.ONOSNode ):
|
||||
switch = self.addSwitch( 'cs%s' % self.switchNum )
|
||||
self.switchNum += 1
|
||||
for n in partition.split( ',' ):
|
||||
node = self.addHost( "onos" + str( n ), cls=cls )
|
||||
self.addLink( switch, node )
|
||||
return switch
|
||||
|
||||
|
||||
CLI = onos.ONOSCLI
|
||||
|
||||
# The main runner
|
||||
def runTest( args ):
|
||||
test = None
|
||||
if args.test == "partition":
|
||||
test=Partition
|
||||
serverCount = args.nodes
|
||||
# NOTE we are ignoring serverCount for this test, using partition assignment instead.
|
||||
topo = HATopo( partitions=args.partition )
|
||||
# FIXME Configurable dataplane topology
|
||||
net = Mininet( topo=SingleSwitchTopo( 3 ),
|
||||
controller=[ onos.ONOSCluster( 'c0', topo=topo, alertAction='warn' ) ],
|
||||
switch=onos.ONOSOVSSwitch )
|
||||
elif args.test == "scaling":
|
||||
test=Scaling
|
||||
serverCount = args.nodes
|
||||
topo = HATopo( serverCount=serverCount, dynamic=True )
|
||||
net = Mininet( topo=SingleSwitchTopo( 3 ),
|
||||
controller=[ DynamicONOSCluster( 'c0', topo=topo, alertAction='warn' ) ],
|
||||
switch=onos.ONOSOVSSwitch )
|
||||
cluster = net.controllers[0]
|
||||
cnet = cluster.net
|
||||
server = cnet.get( 'server' )
|
||||
remote = { 'ip': server.IP(),
|
||||
'port': '8000',
|
||||
'filename':'cluster.json' }
|
||||
for node in cnet.hosts:
|
||||
if isinstance( node, DynamicONOSNode ):
|
||||
node.metadata = '%s/cluster.json' % server.dir
|
||||
node.remote = remote
|
||||
ips = []
|
||||
cluster.activeNodes = [ cnet.get( "onos%s" % ( i + 1 ) ) for i in range( serverCount ) ]
|
||||
for node in cluster.activeNodes:
|
||||
node.shouldStart = True
|
||||
else:
|
||||
print "Incorrect test"
|
||||
return
|
||||
net.start()
|
||||
if args.interactive:
|
||||
CLI( net )
|
||||
test(net)
|
||||
CLI( net )
|
||||
net.stop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
setLogLevel( 'info' )
|
||||
# Base parser
|
||||
parser= argparse.ArgumentParser(
|
||||
description='Mininet based HA tests for ONOS. For more detailed help on a test include the test option' )
|
||||
parser.add_argument(
|
||||
'-n', '--nodes', metavar="NODES", type=int, default=1,
|
||||
help="Number of nodes in the ONOS cluster" )
|
||||
parser.add_argument(
|
||||
'-i', '--interactive',# type=bool,
|
||||
default=False, action="store_true",
|
||||
help="Pause the test in between steps" )
|
||||
test_parsers=parser.add_subparsers( title="Tests", help="Types of HA tests", dest="test" )
|
||||
|
||||
# Partition test parser
|
||||
partition_help = 'Network partition test. Each set of ONOS nodes is connected to their own switch in the control network. Partitions are introduced by removing links between control network switches.'
|
||||
partition_parser = test_parsers.add_parser(
|
||||
"partition", description=partition_help )
|
||||
partition_parser.add_argument(
|
||||
'-p', '--partition', metavar='Partition', required=True,
|
||||
type=str, nargs=2,
|
||||
help='Specify the membership for two partitions by node id. Nodes are comma separated and node count begins at 1. E.g. "1,3 2" will create a network with 3 ONOS nodes and two connected switches. Switch 1 will be connected to ONOS1 and ONOS3 while switch 2 will be connected to ONOS2. A partition will be created by disconnecting the two switches. All ONOS nodes will still be connected to the dataplane.' )
|
||||
|
||||
# Dynamic scaling test parser
|
||||
# FIXME Replace with real values
|
||||
scaling_parser = test_parsers.add_parser( "scaling" )
|
||||
|
||||
args = parser.parse_args()
|
||||
runTest( args )
|
||||
@ -117,7 +117,8 @@ def updateNodeIPs( env, nodes ):
|
||||
for index, node in enumerate( nodes, 1 ):
|
||||
var = 'OC%d' % index
|
||||
env[ var ] = node.IP()
|
||||
env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ]
|
||||
if nodes:
|
||||
env[ 'OCI' ] = env[ 'OCN' ] = env[ 'OC1' ]
|
||||
env[ 'ONOS_INSTANCES' ] = '\n'.join(
|
||||
node.IP() for node in nodes )
|
||||
environ.update( env )
|
||||
@ -242,6 +243,7 @@ class ONOSNode( Controller ):
|
||||
self.ONOS_HOME = '/tmp'
|
||||
self.cmd( 'rm -rf', self.dir )
|
||||
self.ONOS_HOME = unpackONOS( self.dir, run=self.ucmd )
|
||||
self.ONOS_ROOT = ONOS_ROOT
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user