From eae123687ca1eec3f625fce84edbd611593399a3 Mon Sep 17 00:00:00 2001 From: Jian Li Date: Tue, 10 Apr 2018 18:48:32 +0900 Subject: [PATCH] [ONOS-7606] Support ARP broadcast (VxLAN) to handle CP failure Change-Id: Ia0bccf6abaad3e074f2d86a511d5930974743b43 --- .../openstacknetworking/api/Constants.java | 10 + .../impl/OpenstackSwitchingArpHandler.java | 409 +++++++++++++++++- .../OpenstackSwitchingArpHandlerTest.java | 38 ++ 3 files changed, 443 insertions(+), 14 deletions(-) diff --git a/apps/openstacknetworking/api/src/main/java/org/onosproject/openstacknetworking/api/Constants.java b/apps/openstacknetworking/api/src/main/java/org/onosproject/openstacknetworking/api/Constants.java index 131b339a37..136c12eef1 100644 --- a/apps/openstacknetworking/api/src/main/java/org/onosproject/openstacknetworking/api/Constants.java +++ b/apps/openstacknetworking/api/src/main/java/org/onosproject/openstacknetworking/api/Constants.java @@ -27,7 +27,11 @@ public final class Constants { public static final String OPENSTACK_NETWORKING_APP_ID = "org.onosproject.openstacknetworking"; + public static final String ARP_BROADCAST_MODE = "broadcast"; + public static final String ARP_PROXY_MODE = "proxy"; + public static final String DEFAULT_GATEWAY_MAC_STR = "fe:00:00:00:00:02"; + public static final String DEFAULT_ARP_MODE_STR = ARP_BROADCAST_MODE; public static final MacAddress DEFAULT_GATEWAY_MAC = MacAddress.valueOf(DEFAULT_GATEWAY_MAC_STR); public static final MacAddress DEFAULT_EXTERNAL_ROUTER_MAC = MacAddress.valueOf("fe:00:00:00:00:01"); @@ -48,7 +52,13 @@ public final class Constants { public static final int PRIORITY_CT_HOOK_RULE = 30500; public static final int PRIORITY_CT_RULE = 32000; public static final int PRIORITY_CT_DROP_RULE = 32500; + public static final int PRIORITY_ARP_GATEWAY_RULE = 41000; + public static final int PRIORITY_ARP_SUBNET_RULE = 40000; + public static final int PRIORITY_ARP_CONTROL_RULE = 40000; + public static final int PRIORITY_ARP_REPLY_RULE = 40000; + public static final int PRIORITY_ARP_REQUEST_RULE = 40000; + public static final int DHCP_ARP_TABLE = 0; public static final int SRC_VNI_TABLE = 0; public static final int ACL_TABLE = 1; public static final int CT_TABLE = 2; diff --git a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java index e9ac9238c7..b3340c6494 100644 --- a/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java +++ b/apps/openstacknetworking/app/src/main/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandler.java @@ -32,22 +32,36 @@ import org.onlab.packet.IpAddress; import org.onlab.packet.MacAddress; import org.onlab.util.Tools; import org.onosproject.cfg.ComponentConfigService; +import org.onosproject.cluster.ClusterService; +import org.onosproject.cluster.LeadershipService; +import org.onosproject.cluster.NodeId; import org.onosproject.core.ApplicationId; import org.onosproject.core.CoreService; +import org.onosproject.mastership.MastershipService; +import org.onosproject.net.PortNumber; +import org.onosproject.net.device.DeviceService; import org.onosproject.net.flow.DefaultTrafficSelector; import org.onosproject.net.flow.DefaultTrafficTreatment; import org.onosproject.net.flow.TrafficSelector; import org.onosproject.net.flow.TrafficTreatment; import org.onosproject.net.packet.DefaultOutboundPacket; import org.onosproject.net.packet.PacketContext; -import org.onosproject.net.packet.PacketPriority; import org.onosproject.net.packet.PacketProcessor; import org.onosproject.net.packet.PacketService; import org.onosproject.openstacknetworking.api.InstancePort; +import org.onosproject.openstacknetworking.api.InstancePortEvent; +import org.onosproject.openstacknetworking.api.InstancePortListener; import org.onosproject.openstacknetworking.api.InstancePortService; +import org.onosproject.openstacknetworking.api.OpenstackFlowRuleService; import org.onosproject.openstacknetworking.api.OpenstackNetworkEvent; import org.onosproject.openstacknetworking.api.OpenstackNetworkListener; import org.onosproject.openstacknetworking.api.OpenstackNetworkService; +import org.onosproject.openstacknode.api.OpenstackNode; +import org.onosproject.openstacknode.api.OpenstackNodeEvent; +import org.onosproject.openstacknode.api.OpenstackNodeListener; +import org.onosproject.openstacknode.api.OpenstackNodeService; +import org.openstack4j.model.network.Network; +import org.openstack4j.model.network.NetworkType; import org.openstack4j.model.network.Subnet; import org.osgi.service.component.ComponentContext; import org.slf4j.Logger; @@ -55,11 +69,24 @@ import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.util.Dictionary; +import java.util.Objects; import java.util.Set; import static com.google.common.base.Preconditions.checkNotNull; +import static org.onosproject.openstacknetworking.api.Constants.ARP_BROADCAST_MODE; +import static org.onosproject.openstacknetworking.api.Constants.ARP_PROXY_MODE; +import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_ARP_MODE_STR; import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_GATEWAY_MAC_STR; +import static org.onosproject.openstacknetworking.api.Constants.DHCP_ARP_TABLE; import static org.onosproject.openstacknetworking.api.Constants.OPENSTACK_NETWORKING_APP_ID; +import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_CONTROL_RULE; +import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE; +import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_REPLY_RULE; +import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_REQUEST_RULE; +import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_SUBNET_RULE; +import static org.onosproject.openstacknetworking.util.RulePopulatorUtil.buildExtension; +import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.COMPUTE; +import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.GATEWAY; /** * Handles ARP packet from VMs. @@ -70,6 +97,7 @@ public final class OpenstackSwitchingArpHandler { private final Logger log = LoggerFactory.getLogger(getClass()); private static final String GATEWAY_MAC = "gatewayMac"; + private static final String ARP_MODE = "arpMode"; @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) CoreService coreService; @@ -77,34 +105,73 @@ public final class OpenstackSwitchingArpHandler { @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) PacketService packetService; + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) + OpenstackFlowRuleService osFlowRuleService; + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) ComponentConfigService configService; + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) + ClusterService clusterService; + + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) + LeadershipService leadershipService; + + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) + DeviceService deviceService; + + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) + MastershipService mastershipService; + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) InstancePortService instancePortService; @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) OpenstackNetworkService osNetworkService; + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) + protected OpenstackNodeService osNodeService; + @Property(name = GATEWAY_MAC, value = DEFAULT_GATEWAY_MAC_STR, label = "Fake MAC address for virtual network subnet gateway") private String gatewayMac = DEFAULT_GATEWAY_MAC_STR; + @Property(name = ARP_MODE, value = DEFAULT_ARP_MODE_STR, + label = "ARP processing mode, broadcast (default) | proxy ") + protected String arpMode = DEFAULT_ARP_MODE_STR; + private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor(); private final InternalOpenstackNetworkListener osNetworkListener = new InternalOpenstackNetworkListener(); + private final InstancePortListener instancePortListener = new InternalInstancePortListener(); + private final OpenstackNodeListener osNodeListener = new InternalNodeEventListener(); + private final Set gateways = Sets.newConcurrentHashSet(); private ApplicationId appId; + private NodeId localNodeId; @Activate void activate() { appId = coreService.registerApplication(OPENSTACK_NETWORKING_APP_ID); configService.registerProperties(getClass()); + localNodeId = clusterService.getLocalNode().id(); osNetworkService.addListener(osNetworkListener); + osNodeService.addListener(osNodeListener); + leadershipService.runForLeadership(appId.name()); packetService.addProcessor(packetProcessor, PacketProcessor.director(0)); - osNetworkService.subnets().forEach(this::addSubnetGateway); - requestPacket(); + + instancePortService.addListener(instancePortListener); + + osNetworkService.networks().forEach(n -> { + if (n.getNetworkType() != NetworkType.FLAT) { + osNetworkService.subnets().forEach(s -> { + if (s.getNetworkId().equals(n.getId())) { + addSubnetGateway(s); + } + }); + } + }); log.info("Started"); } @@ -113,6 +180,9 @@ public final class OpenstackSwitchingArpHandler { void deactivate() { packetService.removeProcessor(packetProcessor); osNetworkService.removeListener(osNetworkListener); + osNodeService.removeListener(osNodeListener); + instancePortService.removeListener(instancePortListener); + leadershipService.withdraw(appId.name()); configService.unregisterProperties(getClass(), false); log.info("Stopped"); @@ -128,20 +198,16 @@ public final class OpenstackSwitchingArpHandler { gatewayMac = updatedMac; } + String updateArpMode; + + updateArpMode = Tools.get(properties, ARP_MODE); + if (!Strings.isNullOrEmpty(updateArpMode) && !updateArpMode.equals(arpMode)) { + arpMode = updateArpMode; + } + log.info("Modified"); } - private void requestPacket() { - TrafficSelector selector = DefaultTrafficSelector.builder() - .matchEthType(EthType.EtherType.ARP.ethType().toShort()) - .build(); - - packetService.requestPackets( - selector, - PacketPriority.CONTROL, - appId); - } - private void addSubnetGateway(Subnet osSubnet) { if (Strings.isNullOrEmpty(osSubnet.getGateway())) { return; @@ -169,6 +235,12 @@ public final class OpenstackSwitchingArpHandler { * @param ethPacket ethernet packet */ private void processPacketIn(PacketContext context, Ethernet ethPacket) { + + // if the ARP mode is configured as broadcast mode, we simply ignore ARP packet_in + if (arpMode.equals(ARP_BROADCAST_MODE)) { + return; + } + ARP arpPacket = (ARP) ethPacket.getPayload(); if (arpPacket.getOpCode() != ARP.OP_REQUEST) { return; @@ -224,6 +296,10 @@ public final class OpenstackSwitchingArpHandler { } } + /** + * An internal packet processor which processes ARP request, and results in + * packet-out ARP reply. + */ private class InternalPacketProcessor implements PacketProcessor { @Override @@ -240,6 +316,11 @@ public final class OpenstackSwitchingArpHandler { } } + /** + * An internal network listener which listens to openstack network event, + * manages the gateway collection and installs flow rule that handles + * ARP request in data plane. + */ private class InternalOpenstackNetworkListener implements OpenstackNetworkListener { @Override @@ -248,6 +329,23 @@ public final class OpenstackSwitchingArpHandler { if (osSubnet == null) { return false; } + + Network network = osNetworkService.network(event.port().getNetworkId()); + if (network == null) { + log.warn("Network is not specified."); + return false; + } else { + if (network.getNetworkType().equals(NetworkType.FLAT)) { + return false; + } + } + + // do not allow to proceed without leadership + NodeId leader = leadershipService.getLeader(appId.name()); + if (!Objects.equals(localNodeId, leader)) { + return false; + } + return !Strings.isNullOrEmpty(osSubnet.getGateway()); } @@ -257,9 +355,11 @@ public final class OpenstackSwitchingArpHandler { case OPENSTACK_SUBNET_CREATED: case OPENSTACK_SUBNET_UPDATED: addSubnetGateway(event.subnet()); + setFakeGatewayArpRule(event.subnet(), true); break; case OPENSTACK_SUBNET_REMOVED: removeSubnetGateway(event.subnet()); + setFakeGatewayArpRule(event.subnet(), false); break; case OPENSTACK_NETWORK_CREATED: case OPENSTACK_NETWORK_UPDATED: @@ -272,5 +372,286 @@ public final class OpenstackSwitchingArpHandler { break; } } + + /** + * Installs flow rules which convert ARP request packet into ARP reply + * by adding a fake gateway MAC address as Source Hardware Address. + * + * @param osSubnet openstack subnet + * @param install flag which indicates whether to install rule or remove rule + */ + private void setFakeGatewayArpRule(Subnet osSubnet, boolean install) { + + if (arpMode.equals(ARP_BROADCAST_MODE)) { + String gateway = osSubnet.getGateway(); + + TrafficSelector selector = DefaultTrafficSelector.builder() + .matchEthType(EthType.EtherType.ARP.ethType().toShort()) + .matchArpOp(ARP.OP_REQUEST) + .matchArpTpa(Ip4Address.valueOf(gateway)) + .build(); + + TrafficTreatment treatment = DefaultTrafficTreatment.builder() + .setArpOp(ARP.OP_REPLY) + .setArpSha(MacAddress.valueOf(gatewayMac)) + .setArpSpa(Ip4Address.valueOf(gateway)) + .setOutput(PortNumber.IN_PORT) + .build(); + + osNodeService.completeNodes(COMPUTE).forEach(n -> + osFlowRuleService.setRule( + appId, + n.intgBridge(), + selector, + treatment, + PRIORITY_ARP_GATEWAY_RULE, + DHCP_ARP_TABLE, + install + ) + ); + } + } + } + + /** + * An internal openstack node listener which is used for listening openstack + * node activity. As long as a node is in complete state, we will install + * default ARP rule to handle ARP request. + */ + private class InternalNodeEventListener implements OpenstackNodeListener { + + @Override + public boolean isRelevant(OpenstackNodeEvent event) { + // do not allow to proceed without leadership + NodeId leader = leadershipService.getLeader(appId.name()); + return Objects.equals(localNodeId, leader); + } + + @Override + public void event(OpenstackNodeEvent event) { + OpenstackNode osNode = event.subject(); + switch (event.type()) { + case OPENSTACK_NODE_COMPLETE: + setDefaultArpRule(osNode, true); + break; + case OPENSTACK_NODE_INCOMPLETE: + setDefaultArpRule(osNode, false); + break; + case OPENSTACK_NODE_CREATED: + case OPENSTACK_NODE_UPDATED: + case OPENSTACK_NODE_REMOVED: + default: + break; + } + } + + private void setDefaultArpRule(OpenstackNode openstackNode, boolean install) { + if (openstackNode.type().equals(GATEWAY)) { + return; + } + + if (arpMode.equals(ARP_PROXY_MODE)) { + TrafficSelector selector = DefaultTrafficSelector.builder() + .matchEthType(EthType.EtherType.ARP.ethType().toShort()) + .build(); + + TrafficTreatment treatment = DefaultTrafficTreatment.builder() + .punt() + .build(); + + osFlowRuleService.setRule( + appId, + openstackNode.intgBridge(), + selector, + treatment, + PRIORITY_ARP_CONTROL_RULE, + DHCP_ARP_TABLE, + install + ); + } else if (arpMode.equals(ARP_BROADCAST_MODE)) { + TrafficSelector selector = DefaultTrafficSelector.builder() + .matchEthType(EthType.EtherType.ARP.ethType().toShort()) + .matchArpOp(ARP.OP_REQUEST) + .build(); + + TrafficTreatment treatment = DefaultTrafficTreatment.builder() + .setOutput(PortNumber.FLOOD) + .build(); + + osFlowRuleService.setRule( + appId, + openstackNode.intgBridge(), + selector, + treatment, + PRIORITY_ARP_SUBNET_RULE, + DHCP_ARP_TABLE, + install + ); + } else { + log.warn("Invalid ARP mode {}. Please use either broadcast or proxy mode.", arpMode); + } + } + } + + /** + * An internal instance port listener which listens the port events generated + * from VM. When ARP a host which located in a remote compute node, we specify + * both ARP OP mode as REQUEST and Target Protocol Address (TPA) with + * host IP address. When ARP a host which located in a local compute node, + * we specify only ARP OP mode as REQUEST. + */ + private class InternalInstancePortListener implements InstancePortListener { + + @Override + public boolean isRelevant(InstancePortEvent event) { + + if (arpMode.equals(ARP_PROXY_MODE)) { + return false; + } + + InstancePort instPort = event.subject(); + return mastershipService.isLocalMaster(instPort.deviceId()); + } + + @Override + public void event(InstancePortEvent event) { + switch (event.type()) { + case OPENSTACK_INSTANCE_PORT_UPDATED: + case OPENSTACK_INSTANCE_PORT_DETECTED: + setArpRequestRule(event.subject(), true); + setArpReplyRule(event.subject(), true); + break; + case OPENSTACK_INSTANCE_PORT_VANISHED: + setArpRequestRule(event.subject(), false); + setArpReplyRule(event.subject(), false); + break; + default: + break; + } + } + + /** + * Installs flow rules to match ARP request packets. + * + * @param port instance port + * @param install installation flag + */ + private void setArpRequestRule(InstancePort port, boolean install) { + NetworkType type = osNetworkService.network(port.networkId()).getNetworkType(); + + switch (type) { + case VXLAN: + setRemoteArpRequestRuleForVxlan(port, install); + break; + case VLAN: + // TODO: handle VLAN differently + break; + default: + break; + } + } + + /** + * Installs flow rules to match ARP reply packets. + * + * @param port instance port + * @param install installation flag + */ + private void setArpReplyRule(InstancePort port, boolean install) { + NetworkType type = osNetworkService.network(port.networkId()).getNetworkType(); + + switch (type) { + case VXLAN: + setArpReplyRuleVxlan(port, install); + break; + case VLAN: + // TODO: handle VLAN differently + break; + default: + break; + } + } + + /** + * Installs flow rules to match ARP request packets only for VxLAN. + * + * @param port instance port + * @param install installation flag + */ + private void setRemoteArpRequestRuleForVxlan(InstancePort port, boolean install) { + + OpenstackNode localNode = osNodeService.node(port.deviceId()); + + TrafficSelector selector = DefaultTrafficSelector.builder() + .matchEthType(EthType.EtherType.ARP.ethType().toShort()) + .matchArpOp(ARP.OP_REQUEST) + .matchArpTpa(port.ipAddress().getIp4Address()) + .build(); + + setRemoteArpTreatmentForVxlan(selector, port, localNode, install); + } + + /** + * Installs flow rules to match ARP reply packets only for VxLAN. + * + * @param port instance port + * @param install installation flag + */ + private void setArpReplyRuleVxlan(InstancePort port, boolean install) { + + OpenstackNode localNode = osNodeService.node(port.deviceId()); + + TrafficSelector selector = DefaultTrafficSelector.builder() + .matchEthType(EthType.EtherType.ARP.ethType().toShort()) + .matchArpOp(ARP.OP_REPLY) + .matchArpTpa(port.ipAddress().getIp4Address()) + .matchArpTha(port.macAddress()) + .build(); + + TrafficTreatment treatment = DefaultTrafficTreatment.builder() + .setOutput(port.portNumber()) + .build(); + + osFlowRuleService.setRule( + appId, + port.deviceId(), + selector, + treatment, + PRIORITY_ARP_REPLY_RULE, + DHCP_ARP_TABLE, + install + ); + + setRemoteArpTreatmentForVxlan(selector, port, localNode, install); + } + + // a helper method + private void setRemoteArpTreatmentForVxlan(TrafficSelector selector, + InstancePort port, + OpenstackNode localNode, + boolean install) { + for (OpenstackNode remoteNode : osNodeService.completeNodes(COMPUTE)) { + if (!remoteNode.intgBridge().equals(port.deviceId())) { + TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder() + .extension(buildExtension( + deviceService, + remoteNode.intgBridge(), + localNode.dataIp().getIp4Address()), + remoteNode.intgBridge()) + .setOutput(remoteNode.tunnelPortNum()) + .build(); + + osFlowRuleService.setRule( + appId, + remoteNode.intgBridge(), + selector, + treatmentToRemote, + PRIORITY_ARP_REQUEST_RULE, + DHCP_ARP_TABLE, + install + ); + } + } + } } } diff --git a/apps/openstacknetworking/app/src/test/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandlerTest.java b/apps/openstacknetworking/app/src/test/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandlerTest.java index b5074ed7fa..f46d668ba2 100644 --- a/apps/openstacknetworking/app/src/test/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandlerTest.java +++ b/apps/openstacknetworking/app/src/test/java/org/onosproject/openstacknetworking/impl/OpenstackSwitchingArpHandlerTest.java @@ -23,7 +23,11 @@ import org.onlab.packet.Ethernet; import org.onlab.packet.IpAddress; import org.onlab.packet.MacAddress; import org.onosproject.cfg.ComponentConfigAdapter; +import org.onosproject.cluster.ClusterServiceAdapter; +import org.onosproject.cluster.LeadershipServiceAdapter; +import org.onosproject.core.ApplicationId; import org.onosproject.core.CoreServiceAdapter; +import org.onosproject.core.DefaultApplicationId; import org.onosproject.net.DeviceId; import org.onosproject.net.PortNumber; import org.onosproject.net.packet.DefaultInboundPacket; @@ -41,6 +45,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.onosproject.net.NetTestTools.connectPoint; +import static org.onosproject.openstacknetworking.api.Constants.ARP_PROXY_MODE; public class OpenstackSwitchingArpHandlerTest { @@ -61,6 +66,11 @@ public class OpenstackSwitchingArpHandlerTest { arpHandler.instancePortService = new TestInstancePortService(); arpHandler.packetService = new TestPacketService(); arpHandler.osNetworkService = new TestOpenstackNetworkService(); + arpHandler.osNodeService = new TestOpenstackNodeService(); + arpHandler.osFlowRuleService = new TestOpenstackFlowRuleService(); + arpHandler.arpMode = ARP_PROXY_MODE; + arpHandler.clusterService = new TestClusterService(); + arpHandler.leadershipService = new TestLeadershipService(); arpHandler.activate(); } @@ -132,6 +142,10 @@ public class OpenstackSwitchingArpHandlerTest { * Mocks the CoreService. */ private class TestCoreService extends CoreServiceAdapter { + @Override + public ApplicationId registerApplication(String name) { + return new DefaultApplicationId(100, "arpTestApp"); + } } /** @@ -140,6 +154,30 @@ public class OpenstackSwitchingArpHandlerTest { private class TestConfigService extends ComponentConfigAdapter { } + /** + * Mocks the ClusterService. + */ + private class TestClusterService extends ClusterServiceAdapter { + } + + /** + * Mocks the LeadershipService. + */ + private class TestLeadershipService extends LeadershipServiceAdapter { + } + + /** + * Mocks the OpenstackNodeService. + */ + private class TestOpenstackNodeService extends OpenstackNodeServiceAdapter { + } + + /** + * Mocks the OpenstackFlowRuleService. + */ + private class TestOpenstackFlowRuleService extends OpenstackFlowRuleServiceAdapter { + } + /** * Mocks the InstancePortService. */