[ONOS-7606] Support ARP broadcast (VxLAN) to handle CP failure

Change-Id: Ia0bccf6abaad3e074f2d86a511d5930974743b43
This commit is contained in:
Jian Li 2018-04-10 18:48:32 +09:00 committed by Thomas Vachuska
parent 8cbc3b9fd3
commit eae123687c
3 changed files with 443 additions and 14 deletions

View File

@ -27,7 +27,11 @@ public final class Constants {
public static final String OPENSTACK_NETWORKING_APP_ID = "org.onosproject.openstacknetworking";
public static final String ARP_BROADCAST_MODE = "broadcast";
public static final String ARP_PROXY_MODE = "proxy";
public static final String DEFAULT_GATEWAY_MAC_STR = "fe:00:00:00:00:02";
public static final String DEFAULT_ARP_MODE_STR = ARP_BROADCAST_MODE;
public static final MacAddress DEFAULT_GATEWAY_MAC = MacAddress.valueOf(DEFAULT_GATEWAY_MAC_STR);
public static final MacAddress DEFAULT_EXTERNAL_ROUTER_MAC = MacAddress.valueOf("fe:00:00:00:00:01");
@ -48,7 +52,13 @@ public final class Constants {
public static final int PRIORITY_CT_HOOK_RULE = 30500;
public static final int PRIORITY_CT_RULE = 32000;
public static final int PRIORITY_CT_DROP_RULE = 32500;
public static final int PRIORITY_ARP_GATEWAY_RULE = 41000;
public static final int PRIORITY_ARP_SUBNET_RULE = 40000;
public static final int PRIORITY_ARP_CONTROL_RULE = 40000;
public static final int PRIORITY_ARP_REPLY_RULE = 40000;
public static final int PRIORITY_ARP_REQUEST_RULE = 40000;
public static final int DHCP_ARP_TABLE = 0;
public static final int SRC_VNI_TABLE = 0;
public static final int ACL_TABLE = 1;
public static final int CT_TABLE = 2;

View File

@ -32,22 +32,36 @@ import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
import org.onlab.util.Tools;
import org.onosproject.cfg.ComponentConfigService;
import org.onosproject.cluster.ClusterService;
import org.onosproject.cluster.LeadershipService;
import org.onosproject.cluster.NodeId;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.mastership.MastershipService;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.packet.DefaultOutboundPacket;
import org.onosproject.net.packet.PacketContext;
import org.onosproject.net.packet.PacketPriority;
import org.onosproject.net.packet.PacketProcessor;
import org.onosproject.net.packet.PacketService;
import org.onosproject.openstacknetworking.api.InstancePort;
import org.onosproject.openstacknetworking.api.InstancePortEvent;
import org.onosproject.openstacknetworking.api.InstancePortListener;
import org.onosproject.openstacknetworking.api.InstancePortService;
import org.onosproject.openstacknetworking.api.OpenstackFlowRuleService;
import org.onosproject.openstacknetworking.api.OpenstackNetworkEvent;
import org.onosproject.openstacknetworking.api.OpenstackNetworkListener;
import org.onosproject.openstacknetworking.api.OpenstackNetworkService;
import org.onosproject.openstacknode.api.OpenstackNode;
import org.onosproject.openstacknode.api.OpenstackNodeEvent;
import org.onosproject.openstacknode.api.OpenstackNodeListener;
import org.onosproject.openstacknode.api.OpenstackNodeService;
import org.openstack4j.model.network.Network;
import org.openstack4j.model.network.NetworkType;
import org.openstack4j.model.network.Subnet;
import org.osgi.service.component.ComponentContext;
import org.slf4j.Logger;
@ -55,11 +69,24 @@ import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.Dictionary;
import java.util.Objects;
import java.util.Set;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.onosproject.openstacknetworking.api.Constants.ARP_BROADCAST_MODE;
import static org.onosproject.openstacknetworking.api.Constants.ARP_PROXY_MODE;
import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_ARP_MODE_STR;
import static org.onosproject.openstacknetworking.api.Constants.DEFAULT_GATEWAY_MAC_STR;
import static org.onosproject.openstacknetworking.api.Constants.DHCP_ARP_TABLE;
import static org.onosproject.openstacknetworking.api.Constants.OPENSTACK_NETWORKING_APP_ID;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_CONTROL_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_GATEWAY_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_REPLY_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_REQUEST_RULE;
import static org.onosproject.openstacknetworking.api.Constants.PRIORITY_ARP_SUBNET_RULE;
import static org.onosproject.openstacknetworking.util.RulePopulatorUtil.buildExtension;
import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.COMPUTE;
import static org.onosproject.openstacknode.api.OpenstackNode.NodeType.GATEWAY;
/**
* Handles ARP packet from VMs.
@ -70,6 +97,7 @@ public final class OpenstackSwitchingArpHandler {
private final Logger log = LoggerFactory.getLogger(getClass());
private static final String GATEWAY_MAC = "gatewayMac";
private static final String ARP_MODE = "arpMode";
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
CoreService coreService;
@ -77,34 +105,73 @@ public final class OpenstackSwitchingArpHandler {
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
PacketService packetService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
OpenstackFlowRuleService osFlowRuleService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
ComponentConfigService configService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
ClusterService clusterService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
LeadershipService leadershipService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
DeviceService deviceService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
MastershipService mastershipService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
InstancePortService instancePortService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
OpenstackNetworkService osNetworkService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected OpenstackNodeService osNodeService;
@Property(name = GATEWAY_MAC, value = DEFAULT_GATEWAY_MAC_STR,
label = "Fake MAC address for virtual network subnet gateway")
private String gatewayMac = DEFAULT_GATEWAY_MAC_STR;
@Property(name = ARP_MODE, value = DEFAULT_ARP_MODE_STR,
label = "ARP processing mode, broadcast (default) | proxy ")
protected String arpMode = DEFAULT_ARP_MODE_STR;
private final InternalPacketProcessor packetProcessor = new InternalPacketProcessor();
private final InternalOpenstackNetworkListener osNetworkListener =
new InternalOpenstackNetworkListener();
private final InstancePortListener instancePortListener = new InternalInstancePortListener();
private final OpenstackNodeListener osNodeListener = new InternalNodeEventListener();
private final Set<IpAddress> gateways = Sets.newConcurrentHashSet();
private ApplicationId appId;
private NodeId localNodeId;
@Activate
void activate() {
appId = coreService.registerApplication(OPENSTACK_NETWORKING_APP_ID);
configService.registerProperties(getClass());
localNodeId = clusterService.getLocalNode().id();
osNetworkService.addListener(osNetworkListener);
osNodeService.addListener(osNodeListener);
leadershipService.runForLeadership(appId.name());
packetService.addProcessor(packetProcessor, PacketProcessor.director(0));
osNetworkService.subnets().forEach(this::addSubnetGateway);
requestPacket();
instancePortService.addListener(instancePortListener);
osNetworkService.networks().forEach(n -> {
if (n.getNetworkType() != NetworkType.FLAT) {
osNetworkService.subnets().forEach(s -> {
if (s.getNetworkId().equals(n.getId())) {
addSubnetGateway(s);
}
});
}
});
log.info("Started");
}
@ -113,6 +180,9 @@ public final class OpenstackSwitchingArpHandler {
void deactivate() {
packetService.removeProcessor(packetProcessor);
osNetworkService.removeListener(osNetworkListener);
osNodeService.removeListener(osNodeListener);
instancePortService.removeListener(instancePortListener);
leadershipService.withdraw(appId.name());
configService.unregisterProperties(getClass(), false);
log.info("Stopped");
@ -128,20 +198,16 @@ public final class OpenstackSwitchingArpHandler {
gatewayMac = updatedMac;
}
String updateArpMode;
updateArpMode = Tools.get(properties, ARP_MODE);
if (!Strings.isNullOrEmpty(updateArpMode) && !updateArpMode.equals(arpMode)) {
arpMode = updateArpMode;
}
log.info("Modified");
}
private void requestPacket() {
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.build();
packetService.requestPackets(
selector,
PacketPriority.CONTROL,
appId);
}
private void addSubnetGateway(Subnet osSubnet) {
if (Strings.isNullOrEmpty(osSubnet.getGateway())) {
return;
@ -169,6 +235,12 @@ public final class OpenstackSwitchingArpHandler {
* @param ethPacket ethernet packet
*/
private void processPacketIn(PacketContext context, Ethernet ethPacket) {
// if the ARP mode is configured as broadcast mode, we simply ignore ARP packet_in
if (arpMode.equals(ARP_BROADCAST_MODE)) {
return;
}
ARP arpPacket = (ARP) ethPacket.getPayload();
if (arpPacket.getOpCode() != ARP.OP_REQUEST) {
return;
@ -224,6 +296,10 @@ public final class OpenstackSwitchingArpHandler {
}
}
/**
* An internal packet processor which processes ARP request, and results in
* packet-out ARP reply.
*/
private class InternalPacketProcessor implements PacketProcessor {
@Override
@ -240,6 +316,11 @@ public final class OpenstackSwitchingArpHandler {
}
}
/**
* An internal network listener which listens to openstack network event,
* manages the gateway collection and installs flow rule that handles
* ARP request in data plane.
*/
private class InternalOpenstackNetworkListener implements OpenstackNetworkListener {
@Override
@ -248,6 +329,23 @@ public final class OpenstackSwitchingArpHandler {
if (osSubnet == null) {
return false;
}
Network network = osNetworkService.network(event.port().getNetworkId());
if (network == null) {
log.warn("Network is not specified.");
return false;
} else {
if (network.getNetworkType().equals(NetworkType.FLAT)) {
return false;
}
}
// do not allow to proceed without leadership
NodeId leader = leadershipService.getLeader(appId.name());
if (!Objects.equals(localNodeId, leader)) {
return false;
}
return !Strings.isNullOrEmpty(osSubnet.getGateway());
}
@ -257,9 +355,11 @@ public final class OpenstackSwitchingArpHandler {
case OPENSTACK_SUBNET_CREATED:
case OPENSTACK_SUBNET_UPDATED:
addSubnetGateway(event.subnet());
setFakeGatewayArpRule(event.subnet(), true);
break;
case OPENSTACK_SUBNET_REMOVED:
removeSubnetGateway(event.subnet());
setFakeGatewayArpRule(event.subnet(), false);
break;
case OPENSTACK_NETWORK_CREATED:
case OPENSTACK_NETWORK_UPDATED:
@ -272,5 +372,286 @@ public final class OpenstackSwitchingArpHandler {
break;
}
}
/**
* Installs flow rules which convert ARP request packet into ARP reply
* by adding a fake gateway MAC address as Source Hardware Address.
*
* @param osSubnet openstack subnet
* @param install flag which indicates whether to install rule or remove rule
*/
private void setFakeGatewayArpRule(Subnet osSubnet, boolean install) {
if (arpMode.equals(ARP_BROADCAST_MODE)) {
String gateway = osSubnet.getGateway();
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.matchArpOp(ARP.OP_REQUEST)
.matchArpTpa(Ip4Address.valueOf(gateway))
.build();
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.setArpOp(ARP.OP_REPLY)
.setArpSha(MacAddress.valueOf(gatewayMac))
.setArpSpa(Ip4Address.valueOf(gateway))
.setOutput(PortNumber.IN_PORT)
.build();
osNodeService.completeNodes(COMPUTE).forEach(n ->
osFlowRuleService.setRule(
appId,
n.intgBridge(),
selector,
treatment,
PRIORITY_ARP_GATEWAY_RULE,
DHCP_ARP_TABLE,
install
)
);
}
}
}
/**
* An internal openstack node listener which is used for listening openstack
* node activity. As long as a node is in complete state, we will install
* default ARP rule to handle ARP request.
*/
private class InternalNodeEventListener implements OpenstackNodeListener {
@Override
public boolean isRelevant(OpenstackNodeEvent event) {
// do not allow to proceed without leadership
NodeId leader = leadershipService.getLeader(appId.name());
return Objects.equals(localNodeId, leader);
}
@Override
public void event(OpenstackNodeEvent event) {
OpenstackNode osNode = event.subject();
switch (event.type()) {
case OPENSTACK_NODE_COMPLETE:
setDefaultArpRule(osNode, true);
break;
case OPENSTACK_NODE_INCOMPLETE:
setDefaultArpRule(osNode, false);
break;
case OPENSTACK_NODE_CREATED:
case OPENSTACK_NODE_UPDATED:
case OPENSTACK_NODE_REMOVED:
default:
break;
}
}
private void setDefaultArpRule(OpenstackNode openstackNode, boolean install) {
if (openstackNode.type().equals(GATEWAY)) {
return;
}
if (arpMode.equals(ARP_PROXY_MODE)) {
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.build();
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.punt()
.build();
osFlowRuleService.setRule(
appId,
openstackNode.intgBridge(),
selector,
treatment,
PRIORITY_ARP_CONTROL_RULE,
DHCP_ARP_TABLE,
install
);
} else if (arpMode.equals(ARP_BROADCAST_MODE)) {
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.matchArpOp(ARP.OP_REQUEST)
.build();
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.setOutput(PortNumber.FLOOD)
.build();
osFlowRuleService.setRule(
appId,
openstackNode.intgBridge(),
selector,
treatment,
PRIORITY_ARP_SUBNET_RULE,
DHCP_ARP_TABLE,
install
);
} else {
log.warn("Invalid ARP mode {}. Please use either broadcast or proxy mode.", arpMode);
}
}
}
/**
* An internal instance port listener which listens the port events generated
* from VM. When ARP a host which located in a remote compute node, we specify
* both ARP OP mode as REQUEST and Target Protocol Address (TPA) with
* host IP address. When ARP a host which located in a local compute node,
* we specify only ARP OP mode as REQUEST.
*/
private class InternalInstancePortListener implements InstancePortListener {
@Override
public boolean isRelevant(InstancePortEvent event) {
if (arpMode.equals(ARP_PROXY_MODE)) {
return false;
}
InstancePort instPort = event.subject();
return mastershipService.isLocalMaster(instPort.deviceId());
}
@Override
public void event(InstancePortEvent event) {
switch (event.type()) {
case OPENSTACK_INSTANCE_PORT_UPDATED:
case OPENSTACK_INSTANCE_PORT_DETECTED:
setArpRequestRule(event.subject(), true);
setArpReplyRule(event.subject(), true);
break;
case OPENSTACK_INSTANCE_PORT_VANISHED:
setArpRequestRule(event.subject(), false);
setArpReplyRule(event.subject(), false);
break;
default:
break;
}
}
/**
* Installs flow rules to match ARP request packets.
*
* @param port instance port
* @param install installation flag
*/
private void setArpRequestRule(InstancePort port, boolean install) {
NetworkType type = osNetworkService.network(port.networkId()).getNetworkType();
switch (type) {
case VXLAN:
setRemoteArpRequestRuleForVxlan(port, install);
break;
case VLAN:
// TODO: handle VLAN differently
break;
default:
break;
}
}
/**
* Installs flow rules to match ARP reply packets.
*
* @param port instance port
* @param install installation flag
*/
private void setArpReplyRule(InstancePort port, boolean install) {
NetworkType type = osNetworkService.network(port.networkId()).getNetworkType();
switch (type) {
case VXLAN:
setArpReplyRuleVxlan(port, install);
break;
case VLAN:
// TODO: handle VLAN differently
break;
default:
break;
}
}
/**
* Installs flow rules to match ARP request packets only for VxLAN.
*
* @param port instance port
* @param install installation flag
*/
private void setRemoteArpRequestRuleForVxlan(InstancePort port, boolean install) {
OpenstackNode localNode = osNodeService.node(port.deviceId());
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.matchArpOp(ARP.OP_REQUEST)
.matchArpTpa(port.ipAddress().getIp4Address())
.build();
setRemoteArpTreatmentForVxlan(selector, port, localNode, install);
}
/**
* Installs flow rules to match ARP reply packets only for VxLAN.
*
* @param port instance port
* @param install installation flag
*/
private void setArpReplyRuleVxlan(InstancePort port, boolean install) {
OpenstackNode localNode = osNodeService.node(port.deviceId());
TrafficSelector selector = DefaultTrafficSelector.builder()
.matchEthType(EthType.EtherType.ARP.ethType().toShort())
.matchArpOp(ARP.OP_REPLY)
.matchArpTpa(port.ipAddress().getIp4Address())
.matchArpTha(port.macAddress())
.build();
TrafficTreatment treatment = DefaultTrafficTreatment.builder()
.setOutput(port.portNumber())
.build();
osFlowRuleService.setRule(
appId,
port.deviceId(),
selector,
treatment,
PRIORITY_ARP_REPLY_RULE,
DHCP_ARP_TABLE,
install
);
setRemoteArpTreatmentForVxlan(selector, port, localNode, install);
}
// a helper method
private void setRemoteArpTreatmentForVxlan(TrafficSelector selector,
InstancePort port,
OpenstackNode localNode,
boolean install) {
for (OpenstackNode remoteNode : osNodeService.completeNodes(COMPUTE)) {
if (!remoteNode.intgBridge().equals(port.deviceId())) {
TrafficTreatment treatmentToRemote = DefaultTrafficTreatment.builder()
.extension(buildExtension(
deviceService,
remoteNode.intgBridge(),
localNode.dataIp().getIp4Address()),
remoteNode.intgBridge())
.setOutput(remoteNode.tunnelPortNum())
.build();
osFlowRuleService.setRule(
appId,
remoteNode.intgBridge(),
selector,
treatmentToRemote,
PRIORITY_ARP_REQUEST_RULE,
DHCP_ARP_TABLE,
install
);
}
}
}
}
}

View File

@ -23,7 +23,11 @@ import org.onlab.packet.Ethernet;
import org.onlab.packet.IpAddress;
import org.onlab.packet.MacAddress;
import org.onosproject.cfg.ComponentConfigAdapter;
import org.onosproject.cluster.ClusterServiceAdapter;
import org.onosproject.cluster.LeadershipServiceAdapter;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreServiceAdapter;
import org.onosproject.core.DefaultApplicationId;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.packet.DefaultInboundPacket;
@ -41,6 +45,7 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.onosproject.net.NetTestTools.connectPoint;
import static org.onosproject.openstacknetworking.api.Constants.ARP_PROXY_MODE;
public class OpenstackSwitchingArpHandlerTest {
@ -61,6 +66,11 @@ public class OpenstackSwitchingArpHandlerTest {
arpHandler.instancePortService = new TestInstancePortService();
arpHandler.packetService = new TestPacketService();
arpHandler.osNetworkService = new TestOpenstackNetworkService();
arpHandler.osNodeService = new TestOpenstackNodeService();
arpHandler.osFlowRuleService = new TestOpenstackFlowRuleService();
arpHandler.arpMode = ARP_PROXY_MODE;
arpHandler.clusterService = new TestClusterService();
arpHandler.leadershipService = new TestLeadershipService();
arpHandler.activate();
}
@ -132,6 +142,10 @@ public class OpenstackSwitchingArpHandlerTest {
* Mocks the CoreService.
*/
private class TestCoreService extends CoreServiceAdapter {
@Override
public ApplicationId registerApplication(String name) {
return new DefaultApplicationId(100, "arpTestApp");
}
}
/**
@ -140,6 +154,30 @@ public class OpenstackSwitchingArpHandlerTest {
private class TestConfigService extends ComponentConfigAdapter {
}
/**
* Mocks the ClusterService.
*/
private class TestClusterService extends ClusterServiceAdapter {
}
/**
* Mocks the LeadershipService.
*/
private class TestLeadershipService extends LeadershipServiceAdapter {
}
/**
* Mocks the OpenstackNodeService.
*/
private class TestOpenstackNodeService extends OpenstackNodeServiceAdapter {
}
/**
* Mocks the OpenstackFlowRuleService.
*/
private class TestOpenstackFlowRuleService extends OpenstackFlowRuleServiceAdapter {
}
/**
* Mocks the InstancePortService.
*/