From c69a7fd0aee6523c9f6f18597cbb73435e6dc79c Mon Sep 17 00:00:00 2001 From: jiangrui Date: Thu, 19 Nov 2015 15:40:01 +0800 Subject: [PATCH] ONOS-3404 Change p2p tunnel to p2any tunnel by using set tunnel_dst in flow. Change-Id: I06cb6d42772434de9c016e795de5c6d8a6f45dfb --- .../vtn/manager/impl/VTNManager.java | 165 +++++++++++------- .../vtn/table/L2ForwardService.java | 26 +-- .../vtn/table/impl/L2ForwardServiceImpl.java | 36 +++- .../org/onosproject/vtn/util/VtnConfig.java | 24 ++- 4 files changed, 162 insertions(+), 89 deletions(-) diff --git a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/manager/impl/VTNManager.java b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/manager/impl/VTNManager.java index 7192ad95f5..2e38251ee7 100644 --- a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/manager/impl/VTNManager.java +++ b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/manager/impl/VTNManager.java @@ -15,12 +15,14 @@ */ package org.onosproject.vtn.manager.impl; +import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST; import static org.slf4j.LoggerFactory.getLogger; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; @@ -30,6 +32,7 @@ import org.apache.felix.scr.annotations.Deactivate; import org.apache.felix.scr.annotations.Reference; import org.apache.felix.scr.annotations.ReferenceCardinality; import org.apache.felix.scr.annotations.Service; +import org.onlab.packet.Ip4Address; import org.onlab.packet.IpAddress; import org.onlab.packet.MacAddress; import org.onlab.util.KryoNamespace; @@ -43,6 +46,7 @@ import org.onosproject.net.Port; import org.onosproject.net.PortNumber; import org.onosproject.net.behaviour.BridgeConfig; import org.onosproject.net.behaviour.BridgeDescription; +import org.onosproject.net.behaviour.ExtensionTreatmentResolver; import org.onosproject.net.config.NetworkConfigService; import org.onosproject.net.config.basics.BasicDeviceConfig; import org.onosproject.net.device.DeviceEvent; @@ -50,15 +54,24 @@ import org.onosproject.net.device.DeviceListener; import org.onosproject.net.device.DeviceService; import org.onosproject.net.driver.DriverHandler; import org.onosproject.net.driver.DriverService; +import org.onosproject.net.flow.DefaultTrafficTreatment; +import org.onosproject.net.flow.TrafficTreatment.Builder; +import org.onosproject.net.flow.instructions.ExtensionTreatment; import org.onosproject.net.flowobjective.Objective; +import org.onosproject.net.group.DefaultGroupBucket; +import org.onosproject.net.group.DefaultGroupDescription; +import org.onosproject.net.group.DefaultGroupKey; +import org.onosproject.net.group.GroupBucket; +import org.onosproject.net.group.GroupBuckets; +import org.onosproject.net.group.GroupDescription; +import org.onosproject.net.group.GroupKey; +import org.onosproject.net.group.GroupService; import org.onosproject.net.host.HostEvent; import org.onosproject.net.host.HostListener; import org.onosproject.net.host.HostService; import org.onosproject.store.serializers.KryoNamespaces; -import org.onosproject.store.service.ConsistentMap; import org.onosproject.store.service.EventuallyConsistentMap; import org.onosproject.store.service.LogicalClockService; -import org.onosproject.store.service.Serializer; import org.onosproject.store.service.StorageService; import org.onosproject.vtn.manager.VTNService; import org.onosproject.vtn.table.ClassifierService; @@ -79,6 +92,7 @@ import org.onosproject.vtnrsc.tenantnetwork.TenantNetworkService; import org.onosproject.vtnrsc.virtualport.VirtualPortService; import org.slf4j.Logger; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; /** @@ -120,6 +134,9 @@ public class VTNManager implements VTNService { @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) protected MastershipService mastershipService; + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) + protected GroupService groupService; + private ApplicationId appId; private ClassifierService classifierService; private L2ForwardService l2ForwardService; @@ -133,9 +150,10 @@ public class VTNManager implements VTNService { private static final String EX_PORT_NAME = "eth0"; private static final String SWITCHES_OF_CONTROLLER = "switchesOfController"; private static final String SWITCH_OF_LOCAL_HOST_PORTS = "switchOfLocalHostPorts"; + private static final String DEFAULT_IP = "0.0.0.0"; private EventuallyConsistentMap switchesOfController; - private ConsistentMap switchOfLocalHostPorts; + private EventuallyConsistentMap switchOfLocalHostPorts; @Activate public void activate() { @@ -162,9 +180,9 @@ public class VTNManager implements VTNService { .build(); switchOfLocalHostPorts = storageService - .consistentMapBuilder() - .withName(SWITCH_OF_LOCAL_HOST_PORTS) - .withSerializer(Serializer.using(serializer.build())) + .eventuallyConsistentMapBuilder() + .withName(SWITCH_OF_LOCAL_HOST_PORTS).withSerializer(serializer) + .withTimestampProvider((k, v) -> clockService.getTimestamp()) .build(); log.info("Started"); @@ -251,44 +269,10 @@ public class VTNManager implements VTNService { private void programTunnelConfig(DeviceId localDeviceId, IpAddress localIp, DriverHandler localHandler) { - Iterable devices = deviceService.getAvailableDevices(); - Sets.newHashSet(devices).stream() - .filter(d -> Device.Type.CONTROLLER == d.type()) - .filter(d -> !localDeviceId.equals(d.id())).forEach(d -> { - DriverHandler tunHandler = driverService - .createHandler(d.id()); - String remoteIpAddress = d.annotations() - .value(CONTROLLER_IP_KEY); - IpAddress remoteIp = IpAddress.valueOf(remoteIpAddress); - if (remoteIp.toString() - .equalsIgnoreCase(localIp.toString())) { - log.error("The localIp and remoteIp are the same"); - return; - } - if (localHandler != null) { - // Create tunnel in br-int on local controller - if (mastershipService.isLocalMaster(localDeviceId)) { - VtnConfig.applyTunnelConfig(localHandler, localIp, remoteIp); - log.info("Add tunnel between {} and {}", localIp, - remoteIp); - } - // Create tunnel in br-int on other controllers - if (mastershipService.isLocalMaster(d.id())) { - VtnConfig.applyTunnelConfig(tunHandler, remoteIp, - localIp); - log.info("Add tunnel between {} and {}", remoteIp, - localIp); - } - } else { - // remove tunnel in br-int on other controllers - if (mastershipService.isLocalMaster(d.id())) { - VtnConfig.removeTunnelConfig(tunHandler, remoteIp, - localIp); - log.info("Remove tunnel between {} and {}", remoteIp, - localIp); - } - } - }); + if (mastershipService.isLocalMaster(localDeviceId)) { + VtnConfig.applyTunnelConfig(localHandler, localIp, IpAddress.valueOf(DEFAULT_IP)); + log.info("Add tunnel on {}", localIp); + } } private void applyTunnelOut(Device device, Objective.Operation type) { @@ -321,6 +305,7 @@ public class VTNManager implements VTNService { DriverHandler handler = driverService.createHandler(localControllerId); Set ports = VtnConfig.getPortNumbers(handler); Iterable allHosts = hostService.getHosts(); + String tunnelName = "vxlan-" + DEFAULT_IP; if (allHosts != null) { Sets.newHashSet(allHosts).stream().forEach(host -> { MacAddress hostMac = host.mac(); @@ -346,13 +331,12 @@ public class VTNManager implements VTNService { } IpAddress remoteIpAddress = IpAddress .valueOf(remoteControllerIp); - String tunnelName = "vxlan-" + remoteIpAddress.toString(); ports.stream() .filter(p -> p.name().equalsIgnoreCase(tunnelName)) .forEach(p -> { l2ForwardService .programTunnelOut(device.id(), segmentationId, p, - hostMac, type); + hostMac, type, remoteIpAddress); }); }); } @@ -392,16 +376,11 @@ public class VTNManager implements VTNService { Collection localTunnelPorts = VtnData.getLocalTunnelPorts(ports); // Get all the local vm's PortNumber in the current node Map> localHostPorts = switchOfLocalHostPorts - .get(deviceId).value().getNetworkOfLocalHostPorts(); + .get(deviceId).getNetworkOfLocalHostPorts(); Set networkOflocalHostPorts = localHostPorts.get(network.id()); - - l2ForwardService.programLocalBcastRules(deviceId, segmentationId, - inPort, networkOflocalHostPorts, - localTunnelPorts, - type); - - l2ForwardService.programLocalOut(deviceId, segmentationId, inPort, mac, - type); + for (PortNumber p : localTunnelPorts) { + programGroupTable(deviceId, appId, p, devices, type); + } if (type == Objective.Operation.ADD) { if (networkOflocalHostPorts == null) { @@ -409,20 +388,33 @@ public class VTNManager implements VTNService { localHostPorts.putIfAbsent(network.id(), networkOflocalHostPorts); } networkOflocalHostPorts.add(inPort); + l2ForwardService.programLocalBcastRules(deviceId, segmentationId, + inPort, networkOflocalHostPorts, + localTunnelPorts, + type); classifierService.programTunnelIn(deviceId, segmentationId, localTunnelPorts, type); } else if (type == Objective.Operation.REMOVE) { - networkOflocalHostPorts.remove(inPort); - if (networkOflocalHostPorts.isEmpty()) { - classifierService.programTunnelIn(deviceId, segmentationId, - localTunnelPorts, - Objective.Operation.REMOVE); - switchOfLocalHostPorts.get(deviceId).value().getNetworkOfLocalHostPorts() - .remove(virtualPort.networkId()); + if (networkOflocalHostPorts != null) { + l2ForwardService.programLocalBcastRules(deviceId, segmentationId, + inPort, networkOflocalHostPorts, + localTunnelPorts, + type); + networkOflocalHostPorts.remove(inPort); + if (networkOflocalHostPorts.isEmpty()) { + classifierService.programTunnelIn(deviceId, segmentationId, + localTunnelPorts, + type); + switchOfLocalHostPorts.get(deviceId).getNetworkOfLocalHostPorts() + .remove(virtualPort.networkId()); + } } } + l2ForwardService.programLocalOut(deviceId, segmentationId, inPort, mac, + type); + l2ForwardService.programTunnelBcastRules(deviceId, segmentationId, networkOflocalHostPorts, localTunnelPorts, @@ -440,9 +432,11 @@ public class VTNManager implements VTNService { SegmentationId segmentationId, MacAddress dstMac, Objective.Operation type) { - String tunnelName = "vxlan-" + ipAddress.toString(); + String tunnelName = "vxlan-" + DEFAULT_IP; Sets.newHashSet(devices).stream() - .filter(d -> d.type() == Device.Type.CONTROLLER).forEach(d -> { + .filter(d -> d.type() == Device.Type.CONTROLLER) + .filter(d -> !("ovsdb:" + ipAddress).equals(d.id().toString())) + .forEach(d -> { DriverHandler handler = driverService.createHandler(d.id()); BridgeConfig bridgeConfig = handler .behaviour(BridgeConfig.class); @@ -459,7 +453,7 @@ public class VTNManager implements VTNService { .forEach(p -> { l2ForwardService.programTunnelOut(sw.deviceId(), segmentationId, p, - dstMac, type); + dstMac, type, ipAddress); }); } }); @@ -525,4 +519,45 @@ public class VTNManager implements VTNService { } } + private void programGroupTable(DeviceId deviceId, ApplicationId appid, + PortNumber portNumber, Iterable devices, Objective.Operation type) { + if (type.equals(Objective.Operation.REMOVE)) { + return; + } + + List buckets = Lists.newArrayList(); + Sets.newHashSet(devices) + .stream() + .filter(d -> d.type() == Device.Type.CONTROLLER) + .filter(d -> !deviceId.equals(d.id())) + .forEach(d -> { + String ipAddress = d.annotations() + .value(CONTROLLER_IP_KEY); + Ip4Address dst = Ip4Address.valueOf(ipAddress); + Builder builder = DefaultTrafficTreatment.builder(); + + DriverHandler handler = driverService.createHandler(deviceId); + ExtensionTreatmentResolver resolver = handler.behaviour(ExtensionTreatmentResolver.class); + ExtensionTreatment treatment = resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type()); + try { + treatment.setPropertyValue("tunnelDst", dst); + } catch (Exception e) { + log.error("Failed to get extension instruction to set tunnel dst {}", deviceId); + } + + builder.extension(treatment, deviceId); + builder.setOutput(portNumber); + GroupBucket bucket = DefaultGroupBucket + .createAllGroupBucket(builder.build()); + buckets.add(bucket); + }); + final GroupKey key = new DefaultGroupKey(APP_ID.getBytes()); + GroupDescription groupDescription = new DefaultGroupDescription(deviceId, + GroupDescription.Type.ALL, + new GroupBuckets(buckets), + key, + L2ForwardServiceImpl.GROUP_ID, + appid); + groupService.addGroup(groupDescription); + } } diff --git a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/L2ForwardService.java b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/L2ForwardService.java index 3781c2bf42..cb661f8b41 100644 --- a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/L2ForwardService.java +++ b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/L2ForwardService.java @@ -15,6 +15,7 @@ */ package org.onosproject.vtn.table; +import org.onlab.packet.IpAddress; import org.onlab.packet.MacAddress; import org.onosproject.net.DeviceId; import org.onosproject.net.PortNumber; @@ -22,13 +23,14 @@ import org.onosproject.net.flowobjective.Objective; import org.onosproject.vtnrsc.SegmentationId; /** - * Applies L2 flows to the device. + * Applies L2 flows to the device. L2Forward table is Table(50). */ public interface L2ForwardService { /** - * The local broadcast rule that message matches Table(50) Match: broadcast - * mac and vnid Action: output port. + * The local broadcast rule that message matches Table(50). + * Match: broadcast mac and vnid. + * Action: set output port. * * @param deviceId Device Id * @param segmentationId the vnid of the host belong to @@ -45,8 +47,9 @@ public interface L2ForwardService { Objective.Operation type); /** - * The tunnel broadcast rule that message matches Table(50) Match: broadcast - * mac and vnid Action: output port. + * The tunnel broadcast rule that message matches Table(50). + * Match: broadcast mac and vnid. + * Action: output port. * * @param deviceId Device Id * @param segmentationId the vnid of the host belong to @@ -61,8 +64,9 @@ public interface L2ForwardService { Objective.Operation type); /** - * The local out rule that message matches. Table(50) Match: local host mac - * and vnid Action: output local host port. + * The local out rule that message matches Table(50). + * Match: local host mac and vnid. + * Action: output local host port. * * @param deviceId Device Id * @param segmentationId the vnid of the host belong to @@ -75,17 +79,19 @@ public interface L2ForwardService { Objective.Operation type); /** - * The tunnel out rule that message matches. Table(50) Match: host mac and - * vnid Action: output tunnel port. + * The tunnel out rule that message matches Table(50). + * Match: host mac and vnid. + * Action: output tunnel port. * * @param deviceId Device Id * @param segmentationId the vnid of the host belong to * @param tunnelOutPort the port of the tunnel * @param dstMac the mac of the host * @param type the operation of the flow + * @param ipAddress the ipAddress of the node */ void programTunnelOut(DeviceId deviceId, SegmentationId segmentationId, PortNumber tunnelOutPort, MacAddress dstMac, - Objective.Operation type); + Objective.Operation type, IpAddress ipAddress); } diff --git a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/impl/L2ForwardServiceImpl.java b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/impl/L2ForwardServiceImpl.java index 64c5feec18..3581cf6e83 100644 --- a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/impl/L2ForwardServiceImpl.java +++ b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/table/impl/L2ForwardServiceImpl.java @@ -16,19 +16,28 @@ package org.onosproject.vtn.table.impl; import static com.google.common.base.Preconditions.checkNotNull; +import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.NICIRA_SET_TUNNEL_DST; import static org.slf4j.LoggerFactory.getLogger; import org.onlab.osgi.DefaultServiceDirectory; import org.onlab.osgi.ServiceDirectory; +import org.onlab.packet.Ip4Address; +import org.onlab.packet.IpAddress; import org.onlab.packet.MacAddress; import org.onosproject.core.ApplicationId; +import org.onosproject.core.DefaultGroupId; import org.onosproject.net.DeviceId; import org.onosproject.net.PortNumber; +import org.onosproject.net.behaviour.ExtensionTreatmentResolver; +import org.onosproject.net.driver.DriverHandler; +import org.onosproject.net.driver.DriverService; import org.onosproject.net.flow.DefaultTrafficSelector; import org.onosproject.net.flow.DefaultTrafficTreatment; import org.onosproject.net.flow.TrafficSelector; import org.onosproject.net.flow.TrafficTreatment; +import org.onosproject.net.flow.TrafficTreatment.Builder; import org.onosproject.net.flow.criteria.Criteria; +import org.onosproject.net.flow.instructions.ExtensionTreatment; import org.onosproject.net.flowobjective.DefaultForwardingObjective; import org.onosproject.net.flowobjective.FlowObjectiveService; import org.onosproject.net.flowobjective.ForwardingObjective; @@ -47,10 +56,10 @@ public final class L2ForwardServiceImpl implements L2ForwardService { private final Logger log = getLogger(getClass()); private static final int MAC_PRIORITY = 0xffff; - + public static final Integer GROUP_ID = 1; private final FlowObjectiveService flowObjectiveService; private final ApplicationId appId; - + private final DriverService driverService; /** * Constructor. * @@ -60,6 +69,7 @@ public final class L2ForwardServiceImpl implements L2ForwardService { this.appId = checkNotNull(appId, "ApplicationId can not be null"); ServiceDirectory serviceDirectory = new DefaultServiceDirectory(); this.flowObjectiveService = serviceDirectory.get(FlowObjectiveService.class); + this.driverService = serviceDirectory.get(DriverService.class); } @Override @@ -91,9 +101,7 @@ public final class L2ForwardServiceImpl implements L2ForwardService { if (type.equals(Objective.Operation.REMOVE) && inPort == lp) { flag = false; } - for (PortNumber outport : localTunnelPorts) { - treatment.setOutput(outport); - } + treatment.group(new DefaultGroupId(GROUP_ID)); ForwardingObjective.Builder objective = DefaultForwardingObjective .builder().withTreatment(treatment.build()) .withSelector(selector).fromApp(appId).makePermanent() @@ -171,15 +179,26 @@ public final class L2ForwardServiceImpl implements L2ForwardService { public void programTunnelOut(DeviceId deviceId, SegmentationId segmentationId, PortNumber tunnelOutPort, MacAddress dstMac, - Objective.Operation type) { + Objective.Operation type, IpAddress ipAddress) { TrafficSelector selector = DefaultTrafficSelector.builder() .matchEthDst(dstMac).add(Criteria.matchTunnelId(Long .parseLong(segmentationId.toString()))) .build(); - TrafficTreatment treatment = DefaultTrafficTreatment.builder() + + DriverHandler handler = driverService.createHandler(deviceId); + ExtensionTreatmentResolver resolver = handler.behaviour(ExtensionTreatmentResolver.class); + ExtensionTreatment treatment = resolver.getExtensionInstruction(NICIRA_SET_TUNNEL_DST.type()); + try { + treatment.setPropertyValue("tunnelDst", Ip4Address.valueOf(ipAddress.toString())); + } catch (Exception e) { + log.error("Failed to get extension instruction to set tunnel dst {}", deviceId); + } + + Builder builder = DefaultTrafficTreatment.builder(); + builder.extension(treatment, deviceId) .setOutput(tunnelOutPort).build(); ForwardingObjective.Builder objective = DefaultForwardingObjective - .builder().withTreatment(treatment).withSelector(selector) + .builder().withTreatment(builder.build()).withSelector(selector) .fromApp(appId).withFlag(Flag.SPECIFIC) .withPriority(MAC_PRIORITY); if (type.equals(Objective.Operation.ADD)) { @@ -189,5 +208,4 @@ public final class L2ForwardServiceImpl implements L2ForwardService { } } - } diff --git a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/util/VtnConfig.java b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/util/VtnConfig.java index 1f98e55de2..5d70230e5d 100644 --- a/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/util/VtnConfig.java +++ b/apps/vtn/vtnmgr/src/main/java/org/onosproject/vtn/util/VtnConfig.java @@ -15,9 +15,12 @@ */ package org.onosproject.vtn.util; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import org.onlab.packet.IpAddress; +import org.onosproject.net.DefaultAnnotations; import org.onosproject.net.PortNumber; import org.onosproject.net.behaviour.BridgeConfig; import org.onosproject.net.behaviour.BridgeName; @@ -26,6 +29,7 @@ import org.onosproject.net.behaviour.IpTunnelEndPoint; import org.onosproject.net.behaviour.TunnelConfig; import org.onosproject.net.behaviour.TunnelDescription; import org.onosproject.net.behaviour.TunnelEndPoint; +import org.onosproject.net.behaviour.TunnelName; import org.onosproject.net.driver.DriverHandler; /** @@ -34,7 +38,13 @@ import org.onosproject.net.driver.DriverHandler; public final class VtnConfig { private static final String DEFAULT_BRIDGE_NAME = "br-int"; - + private static final String DEFAULT_TUNNEL = "vxlan-0.0.0.0"; + private static final Map DEFAULT_TUNNEL_OPTIONS = new HashMap() { + { + put("key", "flow"); + put("remote_ip", "flow"); + } + }; /** * Constructs a vtn config object. Utility classes should not have a * public or default constructor, otherwise IDE will compile unsuccessfully. This @@ -64,15 +74,19 @@ public final class VtnConfig { */ public static void applyTunnelConfig(DriverHandler handler, IpAddress srcIp, IpAddress dstIp) { + DefaultAnnotations.Builder optionBuilder = DefaultAnnotations.builder(); + for (String key : DEFAULT_TUNNEL_OPTIONS.keySet()) { + optionBuilder.set(key, DEFAULT_TUNNEL_OPTIONS.get(key)); + } TunnelConfig tunnelConfig = handler.behaviour(TunnelConfig.class); TunnelEndPoint tunnelAsSrc = IpTunnelEndPoint.ipTunnelPoint(srcIp); - TunnelEndPoint tunnelAsDst = IpTunnelEndPoint.ipTunnelPoint(dstIp); TunnelDescription tunnel = new DefaultTunnelDescription( tunnelAsSrc, - tunnelAsDst, + null, TunnelDescription.Type.VXLAN, - null); - tunnelConfig.createTunnel(tunnel); + TunnelName.tunnelName(DEFAULT_TUNNEL), + optionBuilder.build()); + tunnelConfig.createTunnelInterface(BridgeName.bridgeName(DEFAULT_BRIDGE_NAME), tunnel); } /**