Merge branch 'master' of ssh://gerrit.onlab.us:29418/onos-next

better with files

Change-Id: Ifb62a2c9a8e394fe3190582d7236eaea942ee3fc

apps/foo/src/main/java/org/onlab/onos/foo/FooComponent.java

@@ -8,6 +8,9 @@ import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.cluster.ClusterEvent;
 import org.onlab.onos.cluster.ClusterEventListener;
 import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.net.device.DeviceEvent;
+import org.onlab.onos.net.device.DeviceListener;
+import org.onlab.onos.net.device.DeviceService;
 import org.slf4j.Logger;
 
 import static org.slf4j.LoggerFactory.getLogger;
@@ -23,17 +26,23 @@ public class FooComponent {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected ClusterService clusterService;
 
-    private ClusterEventListener clusterListener = new InnerClusterListener();
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    private final ClusterEventListener clusterListener = new InnerClusterListener();
+    private final DeviceListener deviceListener = new InnerDeviceListener();
 
     @Activate
     public void activate() {
         clusterService.addListener(clusterListener);
+        deviceService.addListener(deviceListener);
         log.info("Started");
     }
 
     @Deactivate
     public void deactivate() {
         clusterService.removeListener(clusterListener);
+        deviceService.removeListener(deviceListener);
         log.info("Stopped");
     }
 
@@ -43,6 +52,13 @@ public class FooComponent {
             log.info("WOOOOT! {}", event);
         }
     }
+
+    private class InnerDeviceListener implements DeviceListener {
+        @Override
+        public void event(DeviceEvent event) {
+            log.info("YEEEEHAAAAW! {}", event);
+        }
+    }
 }
 
 

apps/foo/src/main/java/org/onlab/onos/foo/package-info.java

@@ -0,0 +1,4 @@
+/**
+ * Sample application for use in various experiments.
+ */
+package org.onlab.onos.foo;

core/api/src/main/java/org/onlab/onos/ApplicationId.java

@@ -0,0 +1,59 @@
+package org.onlab.onos;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Application id generator class.
+ */
+public final class ApplicationId {
+
+    private static AtomicInteger idDispenser;
+    private final Integer id;
+
+    // Ban public construction
+    private ApplicationId(Integer id) {
+        this.id = id;
+    }
+
+    public Integer id() {
+        return id;
+    }
+
+    public static ApplicationId valueOf(Integer id) {
+        return new ApplicationId(id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof ApplicationId)) {
+            return false;
+        }
+        ApplicationId other = (ApplicationId) obj;
+        return Objects.equals(this.id, other.id);
+    }
+
+    /**
+     * Returns a new application id.
+     *
+     * @return app id
+     */
+    public static ApplicationId getAppId() {
+        if (ApplicationId.idDispenser == null) {
+            ApplicationId.idDispenser = new AtomicInteger(1);
+        }
+        return new ApplicationId(ApplicationId.idDispenser.getAndIncrement());
+    }
+
+}

core/api/src/main/java/org/onlab/onos/cluster/MastershipStore.java

@@ -4,12 +4,13 @@ import java.util.Set;
 
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.MastershipRole;
+import org.onlab.onos.store.Store;
 
 /**
  * Manages inventory of mastership roles for devices, across controller
  * instances; not intended for direct use.
  */
-public interface MastershipStore {
+public interface MastershipStore extends Store<MastershipEvent, MastershipStoreDelegate> {
 
     // three things to map: NodeId, DeviceId, MastershipRole
 
@@ -51,9 +52,7 @@ public interface MastershipStore {
      *
      * @param nodeId   controller instance identifier
      * @param deviceId device identifier
-     * @param role     new role
      * @return a mastership event
      */
-    MastershipEvent setRole(NodeId nodeId, DeviceId deviceId,
+    MastershipEvent setMaster(NodeId nodeId, DeviceId deviceId);
-                            MastershipRole role);
 }

core/api/src/main/java/org/onlab/onos/net/device/DeviceStore.java

@@ -5,13 +5,14 @@ import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.Store;
 
 import java.util.List;
 
 /**
  * Manages inventory of infrastructure devices; not intended for direct use.
  */
-public interface DeviceStore {
+public interface DeviceStore extends Store<DeviceEvent, DeviceStoreDelegate> {
 
     /**
      * Returns the number of devices known to the system.

core/api/src/main/java/org/onlab/onos/net/flow/FlowRuleStore.java

@@ -2,11 +2,12 @@ package org.onlab.onos.net.flow;
 
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.store.Store;
 
 /**
  * Manages inventory of flow rules; not intended for direct use.
  */
-public interface FlowRuleStore {
+public interface FlowRuleStore extends Store<FlowRuleEvent, FlowRuleStoreDelegate> {
 
     /**
      * Returns the stored flow.

core/api/src/main/java/org/onlab/onos/net/flow/FlowRuleStoreDelegate.java

@@ -0,0 +1,9 @@
+package org.onlab.onos.net.flow;
+
+import org.onlab.onos.store.StoreDelegate;
+
+/**
+ * Flow rule store delegate abstraction.
+ */
+public interface FlowRuleStoreDelegate extends StoreDelegate<FlowRuleEvent> {
+}

core/api/src/main/java/org/onlab/onos/net/host/HostStore.java

@@ -1,20 +1,21 @@
 package org.onlab.onos.net.host;
 
-import java.util.Set;
-
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.Store;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 
+import java.util.Set;
+
 /**
  * Manages inventory of end-station hosts; not intended for direct use.
  */
-public interface HostStore {
+public interface HostStore extends Store<HostEvent, HostStoreDelegate> {
 
     /**
      * Creates a new host or updates the existing one based on the specified
@@ -133,7 +134,7 @@ public interface HostStore {
      * Returns the address bindings for a particular connection point.
      *
      * @param connectPoint the connection point to return address information
-     * for
+     *                     for
      * @return address information for the connection point
      */
     PortAddresses getAddressBindingsForPort(ConnectPoint connectPoint);

core/api/src/main/java/org/onlab/onos/net/host/HostStoreDelegate.java

@@ -0,0 +1,9 @@
+package org.onlab.onos.net.host;
+
+import org.onlab.onos.store.StoreDelegate;
+
+/**
+ * Infrastructure link store delegate abstraction.
+ */
+public interface HostStoreDelegate extends StoreDelegate<HostEvent> {
+}

core/api/src/main/java/org/onlab/onos/net/link/LinkStore.java

@@ -4,13 +4,14 @@ import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Link;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.Store;
 
 import java.util.Set;
 
 /**
  * Manages inventory of infrastructure links; not intended for direct use.
  */
-public interface LinkStore {
+public interface LinkStore extends Store<LinkEvent, LinkStoreDelegate> {
 
     /**
      * Returns the number of links in the store.

core/api/src/main/java/org/onlab/onos/net/link/LinkStoreDelegate.java

@@ -0,0 +1,9 @@
+package org.onlab.onos.net.link;
+
+import org.onlab.onos.store.StoreDelegate;
+
+/**
+ * Infrastructure link store delegate abstraction.
+ */
+public interface LinkStoreDelegate extends StoreDelegate<LinkEvent> {
+}

core/api/src/main/java/org/onlab/onos/net/topology/TopologyStore.java

@@ -6,6 +6,7 @@ import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Link;
 import org.onlab.onos.net.Path;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.Store;
 
 import java.util.List;
 import java.util.Set;
@@ -13,7 +14,7 @@ import java.util.Set;
 /**
  * Manages inventory of topology snapshots; not intended for direct use.
  */
-public interface TopologyStore {
+public interface TopologyStore extends Store<TopologyEvent, TopologyStoreDelegate> {
 
     /**
      * Returns the current topology snapshot.

core/api/src/main/java/org/onlab/onos/net/topology/TopologyStoreDelegate.java

@@ -0,0 +1,9 @@
+package org.onlab.onos.net.topology;
+
+import org.onlab.onos.store.StoreDelegate;
+
+/**
+ * Topology store delegate abstraction.
+ */
+public interface TopologyStoreDelegate extends StoreDelegate<TopologyEvent> {
+}

core/api/src/main/java/org/onlab/onos/store/AbstractStore.java

@@ -2,6 +2,8 @@ package org.onlab.onos.store;
 
 import org.onlab.onos.event.Event;
 
+import static com.google.common.base.Preconditions.checkState;
+
 /**
  * Base implementation of a store.
  */
@@ -12,12 +14,21 @@ public class AbstractStore<E extends Event, D extends StoreDelegate<E>>
 
     @Override
     public void setDelegate(D delegate) {
+        checkState(this.delegate == null || this.delegate == delegate,
+                   "Store delegate already set");
         this.delegate = delegate;
     }
 
     @Override
-    public D getDelegate() {
+    public void unsetDelegate(D delegate) {
-        return delegate;
+        if (this.delegate == delegate) {
+            this.delegate = null;
+        }
+    }
+
+    @Override
+    public boolean hasDelegate() {
+        return delegate != null;
     }
 
     /**

core/api/src/main/java/org/onlab/onos/store/Store.java

@@ -12,14 +12,25 @@ public interface Store<E extends Event, D extends StoreDelegate<E>> {
      * Sets the delegate on the store.
      *
      * @param delegate new store delegate
+     * @throws java.lang.IllegalStateException if a delegate is already
+     *                                         currently set on the store and is a different one that
      */
     void setDelegate(D delegate);
 
     /**
-     * Get the current store delegate.
+     * Withdraws the delegate from the store.
      *
-     * @return store delegate
+     * @param delegate store delegate to withdraw
+     * @throws java.lang.IllegalArgumentException if the delegate is not
+     *                                            currently set on the store
      */
-    D getDelegate();
+    void unsetDelegate(D delegate);
+
+    /**
+     * Indicates whether the store has a delegate.
+     *
+     * @return true if delegate is set
+     */
+    boolean hasDelegate();
 
 }

core/api/src/main/java/org/onlab/onos/store/StoreDelegate.java

@@ -8,6 +8,11 @@ import org.onlab.onos.event.Event;
  */
 public interface StoreDelegate<E extends Event> {
 
+    /**
+     * Notifies the delegate via the specified event.
+     *
+     * @param event store generated event
+     */
     void notify(E event);
 
 }

core/net/pom.xml

@@ -44,6 +44,13 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.onlab.onos</groupId>
+            <artifactId>onos-core-store</artifactId>
+            <version>${project.version}</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
 
         <dependency>
             <groupId>org.apache.felix</groupId>

core/net/src/main/java/org/onlab/onos/cluster/impl/ClusterManager.java

@@ -53,6 +53,7 @@ public class ClusterManager implements ClusterService, ClusterAdminService {
 
     @Deactivate
     public void deactivate() {
+        store.unsetDelegate(delegate);
         eventDispatcher.removeSink(ClusterEvent.class);
         log.info("Stopped");
     }

core/net/src/main/java/org/onlab/onos/cluster/impl/MastershipManager.java

@@ -64,9 +64,12 @@ public class MastershipManager
         checkNotNull(nodeId, NODE_ID_NULL);
         checkNotNull(deviceId, DEVICE_ID_NULL);
         checkNotNull(role, ROLE_NULL);
-        MastershipEvent event = store.setRole(nodeId, deviceId, role);
+        //TODO figure out appropriate action for non-MASTER roles, if we even set those
-        if (event != null) {
+        if (role.equals(MastershipRole.MASTER)) {
-            post(event);
+            MastershipEvent event = store.setMaster(nodeId, deviceId);
+            if (event != null) {
+                post(event);
+            }
         }
     }
 

core/net/src/main/java/org/onlab/onos/net/device/impl/DeviceManager.java

@@ -26,6 +26,7 @@ import org.onlab.onos.net.device.DeviceProviderRegistry;
 import org.onlab.onos.net.device.DeviceProviderService;
 import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.device.DeviceStore;
+import org.onlab.onos.net.device.DeviceStoreDelegate;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.AbstractProviderRegistry;
 import org.onlab.onos.net.provider.AbstractProviderService;
@@ -33,8 +34,8 @@ import org.slf4j.Logger;
 
 import java.util.List;
 
-import static org.onlab.onos.net.device.DeviceEvent.Type.*;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.onos.net.device.DeviceEvent.Type.DEVICE_MASTERSHIP_CHANGED;
 import static org.slf4j.LoggerFactory.getLogger;
 
 /**
@@ -57,7 +58,9 @@ public class DeviceManager
     protected final AbstractListenerRegistry<DeviceEvent, DeviceListener>
             listenerRegistry = new AbstractListenerRegistry<>();
 
-    private final MastershipListener mastershipListener = new InnerMastershipListener();
+    private DeviceStoreDelegate delegate = new InternalStoreDelegate();
+
+    private final MastershipListener mastershipListener = new InternalMastershipListener();
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DeviceStore store;
@@ -73,6 +76,7 @@ public class DeviceManager
 
     @Activate
     public void activate() {
+        store.setDelegate(delegate);
         eventDispatcher.addSink(DeviceEvent.class, listenerRegistry);
         mastershipService.addListener(mastershipListener);
         log.info("Started");
@@ -80,6 +84,7 @@ public class DeviceManager
 
     @Deactivate
     public void deactivate() {
+        store.unsetDelegate(delegate);
         mastershipService.removeListener(mastershipListener);
         eventDispatcher.removeSink(DeviceEvent.class);
         log.info("Stopped");
@@ -239,7 +244,7 @@ public class DeviceManager
     }
 
     // Intercepts mastership events
-    private class InnerMastershipListener implements MastershipListener {
+    private class InternalMastershipListener implements MastershipListener {
         @Override
         public void event(MastershipEvent event) {
             // FIXME: for now we're taking action only on becoming master
@@ -248,4 +253,12 @@ public class DeviceManager
             }
         }
     }
+
+    // Store delegate to re-post events emitted from the store.
+    private class InternalStoreDelegate implements DeviceStoreDelegate {
+        @Override
+        public void notify(DeviceEvent event) {
+            post(event);
+        }
+    }
 }

core/net/src/main/java/org/onlab/onos/net/flow/impl/FlowRuleManager.java

@@ -1,11 +1,6 @@
 package org.onlab.onos.net.flow.impl;
 
-import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.collect.Lists;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Iterator;
-import java.util.List;
-
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -26,11 +21,16 @@ import org.onlab.onos.net.flow.FlowRuleProviderRegistry;
 import org.onlab.onos.net.flow.FlowRuleProviderService;
 import org.onlab.onos.net.flow.FlowRuleService;
 import org.onlab.onos.net.flow.FlowRuleStore;
+import org.onlab.onos.net.flow.FlowRuleStoreDelegate;
 import org.onlab.onos.net.provider.AbstractProviderRegistry;
 import org.onlab.onos.net.provider.AbstractProviderService;
 import org.slf4j.Logger;
 
-import com.google.common.collect.Lists;
+import java.util.Iterator;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
 
 /**
  * Provides implementation of the flow NB & SB APIs.
@@ -47,6 +47,8 @@ implements FlowRuleService, FlowRuleProviderRegistry {
     private final AbstractListenerRegistry<FlowRuleEvent, FlowRuleListener>
     listenerRegistry = new AbstractListenerRegistry<>();
 
+    private FlowRuleStoreDelegate delegate = new InternalStoreDelegate();
+
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected FlowRuleStore store;
 
@@ -58,12 +60,14 @@ implements FlowRuleService, FlowRuleProviderRegistry {
 
     @Activate
     public void activate() {
+        store.setDelegate(delegate);
         eventDispatcher.addSink(FlowRuleEvent.class, listenerRegistry);
         log.info("Started");
     }
 
     @Deactivate
     public void deactivate() {
+        store.unsetDelegate(delegate);
         eventDispatcher.removeSink(FlowRuleEvent.class);
         log.info("Stopped");
     }
@@ -249,4 +253,11 @@ implements FlowRuleService, FlowRuleProviderRegistry {
         }
     }
 
+    // Store delegate to re-post events emitted from the store.
+    private class InternalStoreDelegate implements FlowRuleStoreDelegate {
+        @Override
+        public void notify(FlowRuleEvent event) {
+            eventDispatcher.post(event);
+        }
+    }
 }

core/net/src/main/java/org/onlab/onos/net/host/impl/HostManager.java

@@ -1,10 +1,5 @@
 package org.onlab.onos.net.host.impl;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Set;
-
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -26,6 +21,7 @@ import org.onlab.onos.net.host.HostProviderRegistry;
 import org.onlab.onos.net.host.HostProviderService;
 import org.onlab.onos.net.host.HostService;
 import org.onlab.onos.net.host.HostStore;
+import org.onlab.onos.net.host.HostStoreDelegate;
 import org.onlab.onos.net.host.PortAddresses;
 import org.onlab.onos.net.provider.AbstractProviderRegistry;
 import org.onlab.onos.net.provider.AbstractProviderService;
@@ -35,6 +31,11 @@ import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.slf4j.Logger;
 
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.slf4j.LoggerFactory.getLogger;
+
 /**
  * Provides basic implementation of the host SB & NB APIs.
  */
@@ -50,6 +51,8 @@ public class HostManager
     private final AbstractListenerRegistry<HostEvent, HostListener>
             listenerRegistry = new AbstractListenerRegistry<>();
 
+    private HostStoreDelegate delegate = new InternalStoreDelegate();
+
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected HostStore store;
 
@@ -59,12 +62,14 @@ public class HostManager
 
     @Activate
     public void activate() {
+        store.setDelegate(delegate);
         eventDispatcher.addSink(HostEvent.class, listenerRegistry);
         log.info("Started");
     }
 
     @Deactivate
     public void deactivate() {
+        store.unsetDelegate(delegate);
         eventDispatcher.removeSink(HostEvent.class);
         log.info("Stopped");
     }
@@ -219,4 +224,11 @@ public class HostManager
         }
     }
 
+    // Store delegate to re-post events emitted from the store.
+    private class InternalStoreDelegate implements HostStoreDelegate {
+        @Override
+        public void notify(HostEvent event) {
+            post(event);
+        }
+    }
 }

core/net/src/main/java/org/onlab/onos/net/link/impl/LinkManager.java

@@ -28,6 +28,7 @@ import org.onlab.onos.net.link.LinkProviderRegistry;
 import org.onlab.onos.net.link.LinkProviderService;
 import org.onlab.onos.net.link.LinkService;
 import org.onlab.onos.net.link.LinkStore;
+import org.onlab.onos.net.link.LinkStoreDelegate;
 import org.onlab.onos.net.provider.AbstractProviderRegistry;
 import org.onlab.onos.net.provider.AbstractProviderService;
 import org.slf4j.Logger;
@@ -52,7 +53,9 @@ public class LinkManager
     protected final AbstractListenerRegistry<LinkEvent, LinkListener>
             listenerRegistry = new AbstractListenerRegistry<>();
 
-    private final DeviceListener deviceListener = new InnerDeviceListener();
+    private LinkStoreDelegate delegate = new InternalStoreDelegate();
+
+    private final DeviceListener deviceListener = new InternalDeviceListener();
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected LinkStore store;
@@ -65,6 +68,7 @@ public class LinkManager
 
     @Activate
     public void activate() {
+        store.setDelegate(delegate);
         eventDispatcher.addSink(LinkEvent.class, listenerRegistry);
         deviceService.addListener(deviceListener);
         log.info("Started");
@@ -72,6 +76,7 @@ public class LinkManager
 
     @Deactivate
     public void deactivate() {
+        store.unsetDelegate(delegate);
         eventDispatcher.removeSink(LinkEvent.class);
         deviceService.removeListener(deviceListener);
         log.info("Stopped");
@@ -154,7 +159,7 @@ public class LinkManager
 
     // Auxiliary interceptor for device remove events to prune links that
     // are associated with the removed device or its port.
-    private class InnerDeviceListener implements DeviceListener {
+    private class InternalDeviceListener implements DeviceListener {
         @Override
         public void event(DeviceEvent event) {
             if (event.type() == DeviceEvent.Type.DEVICE_REMOVED) {
@@ -236,4 +241,11 @@ public class LinkManager
         }
     }
 
+    // Store delegate to re-post events emitted from the store.
+    private class InternalStoreDelegate implements LinkStoreDelegate {
+        @Override
+        public void notify(LinkEvent event) {
+            post(event);
+        }
+    }
 }

core/net/src/main/java/org/onlab/onos/net/topology/impl/TopologyManager.java

@@ -28,6 +28,7 @@ import org.onlab.onos.net.topology.TopologyProviderRegistry;
 import org.onlab.onos.net.topology.TopologyProviderService;
 import org.onlab.onos.net.topology.TopologyService;
 import org.onlab.onos.net.topology.TopologyStore;
+import org.onlab.onos.net.topology.TopologyStoreDelegate;
 import org.slf4j.Logger;
 
 import java.util.List;
@@ -56,6 +57,8 @@ public class TopologyManager
     private final AbstractListenerRegistry<TopologyEvent, TopologyListener>
             listenerRegistry = new AbstractListenerRegistry<>();
 
+    private TopologyStoreDelegate delegate = new InternalStoreDelegate();
+
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected TopologyStore store;
 
@@ -65,12 +68,14 @@ public class TopologyManager
 
     @Activate
     public void activate() {
+        store.setDelegate(delegate);
         eventDispatcher.addSink(TopologyEvent.class, listenerRegistry);
         log.info("Started");
     }
 
     @Deactivate
     public void deactivate() {
+        store.unsetDelegate(delegate);
         eventDispatcher.removeSink(TopologyEvent.class);
         log.info("Stopped");
     }
@@ -188,4 +193,11 @@ public class TopologyManager
         }
     }
 
+    // Store delegate to re-post events emitted from the store.
+    private class InternalStoreDelegate implements TopologyStoreDelegate {
+        @Override
+        public void notify(TopologyEvent event) {
+            eventDispatcher.post(event);
+        }
+    }
 }

core/net/src/test/java/org/onlab/onos/cluster/impl/MastershipManagerTest.java

@@ -0,0 +1,136 @@
+package org.onlab.onos.cluster.impl;
+
+import java.util.Set;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.onlab.onos.cluster.ClusterEventListener;
+import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.ControllerNode.State;
+import org.onlab.onos.cluster.DefaultControllerNode;
+import org.onlab.onos.cluster.MastershipService;
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.event.impl.TestEventDispatcher;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.trivial.impl.SimpleMastershipStore;
+import org.onlab.packet.IpPrefix;
+
+import static org.junit.Assert.assertEquals;
+import static org.onlab.onos.net.MastershipRole.*;
+
+/**
+ * Test codifying the mastership service contracts.
+ */
+public class MastershipManagerTest {
+
+    private static final NodeId NID_LOCAL = new NodeId("local");
+    private static final NodeId NID_OTHER = new NodeId("foo");
+    private static final IpPrefix LOCALHOST = IpPrefix.valueOf("127.0.0.1");
+    private static final DeviceId DEV_MASTER = DeviceId.deviceId("of:1");
+    private static final DeviceId DEV_OTHER = DeviceId.deviceId("of:2");
+
+    private MastershipManager mgr;
+    protected MastershipService service;
+
+    @Before
+    public void setUp() {
+        mgr = new MastershipManager();
+        service = mgr;
+        mgr.store = new SimpleMastershipStore();
+        mgr.eventDispatcher = new TestEventDispatcher();
+        mgr.clusterService = new TestClusterService();
+        mgr.activate();
+    }
+
+    @After
+    public void tearDown() {
+        mgr.deactivate();
+        mgr.clusterService = null;
+        mgr.eventDispatcher = null;
+        mgr.store = null;
+    }
+
+    @Test
+    public void setRole() {
+        mgr.setRole(NID_OTHER, DEV_MASTER, MASTER);
+        assertEquals("wrong local role:", STANDBY, mgr.getLocalRole(DEV_MASTER));
+
+        //set to master
+        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
+        assertEquals("wrong local role:", MASTER, mgr.getLocalRole(DEV_MASTER));
+    }
+
+    @Test
+    public void relinquishMastership() {
+        //TODO
+    }
+
+    @Test
+    public void requestRoleFor() {
+        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
+        mgr.setRole(NID_OTHER, DEV_OTHER, MASTER);
+
+        //local should be master for one but standby for other
+        assertEquals("wrong role:", MASTER, mgr.requestRoleFor(DEV_MASTER));
+        assertEquals("wrong role:", STANDBY, mgr.requestRoleFor(DEV_OTHER));
+    }
+
+    @Test
+    public void getMasterFor() {
+        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
+        mgr.setRole(NID_OTHER, DEV_OTHER, MASTER);
+        assertEquals("wrong master:", NID_LOCAL, mgr.getMasterFor(DEV_MASTER));
+        assertEquals("wrong master:", NID_OTHER, mgr.getMasterFor(DEV_OTHER));
+
+        //have NID_OTHER hand over DEV_OTHER to NID_LOCAL
+        mgr.setRole(NID_LOCAL, DEV_OTHER, MASTER);
+        assertEquals("wrong master:", NID_LOCAL, mgr.getMasterFor(DEV_OTHER));
+    }
+
+    @Test
+    public void getDevicesOf() {
+        mgr.setRole(NID_LOCAL, DEV_MASTER, MASTER);
+        mgr.setRole(NID_LOCAL, DEV_OTHER, STANDBY);
+        assertEquals("should be one device:", 1, mgr.getDevicesOf(NID_LOCAL).size());
+
+        //hand both devices to NID_LOCAL
+        mgr.setRole(NID_LOCAL, DEV_OTHER, MASTER);
+        assertEquals("should be two devices:", 2, mgr.getDevicesOf(NID_LOCAL).size());
+    }
+
+    private final class TestClusterService implements ClusterService {
+
+        ControllerNode local = new DefaultControllerNode(NID_LOCAL, LOCALHOST);
+
+        @Override
+        public ControllerNode getLocalNode() {
+            return local;
+        }
+
+        @Override
+        public Set<ControllerNode> getNodes() {
+            return null;
+        }
+
+        @Override
+        public ControllerNode getNode(NodeId nodeId) {
+            return null;
+        }
+
+        @Override
+        public State getState(NodeId nodeId) {
+            return null;
+        }
+
+        @Override
+        public void addListener(ClusterEventListener listener) {
+        }
+
+        @Override
+        public void removeListener(ClusterEventListener listener) {
+        }
+
+    }
+}

core/net/src/test/java/org/onlab/onos/net/device/impl/DistributedDeviceManagerTest.java

@@ -4,13 +4,15 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import com.hazelcast.config.Config;
 import com.hazelcast.core.Hazelcast;
-import com.hazelcast.core.HazelcastInstance;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.onlab.onos.cluster.DefaultControllerNode;
 import org.onlab.onos.cluster.MastershipServiceAdapter;
 import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.event.Event;
+import org.onlab.onos.event.EventDeliveryService;
 import org.onlab.onos.event.impl.TestEventDispatcher;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
@@ -30,23 +32,26 @@ import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.AbstractProvider;
 import org.onlab.onos.net.provider.ProviderId;
-import org.onlab.onos.store.common.StoreService;
 import org.onlab.onos.store.device.impl.DistributedDeviceStore;
 import org.onlab.onos.store.impl.StoreManager;
+import org.onlab.onos.store.impl.TestStoreManager;
+import org.onlab.packet.IpPrefix;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.Set;
-import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 import static org.junit.Assert.*;
 import static org.onlab.onos.net.Device.Type.SWITCH;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
 
-// FIXME This test is painfully slow starting up Hazelcast on each test cases,
+// FIXME This test is slow starting up Hazelcast on each test cases.
-//       turning it off in repository for now.
 // FIXME DistributedDeviceStore should have it's own test cases.
 
 /**
@@ -67,6 +72,11 @@ public class DistributedDeviceManagerTest {
     private static final PortNumber P2 = PortNumber.portNumber(2);
     private static final PortNumber P3 = PortNumber.portNumber(3);
 
+    private static final DefaultControllerNode SELF
+        = new DefaultControllerNode(new NodeId("foobar"),
+                        IpPrefix.valueOf("127.0.0.1"));
+
+
     private DeviceManager mgr;
 
     protected StoreManager storeManager;
@@ -77,6 +87,8 @@ public class DistributedDeviceManagerTest {
     protected TestProvider provider;
     protected TestListener listener = new TestListener();
     private DistributedDeviceStore dstore;
+    private TestMastershipManager masterManager;
+    private EventDeliveryService eventService;
 
     @Before
     public void setUp() {
@@ -84,26 +96,21 @@ public class DistributedDeviceManagerTest {
         service = mgr;
         admin = mgr;
         registry = mgr;
-        // FIXME should be reading the hazelcast.xml
+        // TODO should find a way to clean Hazelcast instance without shutdown.
-        Config config = new Config();
+        Config config = TestStoreManager.getTestConfig();
-        // avoid accidentally joining other cluster
+
-        config.getGroupConfig().setName(UUID.randomUUID().toString());
+        masterManager = new TestMastershipManager();
-        // quickly form single node cluster
-        config.getNetworkConfig().getJoin()
-                .getTcpIpConfig()
-                .setEnabled(true).setConnectionTimeoutSeconds(0);
-        config.getNetworkConfig().getJoin()
-                .getMulticastConfig()
-                .setEnabled(false);
 
         storeManager = new TestStoreManager(Hazelcast.newHazelcastInstance(config));
         storeManager.activate();
 
-        dstore = new TestDistributedDeviceStore(storeManager);
+        dstore = new TestDistributedDeviceStore();
         dstore.activate();
+
         mgr.store = dstore;
-        mgr.eventDispatcher = new TestEventDispatcher();
+        eventService = new TestEventDispatcher();
-        mgr.mastershipService = new TestMastershipService();
+        mgr.eventDispatcher = eventService;
+        mgr.mastershipService = masterManager;
         mgr.activate();
 
         service.addListener(listener);
@@ -153,7 +160,7 @@ public class DistributedDeviceManagerTest {
     public void deviceDisconnected() {
         connectDevice(DID1, SW1);
         connectDevice(DID2, SW1);
-        validateEvents(DEVICE_ADDED, DEVICE_ADDED);
+        validateEvents(DEVICE_ADDED, DEVICE_ADDED, DEVICE_ADDED, DEVICE_ADDED);
         assertTrue("device should be available", service.isAvailable(DID1));
 
         // Disconnect
@@ -172,7 +179,7 @@ public class DistributedDeviceManagerTest {
     @Test
     public void deviceUpdated() {
         connectDevice(DID1, SW1);
-        validateEvents(DEVICE_ADDED);
+        validateEvents(DEVICE_ADDED, DEVICE_ADDED);
 
         connectDevice(DID1, SW2);
         validateEvents(DEVICE_UPDATED);
@@ -192,7 +199,7 @@ public class DistributedDeviceManagerTest {
         pds.add(new DefaultPortDescription(P2, true));
         pds.add(new DefaultPortDescription(P3, true));
         providerService.updatePorts(DID1, pds);
-        validateEvents(DEVICE_ADDED, PORT_ADDED, PORT_ADDED, PORT_ADDED);
+        validateEvents(DEVICE_ADDED, DEVICE_ADDED, PORT_ADDED, PORT_ADDED, PORT_ADDED);
         pds.clear();
 
         pds.add(new DefaultPortDescription(P1, false));
@@ -208,7 +215,7 @@ public class DistributedDeviceManagerTest {
         pds.add(new DefaultPortDescription(P1, true));
         pds.add(new DefaultPortDescription(P2, true));
         providerService.updatePorts(DID1, pds);
-        validateEvents(DEVICE_ADDED, PORT_ADDED, PORT_ADDED);
+        validateEvents(DEVICE_ADDED, DEVICE_ADDED, PORT_ADDED, PORT_ADDED);
 
         providerService.portStatusChanged(DID1, new DefaultPortDescription(P1, false));
         validateEvents(PORT_UPDATED);
@@ -223,7 +230,7 @@ public class DistributedDeviceManagerTest {
         pds.add(new DefaultPortDescription(P1, true));
         pds.add(new DefaultPortDescription(P2, true));
         providerService.updatePorts(DID1, pds);
-        validateEvents(DEVICE_ADDED, PORT_ADDED, PORT_ADDED);
+        validateEvents(DEVICE_ADDED, DEVICE_ADDED, PORT_ADDED, PORT_ADDED);
         assertEquals("wrong port count", 2, service.getPorts(DID1).size());
 
         Port port = service.getPort(DID1, P1);
@@ -237,10 +244,10 @@ public class DistributedDeviceManagerTest {
         connectDevice(DID2, SW2);
         assertEquals("incorrect device count", 2, service.getDeviceCount());
         admin.removeDevice(DID1);
+        validateEvents(DEVICE_ADDED, DEVICE_ADDED, DEVICE_ADDED, DEVICE_ADDED, DEVICE_REMOVED, DEVICE_REMOVED);
         assertNull("device should not be found", service.getDevice(DID1));
         assertNotNull("device should be found", service.getDevice(DID2));
         assertEquals("incorrect device count", 1, service.getDeviceCount());
-
     }
 
     protected void validateEvents(Enum... types) {
@@ -283,23 +290,21 @@ public class DistributedDeviceManagerTest {
     }
 
     private class TestDistributedDeviceStore extends DistributedDeviceStore {
-        public TestDistributedDeviceStore(StoreService storeService) {
+
-            this.storeService = storeService;
+        public TestDistributedDeviceStore() {
+            this.storeService = storeManager;
         }
     }
 
-    private class TestStoreManager extends StoreManager {
+    private static class TestMastershipManager extends MastershipServiceAdapter {
-        TestStoreManager(HazelcastInstance instance) {
-            this.instance = instance;
-        }
 
-        @Override
+        private ConcurrentMap<DeviceId, NodeId> masters = new ConcurrentHashMap<>();
-        public void activate() {
-            setupKryoPool();
-        }
-    }
 
-    private static class TestMastershipService extends MastershipServiceAdapter {
+        public TestMastershipManager() {
+            // SELF master of all initially
+            masters.put(DID1, SELF.id());
+            masters.put(DID1, SELF.id());
+        }
         @Override
         public MastershipRole getLocalRole(DeviceId deviceId) {
             return MastershipRole.MASTER;
@@ -307,13 +312,27 @@ public class DistributedDeviceManagerTest {
 
         @Override
         public Set<DeviceId> getDevicesOf(NodeId nodeId) {
-            return Sets.newHashSet(DID1, DID2);
+            HashSet<DeviceId> set = Sets.newHashSet();
+            for (Entry<DeviceId, NodeId> e : masters.entrySet()) {
+                if (e.getValue().equals(nodeId)) {
+                    set.add(e.getKey());
+                }
+            }
+            return set;
         }
 
         @Override
         public MastershipRole requestRoleFor(DeviceId deviceId) {
-            return MastershipRole.MASTER;
+            if (SELF.id().equals(masters.get(deviceId))) {
+                return MastershipRole.MASTER;
+            } else {
+                return MastershipRole.STANDBY;
+            }
+        }
+
+        @Override
+        public void relinquishMastership(DeviceId deviceId) {
+            masters.remove(deviceId, SELF.id());
         }
     }
-
 }

core/store/src/main/java/org/onlab/onos/store/cluster/impl/DistributedMastershipStore.java

@@ -4,6 +4,7 @@ import com.google.common.base.Optional;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableSet;
 import com.hazelcast.core.IMap;
+
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -42,6 +43,7 @@ public class DistributedMastershipStore
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected ClusterService clusterService;
 
+    @Override
     @Activate
     public void activate() {
         super.activate();
@@ -52,19 +54,28 @@ public class DistributedMastershipStore
         masters = new AbsentInvalidatingLoadingCache<>(newBuilder().build(nodeLoader));
         rawMasters.addEntryListener(new RemoteEventHandler<>(masters), true);
 
+        loadMasters();
+
         log.info("Started");
     }
 
+    private void loadMasters() {
+        for (byte[] keyBytes : rawMasters.keySet()) {
+            final DeviceId id = deserialize(keyBytes);
+            masters.refresh(id);
+        }
+    }
+
     @Deactivate
     public void deactivate() {
         log.info("Stopped");
     }
 
     @Override
-    public MastershipEvent setRole(NodeId nodeId, DeviceId deviceId, MastershipRole role) {
+    public MastershipEvent setMaster(NodeId nodeId, DeviceId deviceId) {
         synchronized (this) {
             NodeId currentMaster = getMaster(deviceId);
-            if (role == MastershipRole.MASTER && Objects.equals(currentMaster, nodeId)) {
+            if (Objects.equals(currentMaster, nodeId)) {
                 return null;
             }
 
@@ -94,7 +105,7 @@ public class DistributedMastershipStore
     @Override
     public MastershipRole requestRole(DeviceId deviceId) {
         // FIXME: for now we are 'selecting' as master whoever asks
-        setRole(clusterService.getLocalNode().id(), deviceId, MastershipRole.MASTER);
+        setMaster(clusterService.getLocalNode().id(), deviceId);
         return MastershipRole.MASTER;
     }
 

core/store/src/main/java/org/onlab/onos/store/device/impl/DistributedDeviceStore.java

@@ -15,7 +15,6 @@ import org.onlab.onos.net.DefaultDevice;
 import org.onlab.onos.net.DefaultPort;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.MastershipRole;
 import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.device.DeviceDescription;
@@ -61,10 +60,6 @@ public class DistributedDeviceStore
     private IMap<byte[], byte[]> rawDevices;
     private LoadingCache<DeviceId, Optional<DefaultDevice>> devices;
 
-    // private IMap<DeviceId, MastershipRole> roles;
-    private IMap<byte[], byte[]> rawRoles;
-    private LoadingCache<DeviceId, Optional<MastershipRole>> roles;
-
     // private ISet<DeviceId> availableDevices;
     private ISet<byte[]> availableDevices;
 
@@ -73,6 +68,7 @@ public class DistributedDeviceStore
     private IMap<byte[], byte[]> rawDevicePorts;
     private LoadingCache<DeviceId, Optional<Map<PortNumber, Port>>> devicePorts;
 
+    @Override
     @Activate
     public void activate() {
         super.activate();
@@ -86,14 +82,7 @@ public class DistributedDeviceStore
                 = new OptionalCacheLoader<>(storeService, rawDevices);
         devices = new AbsentInvalidatingLoadingCache<>(newBuilder().build(deviceLoader));
         // refresh/populate cache based on notification from other instance
-        rawDevices.addEntryListener(new RemoteEventHandler<>(devices), includeValue);
+        rawDevices.addEntryListener(new RemoteDeviceEventHandler(devices), includeValue);
-
-        rawRoles = theInstance.getMap("roles");
-        final OptionalCacheLoader<DeviceId, MastershipRole> rolesLoader
-                = new OptionalCacheLoader<>(storeService, rawRoles);
-        roles = new AbsentInvalidatingLoadingCache<>(newBuilder().build(rolesLoader));
-        // refresh/populate cache based on notification from other instance
-        rawRoles.addEntryListener(new RemoteEventHandler<>(roles), includeValue);
 
         // TODO cache availableDevices
         availableDevices = theInstance.getSet("availableDevices");
@@ -103,7 +92,9 @@ public class DistributedDeviceStore
                 = new OptionalCacheLoader<>(storeService, rawDevicePorts);
         devicePorts = new AbsentInvalidatingLoadingCache<>(newBuilder().build(devicePortLoader));
         // refresh/populate cache based on notification from other instance
-        rawDevicePorts.addEntryListener(new RemoteEventHandler<>(devicePorts), includeValue);
+        rawDevicePorts.addEntryListener(new RemotePortEventHandler(devicePorts), includeValue);
+
+        loadDeviceCache();
 
         log.info("Started");
     }
@@ -115,22 +106,11 @@ public class DistributedDeviceStore
 
     @Override
     public int getDeviceCount() {
-        // TODO IMap size or cache size?
+        return devices.asMap().size();
-        return rawDevices.size();
     }
 
     @Override
     public Iterable<Device> getDevices() {
-// TODO Revisit if we ever need to do this.
-//        log.info("{}:{}", rawMap.size(), cache.size());
-//        if (rawMap.size() != cache.size()) {
-//            for (Entry<byte[], byte[]> e : rawMap.entrySet()) {
-//                final DeviceId key = deserialize(e.getKey());
-//                final DefaultDevice val = deserialize(e.getValue());
-//                cache.put(key, val);
-//            }
-//        }
-
         // TODO builder v.s. copyOf. Guava semms to be using copyOf?
         Builder<Device> builder = ImmutableSet.builder();
         for (Optional<DefaultDevice> e : devices.asMap().values()) {
@@ -141,6 +121,17 @@ public class DistributedDeviceStore
         return builder.build();
     }
 
+    private void loadDeviceCache() {
+        log.info("{}:{}", rawDevices.size(), devices.size());
+        if (rawDevices.size() != devices.size()) {
+            for (Map.Entry<byte[], byte[]> e : rawDevices.entrySet()) {
+                final DeviceId key = deserialize(e.getKey());
+                final DefaultDevice val = deserialize(e.getValue());
+                devices.put(key, Optional.of(val));
+            }
+        }
+    }
+
     @Override
     public Device getDevice(DeviceId deviceId) {
         // TODO revisit if ignoring exception is safe.
@@ -171,12 +162,8 @@ public class DistributedDeviceStore
             devices.put(deviceId, Optional.of(device));
 
             availableDevices.add(deviceIdBytes);
-
-            // For now claim the device as a master automatically.
-            //rawRoles.put(deviceIdBytes, serialize(MastershipRole.MASTER));
-            //roles.put(deviceId, Optional.of(MastershipRole.MASTER));
         }
-        return new DeviceEvent(DeviceEvent.Type.DEVICE_ADDED, device, null);
+        return new DeviceEvent(DEVICE_ADDED, device, null);
     }
 
     // Updates the device and returns the appropriate event if necessary.
@@ -348,8 +335,6 @@ public class DistributedDeviceStore
     public DeviceEvent removeDevice(DeviceId deviceId) {
         synchronized (this) {
             byte[] deviceIdBytes = serialize(deviceId);
-            rawRoles.remove(deviceIdBytes);
-            roles.invalidate(deviceId);
 
             // TODO conditional remove?
             Device device = deserialize(rawDevices.remove(deviceIdBytes));
@@ -359,6 +344,48 @@ public class DistributedDeviceStore
         }
     }
 
-    // TODO cache serialized DeviceID if we suffer from serialization cost
+    private class RemoteDeviceEventHandler extends RemoteEventHandler<DeviceId, DefaultDevice> {
+        public RemoteDeviceEventHandler(LoadingCache<DeviceId, Optional<DefaultDevice>> cache) {
+            super(cache);
+        }
 
+        @Override
+        protected void onAdd(DeviceId deviceId, DefaultDevice device) {
+            delegate.notify(new DeviceEvent(DEVICE_ADDED, device));
+        }
+
+        @Override
+        protected void onRemove(DeviceId deviceId, DefaultDevice device) {
+            delegate.notify(new DeviceEvent(DEVICE_REMOVED, device));
+        }
+
+        @Override
+        protected void onUpdate(DeviceId deviceId, DefaultDevice device) {
+            delegate.notify(new DeviceEvent(DEVICE_UPDATED, device));
+        }
+    }
+
+    private class RemotePortEventHandler extends RemoteEventHandler<DeviceId, Map<PortNumber, Port>> {
+        public RemotePortEventHandler(LoadingCache<DeviceId, Optional<Map<PortNumber, Port>>> cache) {
+            super(cache);
+        }
+
+        @Override
+        protected void onAdd(DeviceId deviceId, Map<PortNumber, Port> ports) {
+//            delegate.notify(new DeviceEvent(PORT_ADDED, getDevice(deviceId)));
+        }
+
+        @Override
+        protected void onRemove(DeviceId deviceId, Map<PortNumber, Port> ports) {
+//            delegate.notify(new DeviceEvent(PORT_REMOVED, getDevice(deviceId)));
+        }
+
+        @Override
+        protected void onUpdate(DeviceId deviceId, Map<PortNumber, Port> ports) {
+//            delegate.notify(new DeviceEvent(PORT_UPDATED, getDevice(deviceId)));
+        }
+    }
+
+
+    // TODO cache serialized DeviceID if we suffer from serialization cost
 }

core/store/src/main/java/org/onlab/onos/store/impl/AbstractDistributedStore.java

@@ -24,7 +24,7 @@ import static org.slf4j.LoggerFactory.getLogger;
  */
 @Component(componentAbstract = true)
 public abstract class AbstractDistributedStore<E extends Event, D extends StoreDelegate<E>>
-            extends AbstractStore<E, D> {
+        extends AbstractStore<E, D> {
 
     protected final Logger log = getLogger(getClass());
 
@@ -66,7 +66,7 @@ public abstract class AbstractDistributedStore<E extends Event, D extends StoreD
      * @param <K> IMap key type after deserialization
      * @param <V> IMap value type after deserialization
      */
-    public final class RemoteEventHandler<K, V> extends EntryAdapter<byte[], byte[]> {
+    public class RemoteEventHandler<K, V> extends EntryAdapter<byte[], byte[]> {
 
         private LoadingCache<K, Optional<V>> cache;
 
@@ -84,20 +84,59 @@ public abstract class AbstractDistributedStore<E extends Event, D extends StoreD
             cache.invalidateAll();
         }
 
+        @Override
+        public void entryAdded(EntryEvent<byte[], byte[]> event) {
+            K key = deserialize(event.getKey());
+            V newVal = deserialize(event.getValue());
+            Optional<V> newValue = Optional.of(newVal);
+            cache.asMap().putIfAbsent(key, newValue);
+            onAdd(key, newVal);
+        }
+
         @Override
         public void entryUpdated(EntryEvent<byte[], byte[]> event) {
-            cache.put(storeService.<K>deserialize(event.getKey()),
+            K key = deserialize(event.getKey());
-                      Optional.of(storeService.<V>deserialize(event.getValue())));
+            V oldVal = deserialize(event.getOldValue());
+            Optional<V> oldValue = Optional.fromNullable(oldVal);
+            V newVal = deserialize(event.getValue());
+            Optional<V> newValue = Optional.of(newVal);
+            cache.asMap().replace(key, oldValue, newValue);
+            onUpdate(key, newVal);
         }
 
         @Override
         public void entryRemoved(EntryEvent<byte[], byte[]> event) {
-            cache.invalidate(storeService.<K>deserialize(event.getKey()));
+            K key = deserialize(event.getKey());
+            V val = deserialize(event.getValue());
+            cache.invalidate(key);
+            onRemove(key, val);
         }
 
-        @Override
+        /**
-        public void entryAdded(EntryEvent<byte[], byte[]> event) {
+         * Cache entry addition hook.
-            entryUpdated(event);
+         *
+         * @param key    new key
+         * @param newVal new value
+         */
+        protected void onAdd(K key, V newVal) {
+        }
+
+        /**
+         * Cache entry update hook.
+         *
+         * @param key    new key
+         * @param newVal new value
+         */
+        protected void onUpdate(K key, V newVal) {
+        }
+
+        /**
+         * Cache entry remove hook.
+         *
+         * @param key new key
+         * @param val old value
+         */
+        protected void onRemove(K key, V val) {
         }
     }
 

core/store/src/main/java/org/onlab/onos/store/impl/StoreManager.java

@@ -45,7 +45,7 @@ import java.util.HashMap;
 @Service
 public class StoreManager implements StoreService {
 
-    private static final String HAZELCAST_XML_FILE = "etc/hazelcast.xml";
+    protected static final String HAZELCAST_XML_FILE = "etc/hazelcast.xml";
 
     private final Logger log = LoggerFactory.getLogger(getClass());
 

core/store/src/test/java/org/onlab/onos/store/impl/TestStoreManager.java

@@ -0,0 +1,54 @@
+package org.onlab.onos.store.impl;
+
+import java.io.FileNotFoundException;
+import java.util.UUID;
+
+import com.hazelcast.config.Config;
+import com.hazelcast.config.FileSystemXmlConfig;
+import com.hazelcast.core.HazelcastInstance;
+
+/**
+ * Dummy StoreManager to use specified Hazelcast instance.
+ */
+public class TestStoreManager extends StoreManager {
+
+    /**
+     * Gets the Hazelcast Config for testing.
+     *
+     * @return
+     */
+    public static Config getTestConfig() {
+        Config config;
+        try {
+            config = new FileSystemXmlConfig(HAZELCAST_XML_FILE);
+        } catch (FileNotFoundException e) {
+            // falling back to default
+            config = new Config();
+        }
+        // avoid accidentally joining other cluster
+        config.getGroupConfig().setName(UUID.randomUUID().toString());
+        // quickly form single node cluster
+        config.getNetworkConfig().getJoin()
+            .getTcpIpConfig()
+            .setEnabled(true).setConnectionTimeoutSeconds(0);
+        config.getNetworkConfig().getJoin()
+            .getMulticastConfig()
+            .setEnabled(false);
+        return config;
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param instance Hazelast instance to return on #getHazelcastInstance()
+     */
+    public TestStoreManager(HazelcastInstance instance) {
+        this.instance = instance;
+    }
+
+    // Hazelcast setup removed from original code.
+    @Override
+    public void activate() {
+        setupKryoPool();
+    }
+}

core/trivial/src/main/java/org/onlab/onos/net/trivial/impl/SimpleDeviceStore.java

@@ -15,8 +15,10 @@ import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.device.DeviceDescription;
 import org.onlab.onos.net.device.DeviceEvent;
 import org.onlab.onos.net.device.DeviceStore;
+import org.onlab.onos.net.device.DeviceStoreDelegate;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.AbstractStore;
 import org.slf4j.Logger;
 
 import java.util.ArrayList;
@@ -40,7 +42,9 @@ import static org.slf4j.LoggerFactory.getLogger;
  */
 @Component(immediate = true)
 @Service
-public class SimpleDeviceStore implements DeviceStore {
+public class SimpleDeviceStore
+        extends AbstractStore<DeviceEvent, DeviceStoreDelegate>
+        implements DeviceStore {
 
     private final Logger log = getLogger(getClass());
 

core/trivial/src/main/java/org/onlab/onos/net/trivial/impl/SimpleFlowRuleStore.java

@@ -19,6 +19,8 @@ import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleEvent.Type;
 import org.onlab.onos.net.flow.FlowRuleStore;
+import org.onlab.onos.net.flow.FlowRuleStoreDelegate;
+import org.onlab.onos.store.AbstractStore;
 import org.slf4j.Logger;
 
 import com.google.common.collect.ArrayListMultimap;
@@ -30,7 +32,9 @@ import com.google.common.collect.Multimap;
  */
 @Component(immediate = true)
 @Service
-public class SimpleFlowRuleStore implements FlowRuleStore {
+public class SimpleFlowRuleStore
+        extends AbstractStore<FlowRuleEvent, FlowRuleStoreDelegate>
+        implements FlowRuleStore {
 
     private final Logger log = getLogger(getClass());
 

core/trivial/src/main/java/org/onlab/onos/net/trivial/impl/SimpleHostStore.java

@@ -24,8 +24,10 @@ import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostStore;
+import org.onlab.onos.net.host.HostStoreDelegate;
 import org.onlab.onos.net.host.PortAddresses;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.AbstractStore;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
@@ -41,7 +43,9 @@ import com.google.common.collect.Multimap;
  */
 @Component(immediate = true)
 @Service
-public class SimpleHostStore implements HostStore {
+public class SimpleHostStore
+        extends AbstractStore<HostEvent, HostStoreDelegate>
+        implements HostStore {
 
     private final Logger log = getLogger(getClass());
 

core/trivial/src/main/java/org/onlab/onos/net/trivial/impl/SimpleLinkStore.java

@@ -14,7 +14,9 @@ import org.onlab.onos.net.Link;
 import org.onlab.onos.net.link.LinkDescription;
 import org.onlab.onos.net.link.LinkEvent;
 import org.onlab.onos.net.link.LinkStore;
+import org.onlab.onos.net.link.LinkStoreDelegate;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.AbstractStore;
 import org.slf4j.Logger;
 
 import java.util.Collections;
@@ -35,7 +37,9 @@ import static org.slf4j.LoggerFactory.getLogger;
  */
 @Component(immediate = true)
 @Service
-public class SimpleLinkStore implements LinkStore {
+public class SimpleLinkStore
+        extends AbstractStore<LinkEvent, LinkStoreDelegate>
+        implements LinkStore {
 
     private final Logger log = getLogger(getClass());
 

core/trivial/src/main/java/org/onlab/onos/net/trivial/impl/SimpleMastershipStore.java

@@ -3,6 +3,8 @@ package org.onlab.onos.net.trivial.impl;
 import static org.slf4j.LoggerFactory.getLogger;
 
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -15,9 +17,11 @@ import org.onlab.onos.cluster.ControllerNode;
 import org.onlab.onos.cluster.DefaultControllerNode;
 import org.onlab.onos.cluster.MastershipEvent;
 import org.onlab.onos.cluster.MastershipStore;
+import org.onlab.onos.cluster.MastershipStoreDelegate;
 import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.MastershipRole;
+import org.onlab.onos.store.AbstractStore;
 import org.onlab.packet.IpPrefix;
 import org.slf4j.Logger;
 
@@ -25,24 +29,27 @@ import static org.onlab.onos.cluster.MastershipEvent.Type.*;
 
 /**
  * Manages inventory of controller mastership over devices using
- * trivial in-memory structures implementation.
+ * trivial, non-distributed in-memory structures implementation.
  */
 @Component(immediate = true)
 @Service
-public class SimpleMastershipStore implements MastershipStore {
+public class SimpleMastershipStore
-
+        extends AbstractStore<MastershipEvent, MastershipStoreDelegate>
-    public static final IpPrefix LOCALHOST = IpPrefix.valueOf("127.0.0.1");
+        implements MastershipStore {
 
     private final Logger log = getLogger(getClass());
 
-    private ControllerNode instance;
+    public static final IpPrefix LOCALHOST = IpPrefix.valueOf("127.0.0.1");
 
-    protected final ConcurrentMap<DeviceId, MastershipRole> roleMap =
+    private ControllerNode instance =
+            new DefaultControllerNode(new NodeId("local"), LOCALHOST);
+
+    //devices mapped to their masters, to emulate multiple nodes
+    protected final ConcurrentMap<DeviceId, NodeId> masterMap =
             new ConcurrentHashMap<>();
 
     @Activate
     public void activate() {
-        instance = new DefaultControllerNode(new NodeId("local"), LOCALHOST);
         log.info("Started");
     }
 
@@ -52,23 +59,36 @@ public class SimpleMastershipStore implements MastershipStore {
     }
 
     @Override
-    public MastershipEvent setRole(NodeId nodeId, DeviceId deviceId,
+    public MastershipEvent setMaster(NodeId nodeId, DeviceId deviceId) {
-                                   MastershipRole role) {
+
-        if (roleMap.get(deviceId) == null) {
+        NodeId node = masterMap.get(deviceId);
-            return null;
+        if (node == null) {
+            masterMap.put(deviceId, nodeId);
+            return new MastershipEvent(MASTER_CHANGED, deviceId, nodeId);
+        }
+
+        if (node.equals(nodeId)) {
+            return null;
+        } else {
+            masterMap.put(deviceId, nodeId);
+            return new MastershipEvent(MASTER_CHANGED, deviceId, nodeId);
         }
-        roleMap.put(deviceId, role);
-        return new MastershipEvent(MASTER_CHANGED, deviceId, nodeId);
     }
 
     @Override
     public NodeId getMaster(DeviceId deviceId) {
-        return instance.id();
+        return masterMap.get(deviceId);
     }
 
     @Override
     public Set<DeviceId> getDevices(NodeId nodeId) {
-        return Collections.unmodifiableSet(roleMap.keySet());
+        Set<DeviceId> ids = new HashSet<>();
+        for (Map.Entry<DeviceId, NodeId> d : masterMap.entrySet()) {
+            if (d.getValue().equals(nodeId)) {
+                ids.add(d.getKey());
+            }
+        }
+        return Collections.unmodifiableSet(ids);
     }
 
     @Override
@@ -78,11 +98,18 @@ public class SimpleMastershipStore implements MastershipStore {
 
     @Override
     public MastershipRole getRole(NodeId nodeId, DeviceId deviceId) {
-        MastershipRole role = roleMap.get(deviceId);
+        NodeId node = masterMap.get(deviceId);
-        if (role == null) {
+        MastershipRole role;
-            //say MASTER. If clustered, we'd figure out if anyone's got dibs here.
+        if (node != null) {
+            if (node.equals(nodeId)) {
+                role = MastershipRole.MASTER;
+            } else {
+                role = MastershipRole.STANDBY;
+            }
+        } else {
+            //masterMap doesn't contain it.
             role = MastershipRole.MASTER;
-            roleMap.put(deviceId, role);
+            masterMap.put(deviceId, nodeId);
         }
         return role;
     }

core/trivial/src/main/java/org/onlab/onos/net/trivial/impl/SimpleTopologyStore.java

@@ -18,6 +18,8 @@ import org.onlab.onos.net.topology.TopologyCluster;
 import org.onlab.onos.net.topology.TopologyEvent;
 import org.onlab.onos.net.topology.TopologyGraph;
 import org.onlab.onos.net.topology.TopologyStore;
+import org.onlab.onos.net.topology.TopologyStoreDelegate;
+import org.onlab.onos.store.AbstractStore;
 import org.slf4j.Logger;
 
 import java.util.List;
@@ -31,7 +33,9 @@ import static org.slf4j.LoggerFactory.getLogger;
  */
 @Component(immediate = true)
 @Service
-public class SimpleTopologyStore implements TopologyStore {
+public class SimpleTopologyStore
+        extends AbstractStore<TopologyEvent, TopologyStoreDelegate>
+        implements TopologyStore {
 
     private final Logger log = getLogger(getClass());
 

openflow/api/pom.xml

@@ -16,6 +16,18 @@
 
     <description>ONOS OpenFlow controller subsystem API</description>
 
+    <repositories>
+        <!-- FIXME: for Loxigen. Decide how to use Loxigen before release. -->
+        <repository>
+            <id>sonatype-oss-snapshot</id>
+            <name>Sonatype OSS snapshot repository</name>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+        </repository>
+    </repositories>
+
     <dependencies>
         <dependency>
             <groupId>org.projectfloodlight</groupId>

pom.xml

@@ -391,13 +391,13 @@
                         <group>
                             <title>Network Model &amp; Services</title>
                             <packages>
-                                org.onlab.onos:org.onlab.onos.*:
+                                org.onlab.onos:org.onlab.onos.*
                             </packages>
                         </group>
                         <group>
                             <title>Core Subsystems</title>
                             <packages>
-                                org.onlab.onos.cluster.impl:org.onlab.onos.store:org.onlab.onos.net.device.impl:org.onlab.onos.net.link.impl:org.onlab.onos.net.host.impl:org.onlab.onos.net.topology.impl:org.onlab.onos.net.packet.impl:org.onlab.onos.net.flow.impl:org.onlab.onos.net.trivial.*:org.onlab.onos.net.*.impl:org.onlab.onos.cluster:org.onlab.onos.event.impl:org.onlab.onos.store.*
+                                org.onlab.onos.cluster.impl:org.onlab.onos.net.device.impl:org.onlab.onos.net.link.impl:org.onlab.onos.net.host.impl:org.onlab.onos.net.topology.impl:org.onlab.onos.net.packet.impl:org.onlab.onos.net.flow.impl:org.onlab.onos.net.trivial.*:org.onlab.onos.net.*.impl:org.onlab.onos.event.impl:org.onlab.onos.store.*
                             </packages>
                         </group>
                         <group>
@@ -422,7 +422,7 @@
                         <group>
                             <title>Sample Applications</title>
                             <packages>
-                                org.onlab.onos.tvue:org.onlab.onos.fwd
+                                org.onlab.onos.tvue:org.onlab.onos.fwd:org.onlab.onos.foo
                             </packages>
                         </group>
                     </groups>

tools/build/onos-package

@@ -51,7 +51,7 @@ perl -pi.old -e "s|^(featuresRepositories=.*)|\1,mvn:org.onlab.onos/onos-feature
     $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg 
 
 # Patch the Apache Karaf distribution file to load ONOS features
-perl -pi.old -e 's|^(featuresBoot=.*)|\1,onos-api,onos-core,onos-cli,onos-rest,onos-gui,onos-openflow,onos-app-fwd,onos-app-foo|' \
+perl -pi.old -e 's|^(featuresBoot=.*)|\1,webconsole,onos-api,onos-core,onos-cli,onos-rest,onos-gui,onos-openflow,onos-app-fwd,onos-app-foo|' \
     $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg
 
 # Patch the Apache Karaf distribution with ONOS branding bundle

tools/test/bin/onos-config

@@ -8,4 +8,7 @@
 
 remote=$ONOS_USER@${1:-$OCI}
 
-echo "Deprecated!"
+ssh $remote "
+    sudo perl -pi.bak -e \"s/            <interface>.*</            <interface>${ONOS_NIC:-192.168.56.*}</g\" \
+        $ONOS_INSTALL_DIR/$KARAF_DIST/etc/hazelcast.xml
+"

tools/test/bin/onos-install

@@ -9,7 +9,8 @@
 # If the first option is -f attempt uninstall first.
 [ "$1" = "-f" ] && shift && onos-uninstall ${1:-$OCI}
 
-remote=$ONOS_USER@${1:-$OCI}
+node=${1:-$OCI}
+remote=$ONOS_USER@$node
 
 scp -q $ONOS_TAR $remote:/tmp
 
@@ -30,7 +31,10 @@ ssh $remote "
 
     # Remove any previous ON.Lab bits from ~/.m2 repo
     rm -fr ~/.m2/repository/org/onlab
-
-    # Ignite the ONOS service.
-    sudo service onos start
 "
+
+# Configure the ONOS installation
+onos-config $node
+
+# Ignite the ONOS service.
+onos-service $node start

tools/test/cells/.reset

@@ -0,0 +1 @@
+unset OC1 OC2 OC3 OC4 OC5 OC6 OC7 OC8 OC9 OCN ONOS_NIC

tools/test/cells/local

@@ -1,4 +1,5 @@
 # Default virtual box ONOS instances 1,2 & ONOS mininet box
+. $ONOS_ROOT/tools/test/cells/.reset
 
 export OC1="192.168.56.101"
 export OC2="192.168.56.102"

tools/test/cells/prox

@@ -0,0 +1,9 @@
+# ProxMox-based cell of ONOS instances 1,2 & ONOS mininet box
+. $ONOS_ROOT/tools/test/cells/.reset
+
+export ONOS_NIC="10.1.9.*"
+
+export OC1="10.1.9.94"
+export OC2="10.1.9.82"
+
+export OCN="10.1.9.93"