diff --git a/manager/allocator/networkallocator/portallocator.go b/manager/allocator/networkallocator/portallocator.go
index de5412e777..8c56f7c910 100644
--- a/manager/allocator/networkallocator/portallocator.go
+++ b/manager/allocator/networkallocator/portallocator.go
@@ -38,6 +38,71 @@ type portSpace struct {
 	dynamicPortSpace *idm.Idm
 }
 
+type allocatedPorts map[api.PortConfig]map[uint32]*api.PortConfig
+
+// addState add the state of an allocated port to the collection.
+// `allocatedPorts` is a map of portKey:publishedPort:portState.
+// In case the value of the portKey is missing, the map
+// publishedPort:portState is created automatically
+func (ps allocatedPorts) addState(p *api.PortConfig) {
+	portKey := getPortConfigKey(p)
+	if _, ok := ps[portKey]; !ok {
+		ps[portKey] = make(map[uint32]*api.PortConfig)
+	}
+	ps[portKey][p.PublishedPort] = p
+}
+
+// delState delete the state of an allocated port from the collection.
+// `allocatedPorts` is a map of portKey:publishedPort:portState.
+//
+// If publishedPort is non-zero, then it is user defined. We will try to
+// remove the portState from `allocatedPorts` directly and return
+// the portState (or nil if no portState exists)
+//
+// If publishedPort is zero, then it is dynamically allocated. We will try
+// to remove the portState from `allocatedPorts`, as long as there is
+// a portState associated with a non-zero publishedPort.
+// Note multiple dynamically allocated ports might exists. In this case,
+// we will remove only at a time so both allocated ports are tracked.
+//
+// Note because of the potential co-existence of user-defined and dynamically
+// allocated ports, delState has to be called for user-defined port first.
+// dynamically allocated ports should be removed later.
+func (ps allocatedPorts) delState(p *api.PortConfig) *api.PortConfig {
+	portKey := getPortConfigKey(p)
+
+	portStateMap, ok := ps[portKey]
+
+	// If name, port, protocol values don't match then we
+	// are not allocated.
+	if !ok {
+		return nil
+	}
+
+	if p.PublishedPort != 0 {
+		// If SwarmPort was user defined but the port state
+		// SwarmPort doesn't match we are not allocated.
+		v := portStateMap[p.PublishedPort]
+
+		// Delete state from allocatedPorts
+		delete(portStateMap, p.PublishedPort)
+
+		return v
+	}
+
+	// If PublishedPort == 0 and we don't have non-zero port
+	// then we are not allocated
+	for publishedPort, v := range portStateMap {
+		if publishedPort != 0 {
+			// Delete state from allocatedPorts
+			delete(portStateMap, publishedPort)
+			return v
+		}
+	}
+
+	return nil
+}
+
 func newPortAllocator() (*portAllocator, error) {
 	portSpaces := make(map[api.PortConfig_Protocol]*portSpace)
 	for _, protocol := range []api.PortConfig_Protocol{api.ProtocolTCP, api.ProtocolUDP} {