Merge pull request #50 from squeed/host-local-multi

ipam/host-local: support sets of disjoint ranges

Author: dcbw

plugins/ipam/host-local/README.md

@@ -8,24 +8,40 @@ it can include a DNS configuration from a `resolv.conf` file on the host.
 host-local IPAM plugin allocates ip addresses out of a set of address ranges.
 It stores the state locally on the host filesystem, therefore ensuring uniqueness of IP addresses on a single host.
 
+The allocator can allocate multiple ranges, and supports sets of multiple (disjoint) 
+subnets. The allocation strategy is loosely round-robin within each range set.
+
 ## Example configurations
 
+Note that the key `ranges` is a list of range sets. That is to say, the length 
+of the top-level array is the number of addresses returned. The second-level 
+array is a set of subnets to use as a pool of possible addresses.
+
+This example configuration returns 2 IP addresses.
+
 ```json
 {
 	"ipam": {
 		"type": "host-local",
 		"ranges": [
-			{
-				"subnet": "10.10.0.0/16",
-				"rangeStart": "10.10.1.20",
-				"rangeEnd": "10.10.3.50",
-				"gateway": "10.10.0.254"
-			},
-			{
-				"subnet": "3ffe:ffff:0:01ff::/64",
-				"rangeStart": "3ffe:ffff:0:01ff::0010",
-				"rangeEnd": "3ffe:ffff:0:01ff::0020"
-			}
+			[
+				{
+					"subnet": "10.10.0.0/16",
+					"rangeStart": "10.10.1.20",
+					"rangeEnd": "10.10.3.50",
+					"gateway": "10.10.0.254"
+				},
+				{
+					"subnet": "172.16.5.0/24"
+				}
+			],
+			[
+				{
+					"subnet": "3ffe:ffff:0:01ff::/64",
+					"rangeStart": "3ffe:ffff:0:01ff::0010",
+					"rangeEnd": "3ffe:ffff:0:01ff::0020"
+				}
+			]
 		],
 		"routes": [
 			{ "dst": "0.0.0.0/0" },
@@ -58,7 +74,7 @@ deprecated but still supported.
 We can test it out on the command-line:
 
 ```bash
-$ echo '{ "cniVersion": "0.3.1", "name": "examplenet", "ipam": { "type": "host-local", "ranges": [ {"subnet": "203.0.113.0/24"}, {"subnet": "2001:db8:1::/64"}], "dataDir": "/tmp/cni-example"  } }' | CNI_COMMAND=ADD CNI_CONTAINERID=example CNI_NETNS=/dev/null CNI_IFNAME=dummy0 CNI_PATH=. ./host-local
+$ echo '{ "cniVersion": "0.3.1", "name": "examplenet", "ipam": { "type": "host-local", "ranges": [ [{"subnet": "203.0.113.0/24"}], [{"subnet": "2001:db8:1::/64"}]], "dataDir": "/tmp/cni-example"  } }' | CNI_COMMAND=ADD CNI_CONTAINERID=example CNI_NETNS=/dev/null CNI_IFNAME=dummy0 CNI_PATH=. ./host-local
 
 ```
 
@@ -86,7 +102,7 @@ $ echo '{ "cniVersion": "0.3.1", "name": "examplenet", "ipam": { "type": "host-l
 * `routes` (string, optional): list of routes to add to the container namespace. Each route is a dictionary with "dst" and optional "gw" fields. If "gw" is omitted, value of "gateway" will be used.
 * `resolvConf` (string, optional): Path to a `resolv.conf` on the host to parse and return as the DNS configuration
 * `dataDir` (string, optional): Path to a directory to use for maintaining state, e.g. which IPs have been allocated to which containers
-* `ranges`, (array, required, nonempty) an array of range objects:
+* `ranges`, (array, required, nonempty) an array of arrays of range objects:
 	* `subnet` (string, required): CIDR block to allocate out of.
 	* `rangeStart` (string, optional): IP inside of "subnet" from which to start allocating addresses. Defaults to ".2" IP inside of the "subnet" block.
 	* `rangeEnd` (string, optional): IP inside of "subnet" with which to end allocating addresses. Defaults to ".254" IP inside of the "subnet" block for ipv4, ".255" for IPv6

plugins/ipam/host-local/backend/allocator/allocator.go

@@ -15,41 +15,28 @@
 package allocator
 
 import (
-	"encoding/base64"
 	"fmt"
 	"log"
 	"net"
 	"os"
+	"strconv"
 
 	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/plugins/ipam/host-local/backend"
 )
 
 type IPAllocator struct {
-	netName string
-	ipRange Range
-	store   backend.Store
-	rangeID string // Used for tracking last reserved ip
+	rangeset *RangeSet
+	store    backend.Store
+	rangeID  string // Used for tracking last reserved ip
 }
 
-type RangeIter struct {
-	low   net.IP
-	high  net.IP
-	cur   net.IP
-	start net.IP
-}
-
-func NewIPAllocator(netName string, r Range, store backend.Store) *IPAllocator {
-	// The range name (last allocated ip suffix) is just the base64
-	// encoding of the bytes of the first IP
-	rangeID := base64.URLEncoding.EncodeToString(r.RangeStart)
-
+func NewIPAllocator(s *RangeSet, store backend.Store, id int) *IPAllocator {
 	return &IPAllocator{
-		netName: netName,
-		ipRange: r,
-		store:   store,
-		rangeID: rangeID,
+		rangeset: s,
+		store:    store,
+		rangeID:  strconv.Itoa(id),
 	}
 }
 
@@ -58,67 +45,66 @@ func (a *IPAllocator) Get(id string, requestedIP net.IP) (*current.IPConfig, err
 	a.store.Lock()
 	defer a.store.Unlock()
 
-	gw := a.ipRange.Gateway
-
-	var reservedIP net.IP
+	var reservedIP *net.IPNet
+	var gw net.IP
 
 	if requestedIP != nil {
-		if gw != nil && gw.Equal(requestedIP) {
-			return nil, fmt.Errorf("requested IP must differ from gateway IP")
+		if err := canonicalizeIP(&requestedIP); err != nil {
+			return nil, err
 		}
 
-		if err := a.ipRange.IPInRange(requestedIP); err != nil {
+		r, err := a.rangeset.RangeFor(requestedIP)
+		if err != nil {
 			return nil, err
 		}
 
+		if requestedIP.Equal(r.Gateway) {
+			return nil, fmt.Errorf("requested ip %s is subnet's gateway", requestedIP.String())
+		}
+
 		reserved, err := a.store.Reserve(id, requestedIP, a.rangeID)
 		if err != nil {
 			return nil, err
 		}
 		if !reserved {
-			return nil, fmt.Errorf("requested IP address %q is not available in network: %s %s", requestedIP, a.netName, (*net.IPNet)(&a.ipRange.Subnet).String())
+			return nil, fmt.Errorf("requested IP address %s is not available in range set %s", requestedIP, a.rangeset.String())
 		}
-		reservedIP = requestedIP
+		reservedIP = &net.IPNet{IP: requestedIP, Mask: r.Subnet.Mask}
+		gw = r.Gateway
 
 	} else {
 		iter, err := a.GetIter()
 		if err != nil {
 			return nil, err
 		}
 		for {
-			cur := iter.Next()
-			if cur == nil {
+			reservedIP, gw = iter.Next()
+			if reservedIP == nil {
 				break
 			}
 
-			// don't allocate gateway IP
-			if gw != nil && cur.Equal(gw) {
-				continue
-			}
-
-			reserved, err := a.store.Reserve(id, cur, a.rangeID)
+			reserved, err := a.store.Reserve(id, reservedIP.IP, a.rangeID)
 			if err != nil {
 				return nil, err
 			}
 
 			if reserved {
-				reservedIP = cur
 				break
 			}
 		}
 	}
 
 	if reservedIP == nil {
-		return nil, fmt.Errorf("no IP addresses available in network: %s %s", a.netName, (*net.IPNet)(&a.ipRange.Subnet).String())
+		return nil, fmt.Errorf("no IP addresses available in range set: %s", a.rangeset.String())
 	}
 	version := "4"
-	if reservedIP.To4() == nil {
+	if reservedIP.IP.To4() == nil {
 		version = "6"
 	}
 
 	return &current.IPConfig{
 		Version: version,
-		Address: net.IPNet{IP: reservedIP, Mask: a.ipRange.Subnet.Mask},
+		Address: *reservedIP,
 		Gateway: gw,
 	}, nil
 }
@@ -131,15 +117,28 @@ func (a *IPAllocator) Release(id string) error {
 	return a.store.ReleaseByID(id)
 }
 
+type RangeIter struct {
+	rangeset *RangeSet
+
+	// The current range id
+	rangeIdx int
+
+	// Our current position
+	cur net.IP
+
+	// The IP and range index where we started iterating; if we hit this again, we're done.
+	startIP    net.IP
+	startRange int
+}
+
 // GetIter encapsulates the strategy for this allocator.
-// We use a round-robin strategy, attempting to evenly use the whole subnet.
+// We use a round-robin strategy, attempting to evenly use the whole set.
 // More specifically, a crash-looping container will not see the same IP until
 // the entire range has been run through.
 // We may wish to consider avoiding recently-released IPs in the future.
 func (a *IPAllocator) GetIter() (*RangeIter, error) {
-	i := RangeIter{
-		low:  a.ipRange.RangeStart,
-		high: a.ipRange.RangeEnd,
+	iter := RangeIter{
+		rangeset: a.rangeset,
 	}
 
 	// Round-robin by trying to allocate from the last reserved IP + 1
@@ -151,39 +150,68 @@ func (a *IPAllocator) GetIter() (*RangeIter, error) {
 	if err != nil && !os.IsNotExist(err) {
 		log.Printf("Error retrieving last reserved ip: %v", err)
 	} else if lastReservedIP != nil {
-		startFromLastReservedIP = a.ipRange.IPInRange(lastReservedIP) == nil
+		startFromLastReservedIP = a.rangeset.Contains(lastReservedIP)
 	}
 
+	// Find the range in the set with this IP
 	if startFromLastReservedIP {
-		if i.high.Equal(lastReservedIP) {
-			i.start = i.low
-		} else {
-			i.start = ip.NextIP(lastReservedIP)
+		for i, r := range *a.rangeset {
+			if r.Contains(lastReservedIP) {
+				iter.rangeIdx = i
+				iter.startRange = i
+
+				// We advance the cursor on every Next(), so the first call
+				// to next() will return lastReservedIP + 1
+				iter.cur = lastReservedIP
+				break
+			}
 		}
 	} else {
-		i.start = a.ipRange.RangeStart
+		iter.rangeIdx = 0
+		iter.startRange = 0
+		iter.startIP = (*a.rangeset)[0].RangeStart
 	}
-	return &i, nil
+	return &iter, nil
 }
 
-// Next returns the next IP in the iterator, or nil if end is reached
-func (i *RangeIter) Next() net.IP {
-	// If we're at the beginning, time to start
+// Next returns the next IP, its mask, and its gateway. Returns nil
+// if the iterator has been exhausted
+func (i *RangeIter) Next() (*net.IPNet, net.IP) {
+	r := (*i.rangeset)[i.rangeIdx]
+
+	// If this is the first time iterating and we're not starting in the middle
+	// of the range, then start at rangeStart, which is inclusive
 	if i.cur == nil {
-		i.cur = i.start
-		return i.cur
+		i.cur = r.RangeStart
+		i.startIP = i.cur
+		if i.cur.Equal(r.Gateway) {
+			return i.Next()
+		}
+		return &net.IPNet{IP: i.cur, Mask: r.Subnet.Mask}, r.Gateway
 	}
-	//  we returned .high last time, since we're inclusive
-	if i.cur.Equal(i.high) {
-		i.cur = i.low
+
+	// If we've reached the end of this range, we need to advance the range
+	// RangeEnd is inclusive as well
+	if i.cur.Equal(r.RangeEnd) {
+		i.rangeIdx += 1
+		i.rangeIdx %= len(*i.rangeset)
+		r = (*i.rangeset)[i.rangeIdx]
+
+		i.cur = r.RangeStart
 	} else {
 		i.cur = ip.NextIP(i.cur)
 	}
 
-	// If we've looped back to where we started, exit
-	if i.cur.Equal(i.start) {
-		return nil
+	if i.startIP == nil {
+		i.startIP = i.cur
+	} else if i.rangeIdx == i.startRange && i.cur.Equal(i.startIP) {
+		// IF we've looped back to where we started, give up
+		return nil, nil
+	}
+
+	if i.cur.Equal(r.Gateway) {
+		return i.Next()
 	}
 
-	return i.cur
+	return &net.IPNet{IP: i.cur, Mask: r.Subnet.Mask}, r.Gateway
 }