Merge pull request #2285 from crazy-max/update-hashring

vendor: github.com/serialx/hashring 22c0c7ab6b1b (master)

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.0-rc5
 	github.com/pelletier/go-toml v1.9.5
 	github.com/pkg/errors v0.9.1
-	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
+	github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -408,8 +408,8 @@ github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/
 github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 h1:ka9QPuQg2u4LGipiZGsgkg3rJCo4iIUCy75FddM0GRQ=
-github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
+github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU=
+github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
 github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
 github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=

vendor/github.com/serialx/hashring/.gitignore

@@ -0,0 +1,3 @@
+coverage
+*.html
+

vendor/github.com/serialx/hashring/hash.go

@@ -0,0 +1,73 @@
+package hashring
+
+import (
+	"fmt"
+	"hash"
+)
+
+// HashSum allows to use a builder pattern to create different HashFunc objects.
+// See examples for details.
+type HashSum struct {
+	functions []func([]byte) []byte
+}
+
+func (r *HashSum) Use(
+	hashKeyFunc func(bytes []byte) (HashKey, error),
+) (HashFunc, error) {
+
+	// build final hash function
+	composed := func(bytes []byte) []byte {
+		for _, f := range r.functions {
+			bytes = f(bytes)
+		}
+		return bytes
+	}
+
+	// check function composition for errors
+	testResult := composed([]byte("test"))
+	_, err := hashKeyFunc(testResult)
+	if err != nil {
+		const msg = "can't use given hash.Hash with given hashKeyFunc"
+		return nil, fmt.Errorf("%s: %w", msg, err)
+	}
+
+	// build HashFunc
+	return func(key []byte) HashKey {
+		bytes := composed(key)
+		hashKey, err := hashKeyFunc(bytes)
+		if err != nil {
+			// panic because we already checked HashSum earlier
+			panic(fmt.Sprintf("hashKeyFunc failure: %v", err))
+		}
+		return hashKey
+	}, nil
+}
+
+// NewHash creates a new *HashSum object which can be used to create HashFunc.
+// HashFunc object is thread safe if the hasher argument produces a new hash.Hash 
+// each time. The produced hash.Hash is allowed to be non thread-safe.
+func NewHash(hasher func() hash.Hash) *HashSum {
+	return &HashSum{
+		functions: []func(key []byte) []byte{
+			func(key []byte) []byte {
+				hash := hasher()
+				hash.Write(key)
+				return hash.Sum(nil)
+			},
+		},
+	}
+}
+
+func (r *HashSum) FirstBytes(n int) *HashSum {
+	r.functions = append(r.functions, func(bytes []byte) []byte {
+		return bytes[:n]
+	})
+	return r
+}
+
+func (r *HashSum) LastBytes(n int) *HashSum {
+	r.functions = append(r.functions, func(bytes []byte) []byte {
+		return bytes[len(bytes)-n:]
+	})
+	return r
+}

vendor/github.com/serialx/hashring/hashring.go

@@ -2,39 +2,75 @@ package hashring
 
 import (
 	"crypto/md5"
-	"math"
+	"fmt"
 	"sort"
 	"strconv"
 )
 
-type HashKey uint32
+var defaultHashFunc = func() HashFunc {
+	hashFunc, err := NewHash(md5.New).Use(NewInt64PairHashKey)
+	if err != nil {
+		panic(fmt.Sprintf("failed to create defaultHashFunc: %s", err.Error()))
+	}
+	return hashFunc
+}()
+
+type HashKey interface {
+	Less(other HashKey) bool
+}
 type HashKeyOrder []HashKey
 
-func (h HashKeyOrder) Len() int           { return len(h) }
-func (h HashKeyOrder) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
-func (h HashKeyOrder) Less(i, j int) bool { return h[i] < h[j] }
+func (h HashKeyOrder) Len() int      { return len(h) }
+func (h HashKeyOrder) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
+func (h HashKeyOrder) Less(i, j int) bool {
+	return h[i].Less(h[j])
+}
+
+type HashFunc func([]byte) HashKey
 
 type HashRing struct {
 	ring       map[HashKey]string
 	sortedKeys []HashKey
 	nodes      []string
 	weights    map[string]int
+	hashFunc   HashFunc
+}
+
+type Uint32HashKey uint32
+
+func (k Uint32HashKey) Less(other HashKey) bool {
+	return k < other.(Uint32HashKey)
 }
 
 func New(nodes []string) *HashRing {
+	return NewWithHash(nodes, defaultHashFunc)
+}
+
+func NewWithHash(
+	nodes []string,
+	hashKey HashFunc,
+) *HashRing {
 	hashRing := &HashRing{
 		ring:       make(map[HashKey]string),
 		sortedKeys: make([]HashKey, 0),
 		nodes:      nodes,
 		weights:    make(map[string]int),
+		hashFunc:   hashKey,
 	}
 	hashRing.generateCircle()
 	return hashRing
 }
 
 func NewWithWeights(weights map[string]int) *HashRing {
+	return NewWithHashAndWeights(weights, defaultHashFunc)
+}
+
+func NewWithHashAndWeights(
+	weights map[string]int,
+	hashFunc HashFunc,
+) *HashRing {
 	nodes := make([]string, 0, len(weights))
-	for node, _ := range weights {
+	for node := range weights {
 		nodes = append(nodes, node)
 	}
 	hashRing := &HashRing{
@@ -42,6 +78,7 @@ func NewWithWeights(weights map[string]int) *HashRing {
 		sortedKeys: make([]HashKey, 0),
 		nodes:      nodes,
 		weights:    weights,
+		hashFunc:   hashFunc,
 	}
 	hashRing.generateCircle()
 	return hashRing
@@ -66,7 +103,7 @@ func (h *HashRing) UpdateWithWeights(weights map[string]int) {
 	}
 
 	if nodesChgFlg {
-		newhring := NewWithWeights(weights)
+		newhring := NewWithHashAndWeights(weights, h.hashFunc)
 		h.weights = newhring.weights
 		h.nodes = newhring.nodes
 		h.ring = newhring.ring
@@ -88,17 +125,11 @@ func (h *HashRing) generateCircle() {
 	for _, node := range h.nodes {
 		weight := h.weights[node]
 
-		factor := math.Floor(float64(40*len(h.nodes)*weight) / float64(totalWeight))
-
-		for j := 0; j < int(factor); j++ {
+		for j := 0; j < weight; j++ {
 			nodeKey := node + "-" + strconv.FormatInt(int64(j), 10)
-			bKey := hashDigest(nodeKey)
-
-			for i := 0; i < 3; i++ {
-				key := hashVal(bKey[i*4 : i*4+4])
-				h.ring[key] = node
-				h.sortedKeys = append(h.sortedKeys, key)
-			}
+			key := h.hashFunc([]byte(nodeKey))
+			h.ring[key] = node
+			h.sortedKeys = append(h.sortedKeys, key)
 		}
 	}
 
@@ -121,10 +152,10 @@ func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) {
 	key := h.GenKey(stringKey)
 
 	nodes := h.sortedKeys
-	pos = sort.Search(len(nodes), func(i int) bool { return nodes[i] > key })
+	pos = sort.Search(len(nodes), func(i int) bool { return key.Less(nodes[i]) })
 
 	if pos == len(nodes) {
-		// Wrap the search, should return first node
+		// Wrap the search, should return First node
 		return 0, true
 	} else {
 		return pos, true
@@ -132,10 +163,12 @@ func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) {
 }
 
 func (h *HashRing) GenKey(key string) HashKey {
-	bKey := hashDigest(key)
-	return hashVal(bKey[0:4])
+	return h.hashFunc([]byte(key))
 }
 
+// GetNodes iterates over the hash ring and returns the nodes in the order
+// which is determined by the key. GetNodes is thread safe if the hash
+// which was used to configure the hash ring is thread safe.
 func (h *HashRing) GetNodes(stringKey string, size int) (nodes []string, ok bool) {
 	pos, ok := h.GetNodePos(stringKey)
 	if !ok {
@@ -193,6 +226,7 @@ func (h *HashRing) AddWeightedNode(node string, weight int) *HashRing {
 		sortedKeys: make([]HashKey, 0),
 		nodes:      nodes,
 		weights:    weights,
+		hashFunc:   h.hashFunc,
 	}
 	hashRing.generateCircle()
 	return hashRing
@@ -208,7 +242,7 @@ func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing {
 		return h
 	}
 
-	nodes := make([]string, len(h.nodes), len(h.nodes))
+	nodes := make([]string, len(h.nodes))
 	copy(nodes, h.nodes)
 
 	weights := make(map[string]int)
@@ -222,6 +256,7 @@ func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing {
 		sortedKeys: make([]HashKey, 0),
 		nodes:      nodes,
 		weights:    weights,
+		hashFunc:   h.hashFunc,
 	}
 	hashRing.generateCircle()
 	return hashRing
@@ -251,18 +286,8 @@ func (h *HashRing) RemoveNode(node string) *HashRing {
 		sortedKeys: make([]HashKey, 0),
 		nodes:      nodes,
 		weights:    weights,
+		hashFunc:   h.hashFunc,
 	}
 	hashRing.generateCircle()
 	return hashRing
 }
-
-func hashVal(bKey []byte) HashKey {
-	return ((HashKey(bKey[3]) << 24) |
-		(HashKey(bKey[2]) << 16) |
-		(HashKey(bKey[1]) << 8) |
-		(HashKey(bKey[0])))
-}
-
-func hashDigest(key string) [md5.Size]byte {
-	return md5.Sum([]byte(key))
-}

vendor/github.com/serialx/hashring/key.go

@@ -0,0 +1,33 @@
+package hashring
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+type Int64PairHashKey struct {
+	High int64
+	Low  int64
+}
+
+func (k *Int64PairHashKey) Less(other HashKey) bool {
+	o := other.(*Int64PairHashKey)
+	if k.High < o.High {
+		return true
+	}
+	return k.High == o.High && k.Low < o.Low
+}
+
+func NewInt64PairHashKey(bytes []byte) (HashKey, error) {
+	const expected = 16
+	if len(bytes) != expected {
+		return nil, fmt.Errorf(
+			"expected %d bytes, got %d bytes",
+			expected, len(bytes),
+		)
+	}
+	return &Int64PairHashKey{
+		High: int64(binary.LittleEndian.Uint64(bytes[:8])),
+		Low:  int64(binary.LittleEndian.Uint64(bytes[8:])),
+	}, nil
+}

vendor/modules.txt

@@ -672,7 +672,7 @@ github.com/rivo/uniseg
 ## explicit; go 1.17
 github.com/secure-systems-lab/go-securesystemslib/cjson
 github.com/secure-systems-lab/go-securesystemslib/dsse
-# github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
+# github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b
 ## explicit
 github.com/serialx/hashring
 # github.com/shibumi/go-pathspec v1.3.0