diff --git a/go.mod b/go.mod index b71467ce..16fefc8a 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc5 github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 - github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 + github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 39d17a8a..108aeb87 100644 --- a/go.sum +++ b/go.sum @@ -415,8 +415,8 @@ github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/ github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 h1:ka9QPuQg2u4LGipiZGsgkg3rJCo4iIUCy75FddM0GRQ= -github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= +github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= +github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= diff --git a/vendor/github.com/serialx/hashring/.gitignore b/vendor/github.com/serialx/hashring/.gitignore new file mode 100644 index 00000000..ded1de6c --- /dev/null +++ b/vendor/github.com/serialx/hashring/.gitignore @@ -0,0 +1,3 @@ +coverage +*.html + diff --git a/vendor/github.com/serialx/hashring/hash.go b/vendor/github.com/serialx/hashring/hash.go new file mode 100644 index 00000000..ff4c4d04 --- /dev/null +++ b/vendor/github.com/serialx/hashring/hash.go @@ -0,0 +1,73 @@ +package hashring + +import ( + "fmt" + "hash" +) + +// HashSum allows to use a builder pattern to create different HashFunc objects. +// See examples for details. +type HashSum struct { + functions []func([]byte) []byte +} + +func (r *HashSum) Use( + hashKeyFunc func(bytes []byte) (HashKey, error), +) (HashFunc, error) { + + // build final hash function + composed := func(bytes []byte) []byte { + for _, f := range r.functions { + bytes = f(bytes) + } + return bytes + } + + // check function composition for errors + testResult := composed([]byte("test")) + _, err := hashKeyFunc(testResult) + if err != nil { + const msg = "can't use given hash.Hash with given hashKeyFunc" + return nil, fmt.Errorf("%s: %w", msg, err) + } + + // build HashFunc + return func(key []byte) HashKey { + bytes := composed(key) + hashKey, err := hashKeyFunc(bytes) + if err != nil { + // panic because we already checked HashSum earlier + panic(fmt.Sprintf("hashKeyFunc failure: %v", err)) + } + return hashKey + }, nil +} + +// NewHash creates a new *HashSum object which can be used to create HashFunc. +// HashFunc object is thread safe if the hasher argument produces a new hash.Hash +// each time. The produced hash.Hash is allowed to be non thread-safe. +func NewHash(hasher func() hash.Hash) *HashSum { + return &HashSum{ + functions: []func(key []byte) []byte{ + func(key []byte) []byte { + hash := hasher() + hash.Write(key) + return hash.Sum(nil) + }, + }, + } +} + +func (r *HashSum) FirstBytes(n int) *HashSum { + r.functions = append(r.functions, func(bytes []byte) []byte { + return bytes[:n] + }) + return r +} + +func (r *HashSum) LastBytes(n int) *HashSum { + r.functions = append(r.functions, func(bytes []byte) []byte { + return bytes[len(bytes)-n:] + }) + return r +} diff --git a/vendor/github.com/serialx/hashring/hashring.go b/vendor/github.com/serialx/hashring/hashring.go index 3fdeb2da..ad2ac018 100644 --- a/vendor/github.com/serialx/hashring/hashring.go +++ b/vendor/github.com/serialx/hashring/hashring.go @@ -2,39 +2,75 @@ package hashring import ( "crypto/md5" - "math" + "fmt" "sort" "strconv" ) -type HashKey uint32 +var defaultHashFunc = func() HashFunc { + hashFunc, err := NewHash(md5.New).Use(NewInt64PairHashKey) + if err != nil { + panic(fmt.Sprintf("failed to create defaultHashFunc: %s", err.Error())) + } + return hashFunc +}() + +type HashKey interface { + Less(other HashKey) bool +} type HashKeyOrder []HashKey -func (h HashKeyOrder) Len() int { return len(h) } -func (h HashKeyOrder) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h HashKeyOrder) Less(i, j int) bool { return h[i] < h[j] } +func (h HashKeyOrder) Len() int { return len(h) } +func (h HashKeyOrder) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h HashKeyOrder) Less(i, j int) bool { + return h[i].Less(h[j]) +} + +type HashFunc func([]byte) HashKey type HashRing struct { ring map[HashKey]string sortedKeys []HashKey nodes []string weights map[string]int + hashFunc HashFunc +} + +type Uint32HashKey uint32 + +func (k Uint32HashKey) Less(other HashKey) bool { + return k < other.(Uint32HashKey) } func New(nodes []string) *HashRing { + return NewWithHash(nodes, defaultHashFunc) +} + +func NewWithHash( + nodes []string, + hashKey HashFunc, +) *HashRing { hashRing := &HashRing{ ring: make(map[HashKey]string), sortedKeys: make([]HashKey, 0), nodes: nodes, weights: make(map[string]int), + hashFunc: hashKey, } hashRing.generateCircle() return hashRing } func NewWithWeights(weights map[string]int) *HashRing { + return NewWithHashAndWeights(weights, defaultHashFunc) +} + +func NewWithHashAndWeights( + weights map[string]int, + hashFunc HashFunc, +) *HashRing { nodes := make([]string, 0, len(weights)) - for node, _ := range weights { + for node := range weights { nodes = append(nodes, node) } hashRing := &HashRing{ @@ -42,6 +78,7 @@ func NewWithWeights(weights map[string]int) *HashRing { sortedKeys: make([]HashKey, 0), nodes: nodes, weights: weights, + hashFunc: hashFunc, } hashRing.generateCircle() return hashRing @@ -66,7 +103,7 @@ func (h *HashRing) UpdateWithWeights(weights map[string]int) { } if nodesChgFlg { - newhring := NewWithWeights(weights) + newhring := NewWithHashAndWeights(weights, h.hashFunc) h.weights = newhring.weights h.nodes = newhring.nodes h.ring = newhring.ring @@ -88,17 +125,11 @@ func (h *HashRing) generateCircle() { for _, node := range h.nodes { weight := h.weights[node] - factor := math.Floor(float64(40*len(h.nodes)*weight) / float64(totalWeight)) - - for j := 0; j < int(factor); j++ { + for j := 0; j < weight; j++ { nodeKey := node + "-" + strconv.FormatInt(int64(j), 10) - bKey := hashDigest(nodeKey) - - for i := 0; i < 3; i++ { - key := hashVal(bKey[i*4 : i*4+4]) - h.ring[key] = node - h.sortedKeys = append(h.sortedKeys, key) - } + key := h.hashFunc([]byte(nodeKey)) + h.ring[key] = node + h.sortedKeys = append(h.sortedKeys, key) } } @@ -121,10 +152,10 @@ func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) { key := h.GenKey(stringKey) nodes := h.sortedKeys - pos = sort.Search(len(nodes), func(i int) bool { return nodes[i] > key }) + pos = sort.Search(len(nodes), func(i int) bool { return key.Less(nodes[i]) }) if pos == len(nodes) { - // Wrap the search, should return first node + // Wrap the search, should return First node return 0, true } else { return pos, true @@ -132,10 +163,12 @@ func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) { } func (h *HashRing) GenKey(key string) HashKey { - bKey := hashDigest(key) - return hashVal(bKey[0:4]) + return h.hashFunc([]byte(key)) } +// GetNodes iterates over the hash ring and returns the nodes in the order +// which is determined by the key. GetNodes is thread safe if the hash +// which was used to configure the hash ring is thread safe. func (h *HashRing) GetNodes(stringKey string, size int) (nodes []string, ok bool) { pos, ok := h.GetNodePos(stringKey) if !ok { @@ -193,6 +226,7 @@ func (h *HashRing) AddWeightedNode(node string, weight int) *HashRing { sortedKeys: make([]HashKey, 0), nodes: nodes, weights: weights, + hashFunc: h.hashFunc, } hashRing.generateCircle() return hashRing @@ -208,7 +242,7 @@ func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing { return h } - nodes := make([]string, len(h.nodes), len(h.nodes)) + nodes := make([]string, len(h.nodes)) copy(nodes, h.nodes) weights := make(map[string]int) @@ -222,6 +256,7 @@ func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing { sortedKeys: make([]HashKey, 0), nodes: nodes, weights: weights, + hashFunc: h.hashFunc, } hashRing.generateCircle() return hashRing @@ -251,18 +286,8 @@ func (h *HashRing) RemoveNode(node string) *HashRing { sortedKeys: make([]HashKey, 0), nodes: nodes, weights: weights, + hashFunc: h.hashFunc, } hashRing.generateCircle() return hashRing } - -func hashVal(bKey []byte) HashKey { - return ((HashKey(bKey[3]) << 24) | - (HashKey(bKey[2]) << 16) | - (HashKey(bKey[1]) << 8) | - (HashKey(bKey[0]))) -} - -func hashDigest(key string) [md5.Size]byte { - return md5.Sum([]byte(key)) -} diff --git a/vendor/github.com/serialx/hashring/key.go b/vendor/github.com/serialx/hashring/key.go new file mode 100644 index 00000000..08c2cc38 --- /dev/null +++ b/vendor/github.com/serialx/hashring/key.go @@ -0,0 +1,33 @@ +package hashring + +import ( + "encoding/binary" + "fmt" +) + +type Int64PairHashKey struct { + High int64 + Low int64 +} + +func (k *Int64PairHashKey) Less(other HashKey) bool { + o := other.(*Int64PairHashKey) + if k.High < o.High { + return true + } + return k.High == o.High && k.Low < o.Low +} + +func NewInt64PairHashKey(bytes []byte) (HashKey, error) { + const expected = 16 + if len(bytes) != expected { + return nil, fmt.Errorf( + "expected %d bytes, got %d bytes", + expected, len(bytes), + ) + } + return &Int64PairHashKey{ + High: int64(binary.LittleEndian.Uint64(bytes[:8])), + Low: int64(binary.LittleEndian.Uint64(bytes[8:])), + }, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c6aad703..ab4e53df 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -665,7 +665,7 @@ github.com/rivo/uniseg ## explicit; go 1.17 github.com/secure-systems-lab/go-securesystemslib/cjson github.com/secure-systems-lab/go-securesystemslib/dsse -# github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 +# github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b ## explicit github.com/serialx/hashring # github.com/shibumi/go-pathspec v1.3.0