From afcffa239eb8dc9f33f40f0f2766b1c4caded9a4 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 17 Oct 2023 19:34:02 -0300 Subject: [PATCH] add historic storage for keeping track of flagsets --- go.mod | 9 +- go.sum | 14 +- splitio/commitversion.go | 2 +- splitio/proxy/storage/optimized/historic.go | 192 +++++++++++ .../proxy/storage/optimized/historic_test.go | 305 ++++++++++++++++++ 5 files changed, 513 insertions(+), 9 deletions(-) create mode 100644 splitio/proxy/storage/optimized/historic.go create mode 100644 splitio/proxy/storage/optimized/historic_test.go diff --git a/go.mod b/go.mod index 908d8a86..3485f219 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/splitio/split-synchronizer/v5 -go 1.18 +go 1.21 require ( github.com/gin-contrib/cors v1.4.0 @@ -8,8 +8,9 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/google/uuid v1.3.0 github.com/splitio/gincache v1.0.1 - github.com/splitio/go-split-commons/v5 v5.0.0 - github.com/splitio/go-toolkit/v5 v5.3.1 + github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0 + github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a + github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.6 ) @@ -19,6 +20,7 @@ require ( github.com/bytedance/sonic v1.9.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -34,6 +36,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/redis/go-redis/v9 v9.0.4 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect diff --git a/go.sum b/go.sum index bbe5b69d..ff9301c7 100644 --- a/go.sum +++ b/go.sum @@ -3,7 +3,9 @@ github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -31,6 +33,7 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -87,10 +90,10 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/splitio/gincache v1.0.1 h1:dLYdANY/BqH4KcUMCe/LluLyV5WtuE/LEdQWRE06IXU= github.com/splitio/gincache v1.0.1/go.mod h1:CcgJDSM9Af75kyBH0724v55URVwMBuSj5x1eCWIOECY= -github.com/splitio/go-split-commons/v5 v5.0.0 h1:bGRi0cf1JP5VNSi0a4BPQEWv/DACkeSKliazhPMVDPk= -github.com/splitio/go-split-commons/v5 v5.0.0/go.mod h1:lzoVmYJaCqB8UPSxWva0BZe7fF+bRJD+eP0rNi/lL7c= -github.com/splitio/go-toolkit/v5 v5.3.1 h1:9J/byd0fRxWj5/Zg0QZOnUxKBDIAMCGr7rySYzJKdJg= -github.com/splitio/go-toolkit/v5 v5.3.1/go.mod h1:xYhUvV1gga9/1029Wbp5pjnR6Cy8nvBpjw99wAbsMko= +github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0 h1:t7QuH0+4T2LeJOc2gdRP+PkFPkQEB017arfxBccsArg= +github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0/go.mod h1:ksVZQYLs+3ZuzU81vEvf1aCjk24pdrVWjUXNq6Qcayo= +github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a h1:2wjh5hSGlFRuh6Lbmodr0VRqtry2m9pEBNmwiLsY+ss= +github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a/go.mod h1:xYhUvV1gga9/1029Wbp5pjnR6Cy8nvBpjw99wAbsMko= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -101,8 +104,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= diff --git a/splitio/commitversion.go b/splitio/commitversion.go index 47b58123..4ba6024b 100644 --- a/splitio/commitversion.go +++ b/splitio/commitversion.go @@ -5,4 +5,4 @@ This file is created automatically, please do not edit */ // CommitVersion is the version of the last commit previous to release -const CommitVersion = "da63b9f" +const CommitVersion = "fa204db" diff --git a/splitio/proxy/storage/optimized/historic.go b/splitio/proxy/storage/optimized/historic.go new file mode 100644 index 00000000..3805e8d9 --- /dev/null +++ b/splitio/proxy/storage/optimized/historic.go @@ -0,0 +1,192 @@ +package optimized + +import ( + "slices" + "sort" + "strings" + "sync" + + "github.com/splitio/go-split-commons/v5/dtos" +) + +type HistoricChanges struct { + data []FeatureView + mutex sync.RWMutex +} + +func (h *HistoricChanges) GetUpdatedSince(since int64, flagSets []string) []FeatureView { + h.mutex.RLock() + views := h.findNewerThan(since) + toRet := copyAndFilter(views, flagSets, since) + h.mutex.RUnlock() + return toRet +} + +func (h *HistoricChanges) Update(toAdd []dtos.SplitDTO, toRemove []dtos.SplitDTO, newCN int64) { + h.mutex.Lock() + h.updateFrom(toAdd) + h.updateFrom(toRemove) + sort.Slice(h.data, func(i, j int) bool { return h.data[i].LastUpdated < h.data[j].LastUpdated }) + h.mutex.Unlock() +} + +func (h *HistoricChanges) updateFrom(source []dtos.SplitDTO) { + for idx := range source { + if current := h.findByName(source[idx].Name); current != nil { + current.updateFrom(&source[idx]) + } else { + var toAdd FeatureView + toAdd.updateFrom(&source[idx]) + h.data = append(h.data, toAdd) + } + } + +} + +func (h *HistoricChanges) findByName(name string) *FeatureView { + for idx := range h.data { + if h.data[idx].Name == name { // TODO(mredolatti): optimize! + return &h.data[idx] + } + } + return nil +} + +func (h *HistoricChanges) findNewerThan(since int64) []FeatureView { + // precondition: h.data is sorted by CN + start := sort.Search(len(h.data), func(i int) bool { return h.data[i].LastUpdated > since }) + if start == len(h.data) { + return nil + } + return h.data[start:] +} + +type FeatureView struct { + Name string + Active bool + LastUpdated int64 + TrafficTypeName string + FlagSets []FlagSetView +} + +func (f *FeatureView) updateFrom(s *dtos.SplitDTO) { + f.Name = s.Name + f.Active = s.Status == "ACTIVE" + f.LastUpdated = s.ChangeNumber + f.TrafficTypeName = s.TrafficTypeName + f.updateFlagsets(s.Sets, s.ChangeNumber) +} + +func (f *FeatureView) updateFlagsets(incoming []string, lastUpdated int64) { + // TODO(mredolatti): need a copy of incoming? + for idx := range f.FlagSets { + if itemIdx := slices.Index(incoming, f.FlagSets[idx].Name); itemIdx != -1 { + if !f.FlagSets[idx].Active { // Association changed from ARCHIVED to ACTIVE + f.FlagSets[idx].Active = true + f.FlagSets[idx].LastUpdated = lastUpdated + + } + + // "soft delete" the item so that it's not traversed later on + // (replaces the item with the last one, clears the latter and shrinks the slice by 1) + incoming[itemIdx] = incoming[len(incoming)-1] + incoming[len(incoming)-1] = "" + incoming = incoming[:len(incoming)-1] + + } else { // Association changed from ARCHIVED to ACTIVE + f.FlagSets[idx].Active = false + f.FlagSets[idx].LastUpdated = lastUpdated + } + } + + for idx := range incoming { + // the only leftover in `incoming` should be the items that were not + // present in the feature's previously associated flagsets, so they're new & active + f.FlagSets = append(f.FlagSets, FlagSetView{ + Name: incoming[idx], + Active: true, + LastUpdated: lastUpdated, + }) + } + + sort.Slice(f.FlagSets, func(i, j int) bool { return f.FlagSets[i].Name < f.FlagSets[j].Name }) +} + +func (f *FeatureView) findFlagSetByName(name string) *FlagSetView { + // precondition: f.FlagSets is sorted by name + idx := sort.Search(len(f.FlagSets), func(i int) bool { return f.FlagSets[i].Name >= name }) + if idx != len(f.FlagSets) && name == f.FlagSets[idx].Name { + return &f.FlagSets[idx] + } + return nil +} + +func (f *FeatureView) clone() FeatureView { + toRet := FeatureView{ + Name: f.Name, + Active: f.Active, + LastUpdated: f.LastUpdated, + TrafficTypeName: f.TrafficTypeName, + FlagSets: make([]FlagSetView, len(f.FlagSets)), + } + copy(toRet.FlagSets, f.FlagSets) // we need to deep clone to avoid race conditions + return toRet + +} + +func copyAndFilter(views []FeatureView, sets []string, since int64) []FeatureView { + // precondition: f.Flagsets is sorted by name + // precondition: sets is sorted + toRet := make([]FeatureView, 0, len(views)) + if len(sets) == 0 { + for idx := range views { + toRet = append(toRet, views[idx].clone()) + } + return toRet + } + + // this code computes the intersection in o(views * (len(views.sets) + len(sets))) + for idx := range views { + viewFlagSetIndex, requestedSetIndex := 0, 0 + for viewFlagSetIndex < len(views[idx].FlagSets) { + switch strings.Compare(views[idx].FlagSets[viewFlagSetIndex].Name, sets[requestedSetIndex]) { + case 0: // we got a match + fsinfo := views[idx].FlagSets[viewFlagSetIndex] + // if an association is active, it's considered and the Feature is added to the result set. + // if an association is inactive and we're fetching from scratch (since=-1), it's not considered. + // if an association was already inactive at the time of the provided `since`, it's not considered. + // if an association was active on the provided `since` and now isn't, the feature IS added to the returned payload. + // - the consumer is responsible for filtering flagsets where active = false when mapping the outcome of + // this function to a []dtos.SplitChanges response. + if fsinfo.Active || (since > -1 && fsinfo.LastUpdated > since) { + toRet = append(toRet, views[idx].clone()) + } + viewFlagSetIndex++ + incrUpTo(&requestedSetIndex, len(sets)) + case -1: + viewFlagSetIndex++ + case 1: + if incrUpTo(&requestedSetIndex, len(sets)) { + viewFlagSetIndex++ + } + } + } + } + return toRet +} + +type FlagSetView struct { + Name string + Active bool + LastUpdated int64 +} + +// increment `toIncr` by 1 as long as the result is less than `limit`. +// return wether the limit was reached +func incrUpTo(toIncr *int, limit int) bool { + if *toIncr+1 >= limit { + return true + } + *toIncr++ + return false +} diff --git a/splitio/proxy/storage/optimized/historic_test.go b/splitio/proxy/storage/optimized/historic_test.go new file mode 100644 index 00000000..797e1881 --- /dev/null +++ b/splitio/proxy/storage/optimized/historic_test.go @@ -0,0 +1,305 @@ +package optimized + +import ( + "math/rand" + "sort" + "testing" + "time" + + "github.com/splitio/go-split-commons/v5/dtos" + "github.com/stretchr/testify/assert" +) + +func TestHistoricSplitStorage(t *testing.T) { + + var historic HistoricChanges + historic.Update([]dtos.SplitDTO{ + {Name: "f1", Sets: []string{"s1", "s2"}, Status: "ACTIVE", ChangeNumber: 1, TrafficTypeName: "tt1"}, + }, []dtos.SplitDTO{}, 1) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 1}, + }, + historic.GetUpdatedSince(-1, nil)) + + // process an update with no change in flagsets / split status + // - fetching from -1 && 1 should return the same paylaod as before with only `lastUpdated` bumped to 2 + // - fetching from 2 should return empty + historic.Update([]dtos.SplitDTO{ + {Name: "f1", Sets: []string{"s1", "s2"}, Status: "ACTIVE", ChangeNumber: 2, TrafficTypeName: "tt1"}, + }, []dtos.SplitDTO{}, 1) + + // no filter + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(-1, nil)) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(1, nil)) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(2, nil)) + + // filter by s1 + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(-1, []string{"s1"})) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(1, []string{"s1"})) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(2, []string{"s1"})) + + // filter by s2 + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(-1, []string{"s2"})) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(1, []string{"s2"})) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(2, []string{"s2"})) + + // ------------------- + + // process an update with one extra split + // - fetching from -1, & 1 should return the same payload + // - fetching from 2 shuold only return f2 + // - fetching from 3 should return empty + historic.Update([]dtos.SplitDTO{ + {Name: "f2", Sets: []string{"s2", "s3"}, Status: "ACTIVE", ChangeNumber: 3, TrafficTypeName: "tt1"}, + }, []dtos.SplitDTO{}, 1) + + // assert correct behaviours for CN == 1..3 and no flag sets filter + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(-1, nil)) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(1, nil)) + assert.Equal(t, + []FeatureView{ + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(2, nil)) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(3, nil)) + + // filtering by s1: + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(-1, []string{"s1"})) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + }, + historic.GetUpdatedSince(1, []string{"s1"})) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(2, []string{"s1"})) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(3, []string{"s1"})) + + // filtering by s2: + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(-1, []string{"s2"})) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", true, 1}, {"s2", true, 1}}, Active: true, LastUpdated: 2}, + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(1, []string{"s2"})) + assert.Equal(t, + []FeatureView{ + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(2, []string{"s2"})) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(3, []string{"s2"})) + + //filtering by s3 + assert.Equal(t, + []FeatureView{ + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(-1, []string{"s3"})) + assert.Equal(t, + []FeatureView{ + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(1, []string{"s3"})) + assert.Equal(t, + []FeatureView{ + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + }, + historic.GetUpdatedSince(2, []string{"s3"})) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(3, []string{"s3"})) + + // ------------------- + + // process an update that removes f1 from flagset s1 + // - fetching without a filter should remain the same + // - fetching with filter = s1 should not return f1 in CN=-1, should return it without the flagset in greater CNs + historic.Update([]dtos.SplitDTO{ + {Name: "f1", Sets: []string{"s2"}, Status: "ACTIVE", ChangeNumber: 4, TrafficTypeName: "tt1"}, + }, []dtos.SplitDTO{}, 1) + + assert.Equal(t, + []FeatureView{ + {Name: "f2", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s2", true, 3}, {"s3", true, 3}}, Active: true, LastUpdated: 3}, + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", false, 4}, {"s2", true, 1}}, Active: true, LastUpdated: 4}, + }, + historic.GetUpdatedSince(-1, nil)) + + // with filter = s1 (f2 never was associated with s1, f1 is no longer associated) + assert.Equal(t, + []FeatureView{}, + historic.GetUpdatedSince(-1, []string{"s1"})) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", false, 4}, {"s2", true, 1}}, Active: true, LastUpdated: 4}, + }, + historic.GetUpdatedSince(1, []string{"s1"})) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", false, 4}, {"s2", true, 1}}, Active: true, LastUpdated: 4}, + }, + historic.GetUpdatedSince(2, []string{"s1"})) + assert.Equal(t, + []FeatureView{ + {Name: "f1", TrafficTypeName: "tt1", FlagSets: []FlagSetView{{"s1", false, 4}, {"s2", true, 1}}, Active: true, LastUpdated: 4}, + }, + historic.GetUpdatedSince(3, []string{"s1"})) + assert.Equal(t, []FeatureView{}, historic.GetUpdatedSince(4, []string{"s1"})) + +} + +// -- code below is for benchmarking random access using hashsets (map[string]struct{}) vs sorted slices + binary search + +func setupRandomData(flagsetLength int, flagsetCount int, splits int, flagSetsPerSplitMax int, userSets int) benchmarkDataSlices { + const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + rand.Seed(time.Now().UnixNano()) + makeStr := func(n int) string { + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) + } + + flagSets := make([]string, 0, flagsetCount) + for flagsetCount > 0 { + flagSets = append(flagSets, makeStr(flagsetLength)) + flagsetCount-- + } + + views := make([]FeatureView, 0, splits) + for len(views) < cap(views) { + fscount := rand.Intn(flagSetsPerSplitMax) + setsForSplit := make([]FlagSetView, 0, fscount) + for fscount > 0 { + setsForSplit = append(setsForSplit, FlagSetView{ + Name: flagSets[rand.Intn(len(flagSets))], + Active: rand.Intn(2) > 0, + LastUpdated: rand.Int63n(2), + }) + fscount-- + } + sort.Slice(setsForSplit, func(i, j int) bool { return setsForSplit[i].Name < setsForSplit[j].Name }) + views = append(views, FeatureView{ + Name: makeStr(20), + Active: rand.Intn(2) > 0, // rand bool + LastUpdated: rand.Int63n(2), // 1 or 2 (still an int but behaving like a bool if we filter by since=1) + TrafficTypeName: makeStr(10), + FlagSets: setsForSplit, + }) + + } + sort.Slice(views, func(i, j int) bool { return views[i].LastUpdated < views[j].LastUpdated }) + return benchmarkDataSlices{views, flagSets} +} + +type benchmarkDataSlices struct { + views []FeatureView + sets []string +} + +func (b *benchmarkDataSlices) toBenchmarkDataForMaps() benchmarkDataMaps { + setMap := make(map[string]struct{}, len(b.sets)) + for _, s := range b.sets { + setMap[s] = struct{}{} + } + + return benchmarkDataMaps{ + views: b.views, + sets: setMap, + } + +} + +type benchmarkDataMaps struct { + views []FeatureView + sets map[string]struct{} +} + +// reference implementation for benchmarking purposes only +func copyAndFilterUsingMaps(views []FeatureView, sets map[string]struct{}, since int64) []FeatureView { + toRet := make([]FeatureView, 0, len(views)) + for idx := range views { + for fsidx := range views[idx].FlagSets { + if _, ok := sets[views[idx].FlagSets[fsidx].Name]; ok { + fsinfo := views[idx].FlagSets[fsidx] + if fsinfo.Active || fsinfo.LastUpdated > since { + toRet = append(toRet, views[idx].clone()) + } + } + } + + } + return toRet +} + +func BenchmarkFlagSetProcessing(b *testing.B) { + + b.Run("sorted-slice", func(b *testing.B) { + data := make([]benchmarkDataSlices, 0, b.N) + for i := 0; i < b.N; i++ { + data = append(data, setupRandomData(20, 50, 500, 20, 10)) + } + + b.ResetTimer() // to ignore setup time & allocs + + for i := 0; i < b.N; i++ { + copyAndFilter(data[i].views, data[i].sets, 1) + } + }) + + b.Run("maps", func(b *testing.B) { + data := make([]benchmarkDataMaps, 0, b.N) + for i := 0; i < b.N; i++ { + d := setupRandomData(20, 50, 500, 20, 10) + data = append(data, d.toBenchmarkDataForMaps()) + } + + b.ResetTimer() // to ignore setup time & allocs + + for i := 0; i < b.N; i++ { + copyAndFilterUsingMaps(data[i].views, data[i].sets, 1) + } + }) +}