From 40d4b3654e0e6553aa0c97d8bea2f2d5ea063a6c Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Mon, 1 Jul 2024 15:55:08 +0530 Subject: [PATCH 1/3] replace original db with optimized db Signed-off-by: Vivek Kumar Sahu --- pkg/compliance/db.go | 112 ++++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 43 deletions(-) diff --git a/pkg/compliance/db.go b/pkg/compliance/db.go index a5b7bd8..eec5c18 100644 --- a/pkg/compliance/db.go +++ b/pkg/compliance/db.go @@ -2,74 +2,100 @@ package compliance import ( "fmt" - - "github.com/samber/lo" + "sync" ) type db struct { - records []*record + mu sync.RWMutex + records map[int][]*record // store record as a value of a Map with a key as a "check_key" + ids map[string][]*record // store record as a value of a Map with a key as a "id" + keyIds map[string]map[int][]*record // store record as a value of a Map with a key as a "check_key an id" + allIds map[string]struct{} // Set of all unique ids } +// newDB initializes and returns a new database instance. func newDB() *db { - return &db{} + return &db{ + records: make(map[int][]*record), + ids: make(map[string][]*record), + keyIds: make(map[string]map[int][]*record), + allIds: make(map[string]struct{}), + } } +// addRecord adds a single record to the database func (d *db) addRecord(r *record) { - d.records = append(d.records, r) + d.mu.Lock() + defer d.mu.Unlock() + + // store record using a key + d.records[r.check_key] = append(d.records[r.check_key], r) + + // store record using a id + d.ids[r.id] = append(d.ids[r.id], r) + if d.keyIds[r.id] == nil { + d.keyIds[r.id] = make(map[int][]*record) + } + + // store record using a key and id + d.keyIds[r.id][r.check_key] = append(d.keyIds[r.id][r.check_key], r) + + d.allIds[r.id] = struct{}{} } +// addRecords adds multiple records to the database func (d *db) addRecords(rs []*record) { - d.records = append(d.records, rs...) + for _, r := range rs { + d.addRecord(r) + } } -func (d *db) getRecords(key int) []record { - var rs []record - for _, r := range d.records { - if r.check_key == key { - rs = append(rs, *r) - } - } - return rs +// getRecords retrieves records by the given "check_key" +func (d *db) getRecords(key int) []*record { + d.mu.RLock() + defer d.mu.RUnlock() + return d.records[key] } +// getAllIds retrieves all unique ids in the database func (d *db) getAllIds() []string { - var ids []string - for _, r := range d.records { - ids = append(ids, r.id) + d.mu.RLock() + defer d.mu.RUnlock() + ids := make([]string, 0, len(d.allIds)) + for id := range d.allIds { + ids = append(ids, id) } - - return lo.Uniq(ids) + return ids } -func (d *db) getRecordsById(id string) []record { - var rs []record - for _, r := range d.records { - if r.id == id { - rs = append(rs, *r) - } - } - return rs +// getRecordsById retrieves records by the given "id" +func (d *db) getRecordsById(id string) []*record { + d.mu.RLock() + defer d.mu.RUnlock() + return d.ids[id] } -func (d *db) getRecordsByKeyId(key int, id string) []record { - var rs []record - for _, r := range d.records { - if r.check_key == key && r.id == id { - rs = append(rs, *r) - } - } - return rs +// getRecordsByKeyId retrieves records by the given "check_key" and "id" +func (d *db) getRecordsByKeyId(key int, id string) []*record { + d.mu.RLock() + defer d.mu.RUnlock() + return d.keyIds[id][key] } -func (d *db) dumpAll(key []int) { - for _, r := range d.records { - if len(key) == 0 { - fmt.Printf("id: %s, key: %d, value: %s\n", r.id, r.check_key, r.check_value) - continue - } - for _, k := range key { - if r.check_key == k { +// dumpAll prints all records, optionally filtered by the given keys +func (d *db) dumpAll(keys []int) { + d.mu.RLock() + defer d.mu.RUnlock() + for _, records := range d.records { + for _, r := range records { + if len(keys) == 0 { fmt.Printf("id: %s, key: %d, value: %s\n", r.id, r.check_key, r.check_value) + continue + } + for _, k := range keys { + if r.check_key == k { + fmt.Printf("id: %s, key: %d, value: %s\n", r.id, r.check_key, r.check_value) + } } } } From 8bb47be137f61e49b81221d13a379a5518e1763f Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Mon, 1 Jul 2024 15:55:24 +0530 Subject: [PATCH 2/3] add benchmark test Signed-off-by: Vivek Kumar Sahu --- pkg/compliance/db_test.go | 69 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 pkg/compliance/db_test.go diff --git a/pkg/compliance/db_test.go b/pkg/compliance/db_test.go new file mode 100644 index 0000000..72f2062 --- /dev/null +++ b/pkg/compliance/db_test.go @@ -0,0 +1,69 @@ +package compliance + +import ( + "fmt" + "math/rand" + "testing" +) + +const ( + numRecords = 1000000 // Number of records to test with +) + +// Generate large data set +func generateRecords(n int) []*record { + var records []*record + for i := 0; i < n; i++ { + records = append(records, &record{ + check_key: rand.Intn(1000), + check_value: fmt.Sprintf("value_%d", i), + id: fmt.Sprintf("id_%d", rand.Intn(1000)), + score: rand.Float64() * 100, + required: rand.Intn(2) == 0, + }) + } + return records +} + +// Benchmark original db implementation +func BenchmarkOriginalDB(b *testing.B) { + records := generateRecords(numRecords) + db := newDB() + + // Benchmark insertion + b.Run("Insert", func(b *testing.B) { + for i := 0; i < b.N; i++ { + db.addRecords(records) + } + }) + + // Benchmark retrieval by key + b.Run("GetByKey", func(b *testing.B) { + for i := 0; i < b.N; i++ { + db.getRecords(rand.Intn(1000)) + } + }) + + // Benchmark retrieval by ID + b.Run("GetByID", func(b *testing.B) { + for i := 0; i < b.N; i++ { + db.getRecordsById(fmt.Sprintf("id_%d", rand.Intn(1000))) + } + }) + + // Benchmark for combined retrieval by key and ID case + b.Run("GetByKeyAndIDTogether", func(b *testing.B) { + for i := 0; i < b.N; i++ { + key := rand.Intn(1000) + id := fmt.Sprintf("id_%d", rand.Intn(1000)) + db.getRecordsByKeyId(key, id) + } + }) + + // Benchmark for retrieval of all IDs case + b.Run("GetAllIDs", func(b *testing.B) { + for i := 0; i < b.N; i++ { + db.getAllIds() + } + }) +} From a1ebab467eb30669cca3dc0154adb77b2f198eb5 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Wed, 17 Jul 2024 11:12:12 +0530 Subject: [PATCH 3/3] remove mutex due to absence of goroutines Signed-off-by: Vivek Kumar Sahu --- pkg/compliance/db.go | 49 +++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/pkg/compliance/db.go b/pkg/compliance/db.go index eec5c18..1cff02b 100644 --- a/pkg/compliance/db.go +++ b/pkg/compliance/db.go @@ -2,43 +2,38 @@ package compliance import ( "fmt" - "sync" ) type db struct { - mu sync.RWMutex - records map[int][]*record // store record as a value of a Map with a key as a "check_key" - ids map[string][]*record // store record as a value of a Map with a key as a "id" - keyIds map[string]map[int][]*record // store record as a value of a Map with a key as a "check_key an id" - allIds map[string]struct{} // Set of all unique ids + keyRecords map[int][]*record // store record as a value of a Map with a key as a "check_key" + idRecords map[string][]*record // store record as a value of a Map with a key as a "id" + idKeyRecords map[string]map[int][]*record // store record as a value of a Map with a key as a "check_key an id" + allIds map[string]struct{} // Set of all unique ids } // newDB initializes and returns a new database instance. func newDB() *db { return &db{ - records: make(map[int][]*record), - ids: make(map[string][]*record), - keyIds: make(map[string]map[int][]*record), - allIds: make(map[string]struct{}), + keyRecords: make(map[int][]*record), + idRecords: make(map[string][]*record), + idKeyRecords: make(map[string]map[int][]*record), + allIds: make(map[string]struct{}), } } // addRecord adds a single record to the database func (d *db) addRecord(r *record) { - d.mu.Lock() - defer d.mu.Unlock() - // store record using a key - d.records[r.check_key] = append(d.records[r.check_key], r) + d.keyRecords[r.check_key] = append(d.keyRecords[r.check_key], r) // store record using a id - d.ids[r.id] = append(d.ids[r.id], r) - if d.keyIds[r.id] == nil { - d.keyIds[r.id] = make(map[int][]*record) + d.idRecords[r.id] = append(d.idRecords[r.id], r) + if d.idKeyRecords[r.id] == nil { + d.idKeyRecords[r.id] = make(map[int][]*record) } // store record using a key and id - d.keyIds[r.id][r.check_key] = append(d.keyIds[r.id][r.check_key], r) + d.idKeyRecords[r.id][r.check_key] = append(d.idKeyRecords[r.id][r.check_key], r) d.allIds[r.id] = struct{}{} } @@ -52,15 +47,11 @@ func (d *db) addRecords(rs []*record) { // getRecords retrieves records by the given "check_key" func (d *db) getRecords(key int) []*record { - d.mu.RLock() - defer d.mu.RUnlock() - return d.records[key] + return d.keyRecords[key] } // getAllIds retrieves all unique ids in the database func (d *db) getAllIds() []string { - d.mu.RLock() - defer d.mu.RUnlock() ids := make([]string, 0, len(d.allIds)) for id := range d.allIds { ids = append(ids, id) @@ -70,23 +61,17 @@ func (d *db) getAllIds() []string { // getRecordsById retrieves records by the given "id" func (d *db) getRecordsById(id string) []*record { - d.mu.RLock() - defer d.mu.RUnlock() - return d.ids[id] + return d.idRecords[id] } // getRecordsByKeyId retrieves records by the given "check_key" and "id" func (d *db) getRecordsByKeyId(key int, id string) []*record { - d.mu.RLock() - defer d.mu.RUnlock() - return d.keyIds[id][key] + return d.idKeyRecords[id][key] } // dumpAll prints all records, optionally filtered by the given keys func (d *db) dumpAll(keys []int) { - d.mu.RLock() - defer d.mu.RUnlock() - for _, records := range d.records { + for _, records := range d.keyRecords { for _, r := range records { if len(keys) == 0 { fmt.Printf("id: %s, key: %d, value: %s\n", r.id, r.check_key, r.check_value)