diff --git a/CHANGES.md b/CHANGES.md index 6ff0ff40..38415e5f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,7 @@ Release Notes. - Add the `bydbctl analyze series` command to analyze the series data. - Index: Remove sortable field from the stored field. If a field is sortable only, it won't be stored. -- Index: Support InsertIfAbsent functionality which ensures documents are only inserted if their docIDs are not already present in the current index +- Index: Support InsertIfAbsent functionality which ensures documents are only inserted if their docIDs are not already present in the current index. There is a exception for the documents with extra index fields more than the entity's index fields. ### Bug Fixes diff --git a/dist/LICENSE b/dist/LICENSE index 715b531b..a6503207 100644 --- a/dist/LICENSE +++ b/dist/LICENSE @@ -178,7 +178,7 @@ Apache-2.0 licenses ======================================================================== - github.com/SkyAPM/bluge v0.0.0-20241110125856-046bc03b30ab Apache-2.0 + github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201 Apache-2.0 github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118 Apache-2.0 github.com/apache/skywalking-cli v0.0.0-20240227151024-ee371a210afe Apache-2.0 github.com/blevesearch/segment v0.9.1 Apache-2.0 diff --git a/go.mod b/go.mod index 27dacd51..56646ccf 100644 --- a/go.mod +++ b/go.mod @@ -157,7 +157,7 @@ require ( replace ( github.com/benbjohnson/clock v1.3.0 => github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97 - github.com/blugelabs/bluge => github.com/SkyAPM/bluge v0.0.0-20241111105056-b74b30692911 + github.com/blugelabs/bluge => github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201 github.com/blugelabs/bluge_segment_api => github.com/zinclabs/bluge_segment_api v1.0.0 github.com/blugelabs/ice => github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118 ) diff --git a/go.sum b/go.sum index 65979c81..0d9c71db 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/SkyAPM/bluge v0.0.0-20241111105056-b74b30692911 h1:vSBISZR7gZJHPqVirGlfqEa1mmX8gP6liz9dHhj62eM= -github.com/SkyAPM/bluge v0.0.0-20241111105056-b74b30692911/go.mod h1:6o9wC3xO3qb5Q7VmD1x0r54qQBDpO9+ghGAQvuOHsCU= +github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201 h1:QX/WvtL8j5Zrbs68EVEiOE2nFQSvoT5oTkOFh2uNSpg= +github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201/go.mod h1:6o9wC3xO3qb5Q7VmD1x0r54qQBDpO9+ghGAQvuOHsCU= github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97 h1:FKuhJ+6n/DHspGeLleeNbziWnKr9gHKYN4q7NcoCp4s= github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97/go.mod h1:2xGRl9H1pllhxTbEGO1W3gDkip8P9GQaHPni/wpdR44= github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118 h1:Ja62sgOCp2qPTd8Xmldv1U83v11IRIsh6KlB7UaFLj4= diff --git a/pkg/index/inverted/inverted_series.go b/pkg/index/inverted/inverted_series.go index 10c7718c..9d0b15ed 100644 --- a/pkg/index/inverted/inverted_series.go +++ b/pkg/index/inverted/inverted_series.go @@ -68,7 +68,11 @@ func (s *store) SeriesBatch(batch index.Batch) error { if d.Timestamp > 0 { doc.AddField(bluge.NewDateTimeField(timestampField, time.Unix(0, d.Timestamp)).StoreValue()) } - b.InsertIfAbsent(doc.ID(), doc) + if len(d.Fields) == 0 { + b.InsertIfAbsent(doc.ID(), doc) + } else { + b.Update(doc.ID(), doc) + } } return s.writer.Batch(b) } diff --git a/pkg/index/inverted/inverted_series_test.go b/pkg/index/inverted/inverted_series_test.go index c0e09e1e..cd0a3bd4 100644 --- a/pkg/index/inverted/inverted_series_test.go +++ b/pkg/index/inverted/inverted_series_test.go @@ -76,16 +76,6 @@ func TestStore_Search(t *testing.T) { term: [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")}, projection: []index.FieldKey{fieldKeyDuration, fieldKeyServiceName}, want: []index.SeriesDocument{ - { - Key: index.Series{ - ID: common.SeriesID(1), - EntityValues: []byte("test1"), - }, - Fields: map[string][]byte{ - fieldKeyDuration.Marshal(): nil, - fieldKeyServiceName.Marshal(): nil, - }, - }, { Key: index.Series{ ID: common.SeriesID(2), @@ -106,21 +96,22 @@ func TestStore_Search(t *testing.T) { fieldKeyServiceName.Marshal(): nil, }, }, - }, - }, - { - term: [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")}, - projection: []index.FieldKey{fieldKeyDuration}, - want: []index.SeriesDocument{ { Key: index.Series{ ID: common.SeriesID(1), EntityValues: []byte("test1"), }, Fields: map[string][]byte{ - fieldKeyDuration.Marshal(): nil, + fieldKeyDuration.Marshal(): nil, + fieldKeyServiceName.Marshal(): nil, }, }, + }, + }, + { + term: [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")}, + projection: []index.FieldKey{fieldKeyDuration}, + want: []index.SeriesDocument{ { Key: index.Series{ ID: common.SeriesID(2), @@ -139,21 +130,21 @@ func TestStore_Search(t *testing.T) { fieldKeyDuration.Marshal(): convert.Int64ToBytes(int64(500)), }, }, - }, - }, - { - term: [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")}, - projection: []index.FieldKey{fieldKeyServiceName}, - want: []index.SeriesDocument{ { Key: index.Series{ ID: common.SeriesID(1), EntityValues: []byte("test1"), }, Fields: map[string][]byte{ - fieldKeyServiceName.Marshal(): nil, + fieldKeyDuration.Marshal(): nil, }, }, + }, + }, + { + term: [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")}, + projection: []index.FieldKey{fieldKeyServiceName}, + want: []index.SeriesDocument{ { Key: index.Series{ ID: common.SeriesID(2), @@ -172,6 +163,15 @@ func TestStore_Search(t *testing.T) { fieldKeyServiceName.Marshal(): nil, }, }, + { + Key: index.Series{ + ID: common.SeriesID(1), + EntityValues: []byte("test1"), + }, + Fields: map[string][]byte{ + fieldKeyServiceName.Marshal(): nil, + }, + }, }, }, {