Update the series index on purpose

Signed-off-by: Gao Hongtao <hanahmily@gmail.com>

CHANGES.md

@@ -8,7 +8,7 @@ Release Notes.
 
 - Add the `bydbctl analyze series` command to analyze the series data.
 - Index: Remove sortable field from the stored field. If a field is sortable only, it won't be stored.
-- Index: Support InsertIfAbsent functionality which ensures documents are only inserted if their docIDs are not already present in the current index
+- Index: Support InsertIfAbsent functionality which ensures documents are only inserted if their docIDs are not already present in the current index. There is a exception for the documents with extra index fields more than the entity's index fields.
 
 ### Bug Fixes
 

dist/LICENSE

@@ -178,7 +178,7 @@
 Apache-2.0 licenses
 ========================================================================
 
-    github.com/SkyAPM/bluge v0.0.0-20241110125856-046bc03b30ab Apache-2.0
+    github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201 Apache-2.0
     github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118 Apache-2.0
     github.com/apache/skywalking-cli v0.0.0-20240227151024-ee371a210afe Apache-2.0
     github.com/blevesearch/segment v0.9.1 Apache-2.0

go.mod

@@ -157,7 +157,7 @@ require (
 
 replace (
 	github.com/benbjohnson/clock v1.3.0 => github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97
-	github.com/blugelabs/bluge => github.com/SkyAPM/bluge v0.0.0-20241111105056-b74b30692911
+	github.com/blugelabs/bluge => github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201
 	github.com/blugelabs/bluge_segment_api => github.com/zinclabs/bluge_segment_api v1.0.0
 	github.com/blugelabs/ice => github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118
 )

go.sum

@@ -6,8 +6,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
 github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ=
 github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
-github.com/SkyAPM/bluge v0.0.0-20241111105056-b74b30692911 h1:vSBISZR7gZJHPqVirGlfqEa1mmX8gP6liz9dHhj62eM=
-github.com/SkyAPM/bluge v0.0.0-20241111105056-b74b30692911/go.mod h1:6o9wC3xO3qb5Q7VmD1x0r54qQBDpO9+ghGAQvuOHsCU=
+github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201 h1:QX/WvtL8j5Zrbs68EVEiOE2nFQSvoT5oTkOFh2uNSpg=
+github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201/go.mod h1:6o9wC3xO3qb5Q7VmD1x0r54qQBDpO9+ghGAQvuOHsCU=
 github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97 h1:FKuhJ+6n/DHspGeLleeNbziWnKr9gHKYN4q7NcoCp4s=
 github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97/go.mod h1:2xGRl9H1pllhxTbEGO1W3gDkip8P9GQaHPni/wpdR44=
 github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118 h1:Ja62sgOCp2qPTd8Xmldv1U83v11IRIsh6KlB7UaFLj4=

pkg/index/inverted/inverted_series.go

@@ -68,7 +68,11 @@ func (s *store) SeriesBatch(batch index.Batch) error {
 		if d.Timestamp > 0 {
 			doc.AddField(bluge.NewDateTimeField(timestampField, time.Unix(0, d.Timestamp)).StoreValue())
 		}
-		b.InsertIfAbsent(doc.ID(), doc)
+		if len(d.Fields) == 0 {
+			b.InsertIfAbsent(doc.ID(), doc)
+		} else {
+			b.Update(doc.ID(), doc)
+		}
 	}
 	return s.writer.Batch(b)
 }

pkg/index/inverted/inverted_series_test.go

@@ -76,16 +76,6 @@ func TestStore_Search(t *testing.T) {
 			term:       [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")},
 			projection: []index.FieldKey{fieldKeyDuration, fieldKeyServiceName},
 			want: []index.SeriesDocument{
-				{
-					Key: index.Series{
-						ID:           common.SeriesID(1),
-						EntityValues: []byte("test1"),
-					},
-					Fields: map[string][]byte{
-						fieldKeyDuration.Marshal():    nil,
-						fieldKeyServiceName.Marshal(): nil,
-					},
-				},
 				{
 					Key: index.Series{
 						ID:           common.SeriesID(2),
@@ -106,21 +96,22 @@ func TestStore_Search(t *testing.T) {
 						fieldKeyServiceName.Marshal(): nil,
 					},
 				},
-			},
-		},
-		{
-			term:       [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")},
-			projection: []index.FieldKey{fieldKeyDuration},
-			want: []index.SeriesDocument{
 				{
 					Key: index.Series{
 						ID:           common.SeriesID(1),
 						EntityValues: []byte("test1"),
 					},
 					Fields: map[string][]byte{
-						fieldKeyDuration.Marshal(): nil,
+						fieldKeyDuration.Marshal():    nil,
+						fieldKeyServiceName.Marshal(): nil,
 					},
 				},
+			},
+		},
+		{
+			term:       [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")},
+			projection: []index.FieldKey{fieldKeyDuration},
+			want: []index.SeriesDocument{
 				{
 					Key: index.Series{
 						ID:           common.SeriesID(2),
@@ -139,21 +130,21 @@ func TestStore_Search(t *testing.T) {
 						fieldKeyDuration.Marshal(): convert.Int64ToBytes(int64(500)),
 					},
 				},
-			},
-		},
-		{
-			term:       [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")},
-			projection: []index.FieldKey{fieldKeyServiceName},
-			want: []index.SeriesDocument{
 				{
 					Key: index.Series{
 						ID:           common.SeriesID(1),
 						EntityValues: []byte("test1"),
 					},
 					Fields: map[string][]byte{
-						fieldKeyServiceName.Marshal(): nil,
+						fieldKeyDuration.Marshal(): nil,
 					},
 				},
+			},
+		},
+		{
+			term:       [][]byte{[]byte("test1"), []byte("test2"), []byte("test3"), []byte("foo")},
+			projection: []index.FieldKey{fieldKeyServiceName},
+			want: []index.SeriesDocument{
 				{
 					Key: index.Series{
 						ID:           common.SeriesID(2),
@@ -172,6 +163,15 @@ func TestStore_Search(t *testing.T) {
 						fieldKeyServiceName.Marshal(): nil,
 					},
 				},
+				{
+					Key: index.Series{
+						ID:           common.SeriesID(1),
+						EntityValues: []byte("test1"),
+					},
+					Fields: map[string][]byte{
+						fieldKeyServiceName.Marshal(): nil,
+					},
+				},
 			},
 		},
 		{