Improve the inverted index (#555)

.golangci.yml

@@ -16,139 +16,138 @@
 linters:
   disable-all: true
   enable:
-    - asasalint
-    - asciicheck
-    # - containedctx
-    - contextcheck
-    - decorder
-    - dogsled
-    - errname
-    - errcheck
-    - errchkjson
-    - errorlint
-    - exhaustive
-    # TODO:// enable this lint
-    # - exhaustruct
-    - gci
-    - goconst
-    - gocritic
-    - gocyclo
-    - godot
-    - gofumpt
-    - goprintffuncname
-    - gosec
-    - gosimple
-    - govet
-    - importas
-    - ineffassign
-    - lll
-    - misspell
-    - staticcheck
-    - stylecheck
-    - typecheck
-    - unconvert
-    - unparam
-    - unused
-    - usestdlibvars
-    - whitespace
+  - asasalint
+  - asciicheck
+  # - containedctx
+  - contextcheck
+  - decorder
+  - dogsled
+  - errname
+  - errcheck
+  - errchkjson
+  - errorlint
+  - exhaustive
+  # TODO:// enable this lint
+  # - exhaustruct
+  - gci
+  - goconst
+  - gocritic
+  - gocyclo
+  - godot
+  - gofumpt
+  - goprintffuncname
+  - gosec
+  - gosimple
+  - govet
+  - importas
+  - ineffassign
+  - lll
+  - misspell
+  - staticcheck
+  - stylecheck
+  - typecheck
+  - unconvert
+  - unparam
+  - unused
+  - usestdlibvars
+  - whitespace
 linters-settings:
   importas:
     no-unaliased: true
     alias:
-      - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1
-        alias: commonv1
-      - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1
-        alias: databasev1
-      - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1
-        alias: modelv1
-      - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1
-        alias: propertyv1
-      - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1
-        alias: measurev1
-      - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1
-        alias: streamv1
-      - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1
-        alias: clusterv1
-      - pkg: github.com/apache/skywalking-banyandb/pkg/pb/v1
-        alias: pbv1
+    - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1
+      alias: commonv1
+    - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1
+      alias: databasev1
+    - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/model/v1
+      alias: modelv1
+    - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1
+      alias: propertyv1
+    - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1
+      alias: measurev1
+    - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1
+      alias: streamv1
+    - pkg: github.com/apache/skywalking-banyandb/api/proto/banyandb/cluster/v1
+      alias: clusterv1
+    - pkg: github.com/apache/skywalking-banyandb/pkg/pb/v1
+      alias: pbv1
   lll:
     line-length: 170
   goconst:
     min-occurrences: 4
   govet:
     enable:
-      - fieldalignment
-      - shadow
+    - fieldalignment
+    - shadow
   misspell:
     locale: US
   unparam:
     check-exported: false
   gci:
     sections:
-      - standard
-      - default
-      - prefix(github.com/apache/skywalking-banyandb/)
+    - standard
+    - default
+    - prefix(github.com/apache/skywalking-banyandb/)
   gocritic:
     enabled-checks:
-      - appendCombine
-      - boolExprSimplify
-      - builtinShadow
-      - commentedOutCode
-      - commentedOutImport
-      - docStub
-      - emptyFallthrough
-      - equalFold
-      - hexLiteral
-      - indexAlloc
-      - initClause
-      - methodExprCall
-      - nilValReturn
-      - octalLiteral
-      - rangeExprCopy
-      - stringXbytes
-      - typeAssertChain
-      - typeUnparen
-      - unnecessaryBlock
-      - weakCond
+    - appendCombine
+    - boolExprSimplify
+    - builtinShadow
+    - commentedOutCode
+    - commentedOutImport
+    - docStub
+    - emptyFallthrough
+    - equalFold
+    - hexLiteral
+    - indexAlloc
+    - initClause
+    - methodExprCall
+    - nilValReturn
+    - octalLiteral
+    - rangeExprCopy
+    - stringXbytes
+    - typeAssertChain
+    - typeUnparen
+    - unnecessaryBlock
+    - weakCond
   godot:
     #   declarations - for top level declaration comments (default);
     #   toplevel     - for top level comments;
     #   all          - for all comments.
     scope: toplevel
   gosec:
     excludes:
-      - G115 # integer overflow conversion (TODO: verify these)
+    - G115 # integer overflow conversion (TODO: verify these)
   staticcheck:
-    checks: ["all", "-ST1000", "-ST1016", "-ST1020", "-ST1021", "-ST1022"]
+    checks: [ "all", "-ST1000", "-ST1016", "-ST1020", "-ST1021", "-ST1022" ]
   exhaustive:
     check:
-      - switch
-      - map
+    - switch
+    - map
     default-signifies-exhaustive: true
     ignore-enum-members: ".+UNSPECIFIED$"
   exhaustruct:
     exclude:
-      - 'cobra\.Command$'
+    - 'cobra\.Command$'
 
 run:
   go: "1.23"
 issues:
   exclude-rules:
-    - path: ".*\\.pb\\.go"
-      linters:
-        - "*"
-    - path: ".*\\.pb\\.validate\\.go"
-      linters:
-        - "*"
-    - path: ".*\\.gen\\.go"
-      linters:
-        - "*"
-    - path: _test\.go$|^tests/|^samples/
-      linters:
-        - errcheck
-        - maligned
-    - linters:
-        - staticcheck
-      text: "SA1019: package github.com/golang/protobuf"
-  max-per-linter: 0
+  - path: ".*\\.pb\\.go"
+    linters:
+    - "*"
+  - path: ".*\\.pb\\.validate\\.go"
+    linters:
+    - "*"
+  - path: ".*\\.gen\\.go"
+    linters:
+    - "*"
+  - path: _test\.go$|^tests/|^samples/
+    linters:
+    - errcheck
+    - maligned
+  - linters:
+    - staticcheck
+    text: "SA1019: package github.com/golang/protobuf"
   max-same-issues: 0

CHANGES.md

@@ -8,10 +8,12 @@ Release Notes.
 
 - Add the `bydbctl analyze series` command to analyze the series data.
 - Index: Remove sortable field from the stored field. If a field is sortable only, it won't be stored.
+- Index: Support InsertIfAbsent functionality which ensures documents are only inserted if their docIDs are not already present in the current index. There is a exception for the documents with extra index fields more than the entity's index fields.
 
 ### Bug Fixes
 
 - Fix the bug that TopN processing item leak. The item can not be updated but as a new item.
+- Resolve data race in Stats methods of the inverted index.
 
 ### Documentation
 

banyand/internal/storage/index.go

@@ -71,7 +71,7 @@ func newSeriesIndex(ctx context.Context, root string, flushTimeoutSeconds int64,
 }
 
 func (s *seriesIndex) Write(docs index.Documents) error {
-	return s.store.Batch(index.Batch{
+	return s.store.SeriesBatch(index.Batch{
 		Documents: docs,
 	})
 }

dist/LICENSE

@@ -178,6 +178,8 @@
 Apache-2.0 licenses
 ========================================================================
 
+    github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201 Apache-2.0
+    github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118 Apache-2.0
     github.com/apache/skywalking-cli v0.0.0-20240227151024-ee371a210afe Apache-2.0
     github.com/blevesearch/segment v0.9.1 Apache-2.0
     github.com/blevesearch/vellum v1.0.10 Apache-2.0
@@ -208,9 +210,7 @@ Apache-2.0 licenses
     github.com/spf13/afero v1.11.0 Apache-2.0
     github.com/spf13/cobra v1.8.1 Apache-2.0
     github.com/tklauser/numcpus v0.8.0 Apache-2.0
-    github.com/zinclabs/bluge v1.1.5 Apache-2.0
     github.com/zinclabs/bluge_segment_api v1.0.0 Apache-2.0
-    github.com/zinclabs/ice v1.1.3 Apache-2.0
     go.etcd.io/etcd/api/v3 v3.5.16 Apache-2.0
     go.etcd.io/etcd/client/pkg/v3 v3.5.16 Apache-2.0
     go.etcd.io/etcd/client/v2 v2.305.16 Apache-2.0
@@ -283,8 +283,8 @@ BSD-3-Clause licenses
     golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 BSD-3-Clause
     golang.org/x/mod v0.21.0 BSD-3-Clause
     golang.org/x/net v0.29.0 BSD-3-Clause
-    golang.org/x/sys v0.25.0 BSD-3-Clause
-    golang.org/x/text v0.18.0 BSD-3-Clause
+    golang.org/x/sys v0.26.0 BSD-3-Clause
+    golang.org/x/text v0.19.0 BSD-3-Clause
     golang.org/x/time v0.6.0 BSD-3-Clause
     golang.org/x/tools v0.25.0 BSD-3-Clause
     google.golang.org/protobuf v1.34.2 BSD-3-Clause
@@ -293,7 +293,7 @@ BSD-3-Clause licenses
 BSD-3-Clause and Apache-2.0 and MIT licenses
 ========================================================================
 
-    github.com/klauspost/compress v1.17.9 BSD-3-Clause and Apache-2.0 and MIT
+    github.com/klauspost/compress v1.17.11 BSD-3-Clause and Apache-2.0 and MIT
 
 ========================================================================
 CC0-1.0 licenses

go.mod

@@ -96,7 +96,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.9
+	github.com/klauspost/compress v1.17.11
 	github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -144,8 +144,8 @@ require (
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/net v0.29.0 // indirect
-	golang.org/x/sys v0.25.0
-	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/sys v0.26.0
+	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	golang.org/x/tools v0.25.0 // indirect
 	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
@@ -157,7 +157,7 @@ require (
 
 replace (
 	github.com/benbjohnson/clock v1.3.0 => github.com/SkyAPM/clock v1.3.1-0.20220809233656-dc7607c94a97
-	github.com/blugelabs/bluge => github.com/zinclabs/bluge v1.1.5
+	github.com/blugelabs/bluge => github.com/SkyAPM/bluge v0.0.0-20241111124917-c317df1af201
 	github.com/blugelabs/bluge_segment_api => github.com/zinclabs/bluge_segment_api v1.0.0
-	github.com/blugelabs/ice => github.com/zinclabs/ice v1.1.3
+	github.com/blugelabs/ice => github.com/SkyAPM/ice v0.0.0-20241108011032-c3d8eea75118
 )