Skip to content

Commit

Permalink
fix(wal): fix recover with insufficient buffer (#568)
Browse files Browse the repository at this point in the history
### Motivation

We are using `mmap` for WAL, which will reflect a larger buffer than we
need. We should skip the index recovery if the current segment is
insufficient for a record header.


```
panic: runtime error: slice bounds out of range [:67108867] with capacity 67108864

goroutine 704 [running]:
github.com/streamnative/oxia/server/wal/codec.ReadInt(...)
	/src/oxia/server/wal/codec/codec.go:165
github.com/streamnative/oxia/server/wal/codec.(*V2).ReadHeaderWithValidation(0xc001728000?, {0x7fd418691000?, 0x4000000, 0x4000000?}, 0x1a920ac?)
	/src/oxia/server/wal/codec/v2.go:109 +0x3ca
github.com/streamnative/oxia/server/wal/codec.(*V2).RecoverIndex(0x34a4da0, {0x7fd418691000, 0x4000000, 0x4000000}, 0x0, 0xff7299, 0xc00087fe40)
	/src/oxia/server/wal/codec/v2.go:212 +0x13b
github.com/streamnative/oxia/server/wal.newReadWriteSegment({0xc0003595e0, 0x1f}, 0xff7299, 0x4000000, 0xbad4b142, {0x2475060, 0xc00126a900})
	/src/oxia/server/wal/wal_rw_segment.go:105 +0x39f
github.com/streamnative/oxia/server/wal.(*wal).recoverWal(0xc00037b1e0)
	/src/oxia/server/wal/wal_impl.go:561 +0xbc
github.com/streamnative/oxia/server/wal.newWal({0xc00087fbd0, 0xc}, 0xc, 0xc000781220, {0x2475060, 0xc00126a900}, {0x2474740, 0x35b77c0}, 0x8bb2c97000)
	/src/oxia/server/wal/wal_impl.go:144 +0x76f
github.com/streamnative/oxia/server/wal.(*walFactory).NewWal(0x2493280?, {0xc00087fbd0?, 0xc00087fbd0?}, 0xc?, {0x2475060?, 0xc00126a900?})
```
  • Loading branch information
mattisonchao authored Nov 12, 2024
1 parent 919673c commit ab60cd9
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 1 deletion.
2 changes: 1 addition & 1 deletion server/wal/codec/v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func (v *V2) RecoverIndex(buf []byte, startFileOffset uint32, baseEntryOffset in

index = BorrowEmptyIndexBuf()

for newFileOffset < maxSize {
for newFileOffset+v.HeaderSize <= maxSize {
var payloadSize uint32
var payloadCrc uint32
var err error
Expand Down
11 changes: 11 additions & 0 deletions server/wal/codec/v2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,3 +319,14 @@ func TestV2_ReadWithValidation(t *testing.T) {
assert.NoError(t, err)
assert.EqualValues(t, wPayloadCrc, rPayloadCrc)
}

func TestV2_RecoveryWithNotEnoughBuf(t *testing.T) {
buf := make([]byte, 16)
payloadSize := uint32(len(buf)) - v2.HeaderSize - 1
payload := bytes.Repeat([]byte("A"), int(payloadSize))
_, wPayloadCrc := v2.WriteRecord(buf, 0, 0, payload)
_, rLastCrc, _, entryOffset, err := v2.RecoverIndex(buf, 0, 0, nil)
assert.NoError(t, err)
assert.EqualValues(t, wPayloadCrc, rLastCrc)
assert.EqualValues(t, entryOffset, 0)
}

0 comments on commit ab60cd9

Please sign in to comment.