From 99d56092a75fb896b27ee09a3b6c1ac3f521ab2a Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Thu, 14 Dec 2023 06:15:16 +0100 Subject: [PATCH 1/3] feat(collection): Added offset pagination --- src/collection.ts | 27 +++++++++++++++++-- src/types.ts | 7 +++++ src/utils.ts | 11 +++++++- tests/collection/properties.test.ts | 24 ++++++++++++++++- tests/indexable_collection/properties.test.ts | 24 ++++++++++++++++- .../serialized_collection/properties.test.ts | 24 ++++++++++++++++- .../properties.test.ts | 24 ++++++++++++++++- 7 files changed, 134 insertions(+), 7 deletions(-) diff --git a/src/collection.ts b/src/collection.ts index 4bc0a2b..915740a 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -44,6 +44,7 @@ import { checkIndices, compress, createHandlerId, + createListOptions, createListSelector, decompress, deleteIndices, @@ -441,10 +442,21 @@ export class Collection< const selector = createListSelector(keyPrefix, options) // Create hsitory entries iterator - const iter = this.kv.list>(selector, options) + const listOptions = createListOptions(options) + const iter = this.kv.list>(selector, listOptions) // Collect history entries + let count = 0 + const offset = options?.offset ?? 0 for await (const { value, key } of iter) { + // Skip by offset + count++ + + if (count <= offset) { + continue + } + + // Cast history entry let historyEntry: HistoryEntry = value // Handle serialized entries @@ -1052,6 +1064,7 @@ export class Collection< if (selectsAll(options)) { // Create list iterator and empty keys list, init atomic operation const iter = this.kv.list({ prefix: this._keys.base }, options) + const keys: Deno.KvKey[] = [] const atomic = new AtomicWrapper(this.kv, options?.atomicBatchSize) @@ -1926,7 +1939,8 @@ export class Collection< ) { // Create list iterator with given options const selector = createListSelector(prefixKey, options) - const iter = this.kv.list(selector, options) + const listOptions = createListOptions(options) + const iter = this.kv.list(selector, listOptions) // Initiate lists const docs: Document[] = [] @@ -1934,7 +1948,16 @@ export class Collection< const errors: unknown[] = [] // Loop over each document entry + let count = 0 + const offset = options?.offset ?? 0 for await (const entry of iter) { + // Skip by offset + count++ + + if (count <= offset) { + continue + } + // Construct document from entry const doc = await this.constructDocument(entry) diff --git a/src/types.ts b/src/types.ts index fa31097..fefc1f2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -358,6 +358,13 @@ export type ListOptions = Deno.KvListOptions & { */ filter?: (value: T) => boolean + /** + * Number of results to offset by. + * + * If set, the actual limit for the KV.list operation is set equal to offset + limit. + */ + offset?: number + /** Id of document to start from. */ startId?: KvId diff --git a/src/utils.ts b/src/utils.ts index 528d95c..1d861a1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -464,6 +464,14 @@ export function createListSelector( } } +export function createListOptions(options: ListOptions | undefined) { + const limit = options?.limit && options.limit + (options.offset ?? 0) + return { + ...options, + limit, + } +} + /** * Checks whether the specified list options selects all entries or potentially limits the selection. * @@ -479,7 +487,8 @@ export function selectsAll( !options?.endId && !options?.startId && !options?.filter && - !options?.limit + !options?.limit && + !options?.offset ) } diff --git a/tests/collection/properties.test.ts b/tests/collection/properties.test.ts index 0ab85a6..7b60e47 100644 --- a/tests/collection/properties.test.ts +++ b/tests/collection/properties.test.ts @@ -40,7 +40,7 @@ Deno.test("collection - properties", async (t) => { }) }) - await t.step("Should select using pagination", async () => { + await t.step("Should select using cursor pagination", async () => { await useDb(async (db) => { const users = generateUsers(1_000) const cr = await db.users.addMany(users) @@ -66,6 +66,28 @@ Deno.test("collection - properties", async (t) => { }) }) + await t.step("Should select using offset pagination", async () => { + await useDb(async (db) => { + const users = generateUsers(1_000) + const cr = await db.users.addMany(users) + assert(cr.ok) + + const selected: Document[] = [] + const limit = 50 + for (let offset = 0; offset < users.length; offset += limit) { + const { result } = await db.users.getMany({ offset, limit }) + selected.push(...result) + assert(result.length === 50) + } + + assert( + users.every((user) => + selected.some((doc) => doc.value.username === user.username) + ), + ) + }) + }) + await t.step("Should select filtered", async () => { await useDb(async (db) => { const users = generateUsers(10) diff --git a/tests/indexable_collection/properties.test.ts b/tests/indexable_collection/properties.test.ts index 200c938..4c4ab53 100644 --- a/tests/indexable_collection/properties.test.ts +++ b/tests/indexable_collection/properties.test.ts @@ -52,7 +52,7 @@ Deno.test("indexable_collection - properties", async (t) => { }) }) - await t.step("Should select using pagination", async () => { + await t.step("Should select using cursor pagination", async () => { await useDb(async (db) => { const users = generateUsers(1_000) const cr = await db.i_users.addMany(users) @@ -78,6 +78,28 @@ Deno.test("indexable_collection - properties", async (t) => { }) }) + await t.step("Should select using offset pagination", async () => { + await useDb(async (db) => { + const users = generateUsers(1_000) + const cr = await db.i_users.addMany(users) + assert(cr.ok) + + const selected: Document[] = [] + const limit = 50 + for (let offset = 0; offset < users.length; offset += limit) { + const { result } = await db.i_users.getMany({ offset, limit }) + selected.push(...result) + assert(result.length === 50) + } + + assert( + users.every((user) => + selected.some((doc) => doc.value.username === user.username) + ), + ) + }) + }) + await t.step("Should select filtered", async () => { await useDb(async (db) => { const users = generateUsers(10) diff --git a/tests/serialized_collection/properties.test.ts b/tests/serialized_collection/properties.test.ts index 4d1e7a2..ef4b0b7 100644 --- a/tests/serialized_collection/properties.test.ts +++ b/tests/serialized_collection/properties.test.ts @@ -45,7 +45,7 @@ Deno.test("serialized_collection - properties", async (t) => { }) }) - await t.step("Should select using pagination", async () => { + await t.step("Should select using cursor pagination", async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) const cr = await db.s_users.addMany(users) @@ -71,6 +71,28 @@ Deno.test("serialized_collection - properties", async (t) => { }) }) + await t.step("Should select using offset pagination", async () => { + await useDb(async (db) => { + const users = generateLargeUsers(1_000) + const cr = await db.s_users.addMany(users) + assert(cr.ok) + + const selected: Document[] = [] + const limit = 50 + for (let offset = 0; offset < users.length; offset += limit) { + const { result } = await db.s_users.getMany({ offset, limit }) + selected.push(...result) + assert(result.length === 50) + } + + assert( + users.every((user) => + selected.some((doc) => doc.value.username === user.username) + ), + ) + }) + }) + await t.step("Should select filtered", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) diff --git a/tests/serialized_indexable_collection/properties.test.ts b/tests/serialized_indexable_collection/properties.test.ts index e12dc9f..201e93c 100644 --- a/tests/serialized_indexable_collection/properties.test.ts +++ b/tests/serialized_indexable_collection/properties.test.ts @@ -55,7 +55,7 @@ Deno.test("serialized_indexable_collection - properties", async (t) => { }) }) - await t.step("Should select using pagination", async () => { + await t.step("Should select using cursor pagination", async () => { await useDb(async (db) => { const users = generateLargeUsers(1_000) const cr = await db.is_users.addMany(users) @@ -81,6 +81,28 @@ Deno.test("serialized_indexable_collection - properties", async (t) => { }) }) + await t.step("Should select using offset pagination", async () => { + await useDb(async (db) => { + const users = generateLargeUsers(1_000) + const cr = await db.is_users.addMany(users) + assert(cr.ok) + + const selected: Document[] = [] + const limit = 50 + for (let offset = 0; offset < users.length; offset += limit) { + const { result } = await db.is_users.getMany({ offset, limit }) + selected.push(...result) + assert(result.length === 50) + } + + assert( + users.every((user) => + selected.some((doc) => doc.value.username === user.username) + ), + ) + }) + }) + await t.step("Should select filtered", async () => { await useDb(async (db) => { const users = generateLargeUsers(10) From 8a1b621db98ffa4c8f15315efeba8afa215e6d2e Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Thu, 14 Dec 2023 06:27:48 +0100 Subject: [PATCH 2/3] feat(collection): Replaced count options with list options --- src/collection.ts | 5 ++--- src/types.ts | 10 ++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/collection.ts b/src/collection.ts index 915740a..13368f0 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -4,7 +4,6 @@ import type { CollectionKeys, CollectionOptions, CommitResult, - CountOptions, EnqueueOptions, FindManyOptions, FindOptions, @@ -1325,7 +1324,7 @@ export class Collection< * @param options - Count options, optional. * @returns A promise that resolves to a number representing the count. */ - async count(options?: CountOptions>) { + async count(options?: ListOptions>) { // Initiate count result let result = 0 @@ -1363,7 +1362,7 @@ export class Collection< >( index: K, value: CheckKeyOf, - options?: CountOptions>, + options?: ListOptions>, ) { // Serialize and compress index value const serialized = this._serializer.serialize(value) diff --git a/src/types.ts b/src/types.ts index fefc1f2..7c17ddc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -381,10 +381,6 @@ export type AtomicListOptions = & ListOptions & AtomicBatchOptions -export type CountOptions = - & CountAllOptions - & Pick, "filter"> - export type FindOptions = NonNullable[1]> export type FindManyOptions = NonNullable[1]> @@ -428,6 +424,12 @@ export type IdUpsertInput = { update: UpdateData } +/********************/ +/* */ +/* UPSERT TYPES */ +/* */ +/********************/ + export type PrimaryIndexUpsertInput< TInput, TOutput extends KvValue, From 4387a7ec89bc022c00e147dfed66a97db93f5692 Mon Sep 17 00:00:00 2001 From: oliver-oloughlin Date: Thu, 14 Dec 2023 06:35:35 +0100 Subject: [PATCH 3/3] chore(docs): updated readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 75d18fa..0560d4f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ dependencies. It's purpose is to enhance the experience of using Deno's KV store through additional features such as indexing, strongly typed collections and serialization/compression, while maintaining as much of the native functionality -as possible, like watch, atomic operations and queue listeners. +as possible, like atomic operations, real-time watch and queue listeners. _Supported Deno verisons:_ **^1.38.5** @@ -326,8 +326,8 @@ if (result1.ok) { ### update() Update the value of an exisiting document in the KV store. For primitive values, -arrays and built-in objects (Date, RegExp, etc.), this method overwrites the -exisiting data with the new value. For custom objects (Models), this method +arrays and built-in objects (Array, Date, RegExp, etc.), this method overwrites +the exisiting data with the new value. For plain object values, this method performs a partial update, merging the new value with the existing data using deep merge by default, or optionally using shallow merge. Upon completion, a CommitResult object will be returned with the document id, versionstamp and ok