Merge pull request #148 from oliver-oloughlin/feature/offset-pagination

Feature/offset pagination

README.md

@@ -4,7 +4,7 @@
 dependencies. It's purpose is to enhance the experience of using Deno's KV store
 through additional features such as indexing, strongly typed collections and
 serialization/compression, while maintaining as much of the native functionality
-as possible, like watch, atomic operations and queue listeners.
+as possible, like atomic operations, real-time watch and queue listeners.
 
 _Supported Deno verisons:_ **^1.38.5**
 
@@ -326,8 +326,8 @@ if (result1.ok) {
 ### update()
 
 Update the value of an exisiting document in the KV store. For primitive values,
-arrays and built-in objects (Date, RegExp, etc.), this method overwrites the
-exisiting data with the new value. For custom objects (Models), this method
+arrays and built-in objects (Array, Date, RegExp, etc.), this method overwrites
+the exisiting data with the new value. For plain object values, this method
 performs a partial update, merging the new value with the existing data using
 deep merge by default, or optionally using shallow merge. Upon completion, a
 CommitResult object will be returned with the document id, versionstamp and ok

src/collection.ts

@@ -4,7 +4,6 @@ import type {
   CollectionKeys,
   CollectionOptions,
   CommitResult,
-  CountOptions,
   EnqueueOptions,
   FindManyOptions,
   FindOptions,
@@ -44,6 +43,7 @@ import {
   checkIndices,
   compress,
   createHandlerId,
+  createListOptions,
   createListSelector,
   decompress,
   deleteIndices,
@@ -441,10 +441,21 @@ export class Collection<
     const selector = createListSelector(keyPrefix, options)
 
     // Create hsitory entries iterator
-    const iter = this.kv.list<HistoryEntry<TOutput>>(selector, options)
+    const listOptions = createListOptions(options)
+    const iter = this.kv.list<HistoryEntry<TOutput>>(selector, listOptions)
 
     // Collect history entries
+    let count = 0
+    const offset = options?.offset ?? 0
     for await (const { value, key } of iter) {
+      // Skip by offset
+      count++
+
+      if (count <= offset) {
+        continue
+      }
+
+      // Cast history entry
       let historyEntry: HistoryEntry<TOutput> = value
 
       // Handle serialized entries
@@ -1052,6 +1063,7 @@ export class Collection<
     if (selectsAll(options)) {
       // Create list iterator and empty keys list, init atomic operation
       const iter = this.kv.list({ prefix: this._keys.base }, options)
+
       const keys: Deno.KvKey[] = []
       const atomic = new AtomicWrapper(this.kv, options?.atomicBatchSize)
 
@@ -1312,7 +1324,7 @@ export class Collection<
    * @param options - Count options, optional.
    * @returns A promise that resolves to a number representing the count.
    */
-  async count(options?: CountOptions<Document<TOutput>>) {
+  async count(options?: ListOptions<Document<TOutput>>) {
     // Initiate count result
     let result = 0
 
@@ -1350,7 +1362,7 @@ export class Collection<
   >(
     index: K,
     value: CheckKeyOf<K, TOutput>,
-    options?: CountOptions<Document<TOutput>>,
+    options?: ListOptions<Document<TOutput>>,
   ) {
     // Serialize and compress index value
     const serialized = this._serializer.serialize(value)
@@ -1926,15 +1938,25 @@ export class Collection<
   ) {
     // Create list iterator with given options
     const selector = createListSelector(prefixKey, options)
-    const iter = this.kv.list<KvValue>(selector, options)
+    const listOptions = createListOptions(options)
+    const iter = this.kv.list<KvValue>(selector, listOptions)
 
     // Initiate lists
     const docs: Document<TOutput>[] = []
     const result: Awaited<T>[] = []
     const errors: unknown[] = []
 
     // Loop over each document entry
+    let count = 0
+    const offset = options?.offset ?? 0
     for await (const entry of iter) {
+      // Skip by offset
+      count++
+
+      if (count <= offset) {
+        continue
+      }
+
       // Construct document from entry
       const doc = await this.constructDocument(entry)
 

src/types.ts

@@ -358,6 +358,13 @@ export type ListOptions<T> = Deno.KvListOptions & {
    */
   filter?: (value: T) => boolean
 
+  /**
+   * Number of results to offset by.
+   *
+   * If set, the actual limit for the KV.list operation is set equal to offset + limit.
+   */
+  offset?: number
+
   /** Id of document to start from. */
   startId?: KvId
 
@@ -374,10 +381,6 @@ export type AtomicListOptions<T> =
   & ListOptions<T>
   & AtomicBatchOptions
 
-export type CountOptions<T> =
-  & CountAllOptions
-  & Pick<ListOptions<T>, "filter">
-
 export type FindOptions = NonNullable<Parameters<Deno.Kv["get"]>[1]>
 
 export type FindManyOptions = NonNullable<Parameters<Deno.Kv["getMany"]>[1]>
@@ -421,6 +424,12 @@ export type IdUpsertInput<TInput, TOutput extends KvValue> = {
   update: UpdateData<TOutput>
 }
 
+/********************/
+/*                  */
+/*   UPSERT TYPES   */
+/*                  */
+/********************/
+
 export type PrimaryIndexUpsertInput<
   TInput,
   TOutput extends KvValue,

src/utils.ts

@@ -464,6 +464,14 @@ export function createListSelector<T>(
   }
 }
 
+export function createListOptions<T>(options: ListOptions<T> | undefined) {
+  const limit = options?.limit && options.limit + (options.offset ?? 0)
+  return {
+    ...options,
+    limit,
+  }
+}
+
 /**
  *  Checks whether the specified list options selects all entries or potentially limits the selection.
  *
@@ -479,7 +487,8 @@ export function selectsAll<T>(
     !options?.endId &&
     !options?.startId &&
     !options?.filter &&
-    !options?.limit
+    !options?.limit &&
+    !options?.offset
   )
 }
 

tests/collection/properties.test.ts

@@ -40,7 +40,7 @@ Deno.test("collection - properties", async (t) => {
     })
   })
 
-  await t.step("Should select using pagination", async () => {
+  await t.step("Should select using cursor pagination", async () => {
     await useDb(async (db) => {
       const users = generateUsers(1_000)
       const cr = await db.users.addMany(users)
@@ -66,6 +66,28 @@ Deno.test("collection - properties", async (t) => {
     })
   })
 
+  await t.step("Should select using offset pagination", async () => {
+    await useDb(async (db) => {
+      const users = generateUsers(1_000)
+      const cr = await db.users.addMany(users)
+      assert(cr.ok)
+
+      const selected: Document<User>[] = []
+      const limit = 50
+      for (let offset = 0; offset < users.length; offset += limit) {
+        const { result } = await db.users.getMany({ offset, limit })
+        selected.push(...result)
+        assert(result.length === 50)
+      }
+
+      assert(
+        users.every((user) =>
+          selected.some((doc) => doc.value.username === user.username)
+        ),
+      )
+    })
+  })
+
   await t.step("Should select filtered", async () => {
     await useDb(async (db) => {
       const users = generateUsers(10)

tests/indexable_collection/properties.test.ts

@@ -52,7 +52,7 @@ Deno.test("indexable_collection - properties", async (t) => {
     })
   })
 
-  await t.step("Should select using pagination", async () => {
+  await t.step("Should select using cursor pagination", async () => {
     await useDb(async (db) => {
       const users = generateUsers(1_000)
       const cr = await db.i_users.addMany(users)
@@ -78,6 +78,28 @@ Deno.test("indexable_collection - properties", async (t) => {
     })
   })
 
+  await t.step("Should select using offset pagination", async () => {
+    await useDb(async (db) => {
+      const users = generateUsers(1_000)
+      const cr = await db.i_users.addMany(users)
+      assert(cr.ok)
+
+      const selected: Document<User>[] = []
+      const limit = 50
+      for (let offset = 0; offset < users.length; offset += limit) {
+        const { result } = await db.i_users.getMany({ offset, limit })
+        selected.push(...result)
+        assert(result.length === 50)
+      }
+
+      assert(
+        users.every((user) =>
+          selected.some((doc) => doc.value.username === user.username)
+        ),
+      )
+    })
+  })
+
   await t.step("Should select filtered", async () => {
     await useDb(async (db) => {
       const users = generateUsers(10)

tests/serialized_collection/properties.test.ts

@@ -45,7 +45,7 @@ Deno.test("serialized_collection - properties", async (t) => {
     })
   })
 
-  await t.step("Should select using pagination", async () => {
+  await t.step("Should select using cursor pagination", async () => {
     await useDb(async (db) => {
       const users = generateLargeUsers(1_000)
       const cr = await db.s_users.addMany(users)
@@ -71,6 +71,28 @@ Deno.test("serialized_collection - properties", async (t) => {
     })
   })
 
+  await t.step("Should select using offset pagination", async () => {
+    await useDb(async (db) => {
+      const users = generateLargeUsers(1_000)
+      const cr = await db.s_users.addMany(users)
+      assert(cr.ok)
+
+      const selected: Document<User>[] = []
+      const limit = 50
+      for (let offset = 0; offset < users.length; offset += limit) {
+        const { result } = await db.s_users.getMany({ offset, limit })
+        selected.push(...result)
+        assert(result.length === 50)
+      }
+
+      assert(
+        users.every((user) =>
+          selected.some((doc) => doc.value.username === user.username)
+        ),
+      )
+    })
+  })
+
   await t.step("Should select filtered", async () => {
     await useDb(async (db) => {
       const users = generateLargeUsers(10)

tests/serialized_indexable_collection/properties.test.ts

@@ -55,7 +55,7 @@ Deno.test("serialized_indexable_collection - properties", async (t) => {
     })
   })
 
-  await t.step("Should select using pagination", async () => {
+  await t.step("Should select using cursor pagination", async () => {
     await useDb(async (db) => {
       const users = generateLargeUsers(1_000)
       const cr = await db.is_users.addMany(users)
@@ -81,6 +81,28 @@ Deno.test("serialized_indexable_collection - properties", async (t) => {
     })
   })
 
+  await t.step("Should select using offset pagination", async () => {
+    await useDb(async (db) => {
+      const users = generateLargeUsers(1_000)
+      const cr = await db.is_users.addMany(users)
+      assert(cr.ok)
+
+      const selected: Document<User>[] = []
+      const limit = 50
+      for (let offset = 0; offset < users.length; offset += limit) {
+        const { result } = await db.is_users.getMany({ offset, limit })
+        selected.push(...result)
+        assert(result.length === 50)
+      }
+
+      assert(
+        users.every((user) =>
+          selected.some((doc) => doc.value.username === user.username)
+        ),
+      )
+    })
+  })
+
   await t.step("Should select filtered", async () => {
     await useDb(async (db) => {
       const users = generateLargeUsers(10)