Fix use of small limit on collections request (#2705)

* Fix use of small limit on collections request

---------

Co-authored-by: Stuart Corbishley <corbish@gmail.com>

CHANGELOG.md

@@ -32,6 +32,9 @@ and this project adheres to
 
 ### Fixed
 
+
+- Fix cursor for small limit on collections request
+  [#2683](https://github.com/OpenFn/lightning/issues/2683)
 - Disable save and run actions on deleted workflows
   [#2170](https://github.com/OpenFn/lightning/issues/2170)
 - Distinguish active and inactive sort arrows in projects overview table

lib/lightning_web/controllers/collections_controller.ex

@@ -107,8 +107,12 @@ defmodule LightningWeb.CollectionsController do
     end
   end
 
+  # Streams records from database without depending on holding a transaction from database pool.
+  # It streams one more than the limit to allow determining if there are more items for the response cursor.
   defp stream_all_in_chunks(collection, %{limit: limit} = filters, key_pattern)
        when limit <= @max_database_limit do
+    filters = Map.put(filters, :limit, limit + 1)
+
     Collections.get_all(collection, filters, key_pattern)
   end
 
@@ -117,7 +121,6 @@ defmodule LightningWeb.CollectionsController do
          %{cursor: initial_cursor} = filters,
          key_pattern
        ) do
-    # returns one more than the limit to determine if there are more items for the cursor
     filters = Map.put(filters, :limit, @max_database_limit + 1)
 
     Stream.unfold(initial_cursor, fn cursor ->

test/lightning_web/collections_controller_test.exs

@@ -704,6 +704,30 @@ defmodule LightningWeb.API.CollectionsControllerTest do
 
       assert conn.state == :chunked
 
+      assert json_response(conn, 200) == %{
+               "items" => expected_items,
+               "cursor" =>
+                 Base.encode64(DateTime.to_iso8601(last_item.inserted_at))
+             }
+
+      # Test for the existence of a cursor when the limit is less than the
+      # database limit
+      half_limit = (@max_database_limit / 2) |> floor()
+
+      expected_items =
+        all_items |> Enum.take(half_limit)
+
+      last_item =
+        collection.items
+        |> Enum.take(half_limit)
+        |> List.last()
+
+      conn =
+        conn
+        |> get(~p"/collections/#{collection.name}",
+          limit: half_limit
+        )
+
       assert json_response(conn, 200) == %{
                "items" => expected_items,
                "cursor" =>