Fix use of small limit on collections request (#2705)

* Fix use of small limit on collections request --------- Co-authored-by: Stuart Corbishley <[email protected]>
OpenFn · Nov 22, 2024 · 23307f9 · 23307f9
1 parent 5c52962
commit 23307f9
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -32,6 +32,9 @@ and this project adheres to
 
 ### Fixed
 
+
+- Fix cursor for small limit on collections request
+  [#2683](https://github.com/OpenFn/lightning/issues/2683)
 - Disable save and run actions on deleted workflows
   [#2170](https://github.com/OpenFn/lightning/issues/2170)
 - Distinguish active and inactive sort arrows in projects overview table

diff --git a/lib/lightning_web/controllers/collections_controller.ex b/lib/lightning_web/controllers/collections_controller.ex
@@ -107,8 +107,12 @@ defmodule LightningWeb.CollectionsController do
     end
   end
 
+  # Streams records from database without depending on holding a transaction from database pool.
+  # It streams one more than the limit to allow determining if there are more items for the response cursor.
   defp stream_all_in_chunks(collection, %{limit: limit} = filters, key_pattern)
        when limit <= @max_database_limit do
+    filters = Map.put(filters, :limit, limit + 1)
+
     Collections.get_all(collection, filters, key_pattern)
   end
 
@@ -117,7 +121,6 @@ defmodule LightningWeb.CollectionsController do
          %{cursor: initial_cursor} = filters,
          key_pattern
        ) do
-    # returns one more than the limit to determine if there are more items for the cursor
     filters = Map.put(filters, :limit, @max_database_limit + 1)
 
     Stream.unfold(initial_cursor, fn cursor ->

diff --git a/test/lightning_web/collections_controller_test.exs b/test/lightning_web/collections_controller_test.exs
@@ -704,6 +704,30 @@ defmodule LightningWeb.API.CollectionsControllerTest do
 
       assert conn.state == :chunked
 
+      assert json_response(conn, 200) == %{
+               "items" => expected_items,
+               "cursor" =>
+                 Base.encode64(DateTime.to_iso8601(last_item.inserted_at))
+             }
+
+      # Test for the existence of a cursor when the limit is less than the
+      # database limit
+      half_limit = (@max_database_limit / 2) |> floor()
+
+      expected_items =
+        all_items |> Enum.take(half_limit)
+
+      last_item =
+        collection.items
+        |> Enum.take(half_limit)
+        |> List.last()
+
+      conn =
+        conn
+        |> get(~p"/collections/#{collection.name}",
+          limit: half_limit
+        )
+
       assert json_response(conn, 200) == %{
                "items" => expected_items,
                "cursor" =>