From 23307f97a363ad2ec589d43248026a0aca863e08 Mon Sep 17 00:00:00 2001
From: Rogerio Pontual <44991200+jyeshe@users.noreply.github.com>
Date: Fri, 22 Nov 2024 08:25:25 +0100
Subject: [PATCH] Fix use of small limit on collections request (#2705)

* Fix use of small limit on collections request

---------

Co-authored-by: Stuart Corbishley <corbish@gmail.com>
---
 CHANGELOG.md                                  |  3 +++
 .../controllers/collections_controller.ex     |  5 +++-
 .../collections_controller_test.exs           | 24 +++++++++++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 45c679024f..dac1fc7000 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,9 @@ and this project adheres to
 
 ### Fixed
 
+
+- Fix cursor for small limit on collections request
+  [#2683](https://github.com/OpenFn/lightning/issues/2683)
 - Disable save and run actions on deleted workflows
   [#2170](https://github.com/OpenFn/lightning/issues/2170)
 - Distinguish active and inactive sort arrows in projects overview table
diff --git a/lib/lightning_web/controllers/collections_controller.ex b/lib/lightning_web/controllers/collections_controller.ex
index 0fb944a82b..e528864434 100644
--- a/lib/lightning_web/controllers/collections_controller.ex
+++ b/lib/lightning_web/controllers/collections_controller.ex
@@ -107,8 +107,12 @@ defmodule LightningWeb.CollectionsController do
     end
   end
 
+  # Streams records from database without depending on holding a transaction from database pool.
+  # It streams one more than the limit to allow determining if there are more items for the response cursor.
   defp stream_all_in_chunks(collection, %{limit: limit} = filters, key_pattern)
        when limit <= @max_database_limit do
+    filters = Map.put(filters, :limit, limit + 1)
+
     Collections.get_all(collection, filters, key_pattern)
   end
 
@@ -117,7 +121,6 @@ defmodule LightningWeb.CollectionsController do
          %{cursor: initial_cursor} = filters,
          key_pattern
        ) do
-    # returns one more than the limit to determine if there are more items for the cursor
     filters = Map.put(filters, :limit, @max_database_limit + 1)
 
     Stream.unfold(initial_cursor, fn cursor ->
diff --git a/test/lightning_web/collections_controller_test.exs b/test/lightning_web/collections_controller_test.exs
index 82b45f674b..fc21a28bd1 100644
--- a/test/lightning_web/collections_controller_test.exs
+++ b/test/lightning_web/collections_controller_test.exs
@@ -704,6 +704,30 @@ defmodule LightningWeb.API.CollectionsControllerTest do
 
       assert conn.state == :chunked
 
+      assert json_response(conn, 200) == %{
+               "items" => expected_items,
+               "cursor" =>
+                 Base.encode64(DateTime.to_iso8601(last_item.inserted_at))
+             }
+
+      # Test for the existence of a cursor when the limit is less than the
+      # database limit
+      half_limit = (@max_database_limit / 2) |> floor()
+
+      expected_items =
+        all_items |> Enum.take(half_limit)
+
+      last_item =
+        collection.items
+        |> Enum.take(half_limit)
+        |> List.last()
+
+      conn =
+        conn
+        |> get(~p"/collections/#{collection.name}",
+          limit: half_limit
+        )
+
       assert json_response(conn, 200) == %{
                "items" => expected_items,
                "cursor" =>