Dynamic Pagination of Issues & Pull Requests

lazy_github/lib/github/issues.py

@@ -4,16 +4,20 @@
 from lazy_github.lib.context import LazyGithubContext, github_headers
 from lazy_github.models.github import Issue, IssueComment, PartialPullRequest, Repository
 
+DEFAULT_PAGE_SIZE = 30
+
 
 class UpdateIssuePayload(TypedDict):
     title: str | None
     body: str | None
     state: str | None
 
 
-async def list_issues(repo: Repository, state: IssueStateFilter, owner: IssueOwnerFilter) -> list[Issue]:
+async def list_issues(
+    repo: Repository, state: IssueStateFilter, owner: IssueOwnerFilter, page: int = 1, per_page: int = DEFAULT_PAGE_SIZE
+) -> list[Issue]:
     """Fetch issues (included pull requests) from the repo matching the state/owner filters"""
-    query_params = {"state": str(state).lower()}
+    query_params = {"state": str(state).lower(), "page": page, "per_page": per_page}
     if owner == IssueOwnerFilter.MINE:
         user = await LazyGithubContext.client.user()
         query_params["creator"] = user.login

lazy_github/ui/widgets/actions.py

@@ -1,13 +1,14 @@
 from textual.app import ComposeResult
+from textual.widgets import DataTable
 
 from lazy_github.lib.messages import RepoSelected
-from lazy_github.ui.widgets.common import LazyGithubContainer, LazyGithubDataTable
+from lazy_github.ui.widgets.common import LazyGithubContainer
 
 
 class ActionsContainer(LazyGithubContainer):
     def compose(self) -> ComposeResult:
         self.border_title = "[4] Actions"
-        yield LazyGithubDataTable(id="actions_table")
+        yield DataTable(id="actions_table")
 
     async def on_repo_selected(self, message: RepoSelected) -> None:
         # TODO: Load the actions for the selected repo

lazy_github/ui/widgets/common.py

@@ -1,32 +1,37 @@
-from typing import Iterable
+from typing import Awaitable, Callable, Iterable
 
-from textual import on
+from textual import on, work
 from textual.app import ComposeResult
 from textual.containers import Container, Vertical
 from textual.events import Blur
 from textual.widgets import DataTable, Input
-from textual.widgets.data_table import CellType
 
+TABLE_POPULATION_FUNCTION = Callable[[int, int], Awaitable[list[tuple[str | int, ...]]]]
 
-class LazyGithubDataTable(DataTable):
+
+class _VimLikeDataTable(DataTable):
     "An data table for LazyGithub that provides some more vim-like bindings"
 
     BINDINGS = [
         # Add space as an additional selection key
         ("space", "select"),
         # Add some vim bindings
         ("j", "cursor_down"),
+        ("J", "page_down"),
         ("k", "cursor_up"),
+        ("K", "page_up"),
         ("l", "scroll_right"),
+        ("L", "page_right"),
         ("h", "scroll_left"),
+        ("H", "page_left"),
         ("g", "scroll_top"),
         ("G", "scroll_bottom"),
         ("^", "page_left"),
         ("$", "page_right"),
     ]
 
 
-class LazyGithubDataTableSearchInput(Input):
+class SearchableDataTableSearchInput(Input):
     def _on_blur(self, event: Blur) -> None:
         if not self.value.strip():
             # If we lose focus and the content is empty, hide it
@@ -35,11 +40,11 @@ def _on_blur(self, event: Blur) -> None:
         return super()._on_blur(event)
 
 
-class SearchableLazyGithubDataTable(Vertical):
+class SearchableDataTable(Vertical):
     BINDINGS = [("/", "focus_search", "Search")]
 
     DEFAULT_CSS = """
-    LazyGithubDataTableSearchInput {
+    SearchableDataTableSearchInput {
         margin-bottom: 1;
     }
     """
@@ -48,13 +53,13 @@ def __init__(
         self, table_id: str, search_input_id: str, sort_key: str, *args, reverse_sort: bool = False, **kwargs
     ) -> None:
         super().__init__(*args, **kwargs)
-        self.table = LazyGithubDataTable(id=table_id)
-        self.search_input = LazyGithubDataTableSearchInput(placeholder="Search...", id=search_input_id)
+        self.table = _VimLikeDataTable(id=table_id)
+        self.search_input = SearchableDataTableSearchInput(placeholder="Search...", id=search_input_id)
         self.search_input.display = False
         self.search_input.can_focus = False
         self.sort_key = sort_key
         self.reverse_sort = reverse_sort
-        self._rows_cache = []
+        self._rows_cache: list[tuple[str | int, ...]] = []
 
     def sort(self):
         self.table.sort(self.sort_key, reverse=self.reverse_sort)
@@ -69,29 +74,101 @@ async def action_focus_search(self) -> None:
         self.search_input.focus()
 
     def clear_rows(self):
+        """Removes all rows currently displayed and tracked in this table"""
+        self._rows_cache = []
         self.table.clear()
 
-    def add_rows(self, rows: Iterable[Iterable[CellType]]) -> None:
-        self._set_rows(rows)
+    def append_rows(self, rows: Iterable[tuple[str | int, ...]]) -> None:
+        """Add new rows to the currently displayed table and cache"""
+        self._rows_cache.extend(rows)
+        # TODO: Should this actually call handle_submitted_search so that new rows which don't match criteria aren't
+        # shown?
+        self.table.add_rows(rows)
+        self.sort()
+
+    def set_rows(self, rows: list[tuple[str | int, ...]]) -> None:
+        """Override the set of rows contained in this table. This will remove any existing rows"""
         self._rows_cache = rows
+        self.change_displayed_rows(rows)
 
-    def _set_rows(self, rows: Iterable[Iterable[CellType]]) -> None:
+    def change_displayed_rows(self, rows: Iterable[tuple[str | int, ...]]) -> None:
+        """Change which rows are currently displayed in the table"""
         self.table.clear()
         self.table.add_rows(rows)
         self.sort()
 
     @on(Input.Submitted)
     async def handle_submitted_search(self) -> None:
         search_query = self.search_input.value.strip().lower()
-        filtered_rows: Iterable[Iterable] = []
+        filtered_rows: Iterable[tuple[str | int, ...]] = []
         for row in self._rows_cache:
             if search_query in str(row).lower() or not search_query:
                 filtered_rows.append(row)
 
-        self._set_rows(filtered_rows)
+        self.change_displayed_rows(filtered_rows)
         self.table.focus()
 
 
+class LazilyLoadedDataTable(SearchableDataTable):
+    """A searchable data table that is lazily loaded when you have viewed the currently loaded data"""
+
+    def __init__(
+        self,
+        table_id: str,
+        search_input_id: str,
+        sort_key: str,
+        load_function: TABLE_POPULATION_FUNCTION | None,
+        batch_size: int,
+        *args,
+        load_more_data_buffer: int = 5,
+        reverse_sort: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(table_id, search_input_id, sort_key, *args, reverse_sort=reverse_sort, **kwargs)
+        self.load_function = load_function
+        self.batch_size = batch_size
+        self.load_more_data_buffer = load_more_data_buffer
+        self.current_batch = 0
+
+        # We initialize this to true and set it to false later if we believe we've run out of data to load from the load
+        # function.
+        self.can_load_more = True
+
+    async def initialize(self) -> None:
+        if not self.load_function:
+            return
+
+        initial_data = await self.load_function(self.batch_size, self.current_batch)
+        self.set_rows(initial_data)
+
+        if len(initial_data) == 0:
+            self.can_load_more = False
+
+    def change_load_function(self, new_load_function: TABLE_POPULATION_FUNCTION | None) -> None:
+        self.load_function = new_load_function
+
+    @work(exclusive=True)
+    async def load_more_data(self, row_highlighted: DataTable.RowHighlighted) -> None:
+        # TODO: This should probably lock instead of relying on exclusive=True
+        rows_remaining = len(self._rows_cache) - row_highlighted.cursor_row
+        if not (self.can_load_more and self.load_function):
+            return
+
+        if rows_remaining > self.load_more_data_buffer:
+            return
+
+        additional_data = await self.load_function(self.batch_size, self.current_batch + 1)
+        self.current_batch += 1
+        if len(additional_data) == 0:
+            self.can_load_more = False
+
+        self.append_rows(additional_data)
+
+    @on(DataTable.RowHighlighted)
+    async def check_highlighted_row_boundary(self, row_highlighted: DataTable.RowHighlighted) -> None:
+        self.load_more_data(row_highlighted)
+
+
 class LazyGithubContainer(Container):
     """
     Base container class for focusible containers within the Lazy Github UI

lazy_github/ui/widgets/issues.py

@@ -4,21 +4,25 @@
 from textual.app import ComposeResult
 from textual.containers import ScrollableContainer, VerticalScroll
 from textual.coordinate import Coordinate
-from textual.widgets import Label, Markdown, Rule, TabPane
+from textual.widgets import DataTable, Label, Markdown, Rule, TabPane
 from textual.widgets.data_table import CellDoesNotExist
 
 from lazy_github.lib.context import LazyGithubContext
-from lazy_github.lib.github.issues import get_comments
+from lazy_github.lib.github.issues import get_comments, list_issues
 from lazy_github.lib.messages import IssuesAndPullRequestsFetched, IssueSelected
 from lazy_github.lib.utils import link
-from lazy_github.models.github import Issue, IssueState
+from lazy_github.models.github import Issue, IssueState, PartialPullRequest
 from lazy_github.ui.screens.edit_issue import EditIssueModal
 from lazy_github.ui.screens.new_issue import NewIssueModal
 from lazy_github.ui.widgets.command_log import log_event
-from lazy_github.ui.widgets.common import LazyGithubContainer, LazyGithubDataTable, SearchableLazyGithubDataTable
+from lazy_github.ui.widgets.common import LazilyLoadedDataTable, LazyGithubContainer
 from lazy_github.ui.widgets.conversations import IssueCommentContainer
 
 
+def issue_to_cell(issue: Issue) -> tuple[str | int, ...]:
+    return (issue.number, str(issue.state), issue.user.login, issue.title)
+
+
 class IssuesContainer(LazyGithubContainer):
     BINDINGS = [
         ("E", "edit_issue", "Edit Issue"),
@@ -32,21 +36,40 @@ class IssuesContainer(LazyGithubContainer):
 
     def compose(self) -> ComposeResult:
         self.border_title = "[3] Issues"
-        yield SearchableLazyGithubDataTable(
+        yield LazilyLoadedDataTable(
             id="searchable_issues_table",
             table_id="issues_table",
             search_input_id="issues_search",
             sort_key="number",
+            load_function=None,
+            batch_size=30,
             reverse_sort=True,
         )
 
+    async def fetch_more_issues(self, batch_size: int, batch_to_fetch: int) -> list[tuple[str | int, ...]]:
+        if not LazyGithubContext.current_repo:
+            return []
+
+        next_page = await list_issues(
+            LazyGithubContext.current_repo,
+            LazyGithubContext.config.issues.state_filter,
+            LazyGithubContext.config.issues.owner_filter,
+            page=batch_to_fetch,
+            per_page=batch_size,
+        )
+
+        new_issues = [i for i in next_page if not isinstance(i, PartialPullRequest)]
+        self.issues.update({i.number: i for i in new_issues})
+
+        return [issue_to_cell(i) for i in new_issues]
+
     @property
-    def searchable_table(self) -> SearchableLazyGithubDataTable:
-        return self.query_one("#searchable_issues_table", SearchableLazyGithubDataTable)
+    def searchable_table(self) -> LazilyLoadedDataTable:
+        return self.query_one("#searchable_issues_table", LazilyLoadedDataTable)
 
     @property
-    def table(self) -> LazyGithubDataTable:
-        return self.query_one("#issues_table", LazyGithubDataTable)
+    def table(self) -> DataTable:
+        return self.query_one("#issues_table", DataTable)
 
     def on_mount(self) -> None:
         self.table.cursor_type = "row"
@@ -68,7 +91,10 @@ async def on_issues_and_pull_requests_fetched(self, message: IssuesAndPullReques
         for issue in message.issues:
             self.issues[issue.number] = issue
             rows.append((issue.number, issue.state, issue.user.login, issue.title))
-        self.searchable_table.add_rows(rows)
+        self.searchable_table.set_rows(rows)
+        self.searchable_table.change_load_function(self.fetch_more_issues)
+        self.searchable_table.can_load_more = True
+        self.searchable_table.current_batch = 1
 
     async def get_selected_issue(self) -> Issue:
         pr_number_coord = Coordinate(self.table.cursor_row, self.number_column_index)
@@ -88,7 +114,7 @@ async def action_new_issue(self) -> None:
         else:
             self.notify("No repository currently selected", severity="error")
 
-    @on(LazyGithubDataTable.RowSelected, "#issues_table")
+    @on(DataTable.RowSelected, "#issues_table")
     async def issue_selected(self) -> None:
         issue = await self.get_selected_issue()
         log_event(f"Selected Issue: #{issue.number}")