fix(market): agent pagination and search errors (#8336)

* fix(market): agent pagination and search errors * fix(frontend): search was not paginated * fix: linting * feat(market): linting ci * fix(ci): branch limit name
Significant-Gravitas · Oct 16, 2024 · d6d2820 · d6d2820
1 parent c029fde
commit d6d2820
Show file tree

Hide file tree

Showing 15 changed files with 325 additions and 93 deletions.
diff --git a/.github/workflows/platform-market-ci.yml b/.github/workflows/platform-market-ci.yml
@@ -0,0 +1,125 @@
+name: AutoGPT Platform - Backend CI
+
+on:
+  push:
+    branches: [master, dev, ci-test*]
+    paths:
+      - ".github/workflows/platform-market-ci.yml"
+      - "autogpt_platform/market/**"
+  pull_request:
+    branches: [master, dev, release-*]
+    paths:
+      - ".github/workflows/platform-market-ci.yml"
+      - "autogpt_platform/market/**"
+
+concurrency:
+  group: ${{ format('backend-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
+  cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}
+
+defaults:
+  run:
+    shell: bash
+    working-directory: autogpt_platform/market
+
+jobs:
+  test:
+    permissions:
+      contents: read
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Supabase
+        uses: supabase/setup-cli@v1
+        with:
+          version: latest
+
+      - id: get_date
+        name: Get date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Set up Python dependency cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pypoetry
+          key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/market/poetry.lock') }}
+
+      - name: Install Poetry (Unix)
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+
+          if [ "${{ runner.os }}" = "macOS" ]; then
+            PATH="$HOME/.local/bin:$PATH"
+            echo "$HOME/.local/bin" >> $GITHUB_PATH
+          fi
+
+      - name: Install Python dependencies
+        run: poetry install
+
+      - name: Generate Prisma Client
+        run: poetry run prisma generate
+
+      - id: supabase
+        name: Start Supabase
+        working-directory: .
+        run: |
+          supabase init
+          supabase start --exclude postgres-meta,realtime,storage-api,imgproxy,inbucket,studio,edge-runtime,logflare,vector,supavisor
+          supabase status -o env | sed 's/="/=/; s/"$//' >> $GITHUB_OUTPUT
+        # outputs:
+        # DB_URL, API_URL, GRAPHQL_URL, ANON_KEY, SERVICE_ROLE_KEY, JWT_SECRET
+
+      - name: Run Database Migrations
+        run: poetry run prisma migrate dev --name updates
+        env:
+          DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
+
+      - id: lint
+        name: Run Linter
+        run: poetry run lint
+
+      # Tests comment out because they do not work with prisma mock, nor have they been updated since they were created
+      # - name: Run pytest with coverage
+      #   run: |
+      #     if [[ "${{ runner.debug }}" == "1" ]]; then
+      #       poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG test
+      #     else
+      #       poetry run pytest -s -vv test
+      #     fi
+      #   if: success() || (failure() && steps.lint.outcome == 'failure')
+      #   env:
+      #     LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
+      #     DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
+      #     SUPABASE_URL: ${{ steps.supabase.outputs.API_URL }}
+      #     SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.SERVICE_ROLE_KEY }}
+      #     SUPABASE_JWT_SECRET: ${{ steps.supabase.outputs.JWT_SECRET }}
+      #     REDIS_HOST: 'localhost'
+      #     REDIS_PORT: '6379'
+          # REDIS_PASSWORD: 'testpassword'
+
+    env:
+      CI: true
+      PLAIN_OUTPUT: True
+      RUN_ENV: local
+      PORT: 8080
+
+      # - name: Upload coverage reports to Codecov
+      #   uses: codecov/codecov-action@v4
+      #   with:
+      #     token: ${{ secrets.CODECOV_TOKEN }}
+      #     flags: backend,${{ runner.os }}
diff --git a/autogpt_platform/frontend/src/app/admin/marketplace/page.tsx b/autogpt_platform/frontend/src/app/admin/marketplace/page.tsx
@@ -10,7 +10,7 @@ async function AdminMarketplace() {
 
   return (
     <>
-      <AdminMarketplaceAgentList agents={reviewableAgents.agents} />
+      <AdminMarketplaceAgentList agents={reviewableAgents.items} />
       <Separator className="my-4" />
       <AdminFeaturedAgentsControl className="mt-4" />
     </>

diff --git a/autogpt_platform/frontend/src/app/marketplace/page.tsx b/autogpt_platform/frontend/src/app/marketplace/page.tsx
@@ -6,7 +6,6 @@ import { Input } from "@/components/ui/input";
 import { Button } from "@/components/ui/button";
 import MarketplaceAPI, {
   AgentResponse,
-  AgentListResponse,
   AgentWithRank,
 } from "@/lib/marketplace-api";
 import {
@@ -192,17 +191,19 @@ const Marketplace: React.FC = () => {
   const [searchResults, setSearchResults] = useState<Agent[]>([]);
   const [featuredAgents, setFeaturedAgents] = useState<Agent[]>([]);
   const [topAgents, setTopAgents] = useState<Agent[]>([]);
-  const [page, setPage] = useState(1);
-  const [totalPages, setTotalPages] = useState(1);
   const [isLoading, setIsLoading] = useState(false);
+  const [topAgentsPage, setTopAgentsPage] = useState(1);
+  const [searchPage, setSearchPage] = useState(1);
+  const [topAgentsTotalPages, setTopAgentsTotalPages] = useState(1);
+  const [searchTotalPages, setSearchTotalPages] = useState(1);
 
   const fetchTopAgents = useCallback(
     async (currentPage: number) => {
       setIsLoading(true);
       try {
         const response = await api.getTopDownloadedAgents(currentPage, 9);
-        setTopAgents(response.agents);
-        setTotalPages(response.total_pages);
+        setTopAgents(response.items);
+        setTopAgentsTotalPages(response.total_pages);
       } catch (error) {
         console.error("Error fetching top agents:", error);
       } finally {
@@ -215,19 +216,20 @@ const Marketplace: React.FC = () => {
   const fetchFeaturedAgents = useCallback(async () => {
     try {
       const featured = await api.getFeaturedAgents();
-      setFeaturedAgents(featured.agents);
+      setFeaturedAgents(featured.items);
     } catch (error) {
       console.error("Error fetching featured agents:", error);
     }
   }, [api]);
 
   const searchAgents = useCallback(
-    async (searchTerm: string) => {
+    async (searchTerm: string, currentPage: number) => {
       setIsLoading(true);
       try {
-        const response = await api.searchAgents(searchTerm, 1, 30);
-        const filteredAgents = response.filter((agent) => agent.rank > 0);
+        const response = await api.searchAgents(searchTerm, currentPage, 9);
+        const filteredAgents = response.items.filter((agent) => agent.rank > 0);
         setSearchResults(filteredAgents);
+        setSearchTotalPages(response.total_pages);
       } catch (error) {
         console.error("Error searching agents:", error);
       } finally {
@@ -244,30 +246,42 @@ const Marketplace: React.FC = () => {
 
   useEffect(() => {
     if (searchValue) {
-      debouncedSearch(searchValue);
+      searchAgents(searchValue, searchPage);
     } else {
-      fetchTopAgents(page);
+      fetchTopAgents(topAgentsPage);
     }
-  }, [searchValue, page, debouncedSearch, fetchTopAgents]);
+  }, [searchValue, searchPage, topAgentsPage, searchAgents, fetchTopAgents]);
 
   useEffect(() => {
     fetchFeaturedAgents();
   }, [fetchFeaturedAgents]);
 
   const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
     setSearchValue(e.target.value);
-    setPage(1);
+    setSearchPage(1);
   };
 
   const handleNextPage = () => {
-    if (page < totalPages) {
-      setPage(page + 1);
+    if (searchValue) {
+      if (searchPage < searchTotalPages) {
+        setSearchPage(searchPage + 1);
+      }
+    } else {
+      if (topAgentsPage < topAgentsTotalPages) {
+        setTopAgentsPage(topAgentsPage + 1);
+      }
     }
   };
 
   const handlePrevPage = () => {
-    if (page > 1) {
-      setPage(page - 1);
+    if (searchValue) {
+      if (searchPage > 1) {
+        setSearchPage(searchPage - 1);
+      }
+    } else {
+      if (topAgentsPage > 1) {
+        setTopAgentsPage(topAgentsPage - 1);
+      }
     }
   };
 
@@ -283,7 +297,15 @@ const Marketplace: React.FC = () => {
           </div>
         ) : searchValue ? (
           searchResults.length > 0 ? (
-            <AgentGrid agents={searchResults} title="Search Results" />
+            <>
+              <AgentGrid agents={searchResults} title="Search Results" />
+              <Pagination
+                page={searchPage}
+                totalPages={searchTotalPages}
+                onPrevPage={handlePrevPage}
+                onNextPage={handleNextPage}
+              />
+            </>
           ) : (
             <div className="py-12 text-center">
               <p className="text-gray-600">
@@ -302,8 +324,8 @@ const Marketplace: React.FC = () => {
             )}
             <AgentGrid agents={topAgents} title="Top Downloaded Agents" />
             <Pagination
-              page={page}
-              totalPages={totalPages}
+              page={topAgentsPage}
+              totalPages={topAgentsTotalPages}
               onPrevPage={handlePrevPage}
               onNextPage={handleNextPage}
             />

diff --git a/autogpt_platform/frontend/src/components/admin/marketplace/AdminFeaturedAgentsControl.tsx b/autogpt_platform/frontend/src/components/admin/marketplace/AdminFeaturedAgentsControl.tsx
@@ -46,11 +46,11 @@ export default async function AdminFeaturedAgentsControl({
         <h3 className="text-lg font-semibold">Featured Agent Controls</h3>
         <AdminAddFeaturedAgentDialog
           categories={categories.unique_categories}
-          agents={notFeaturedAgents.agents}
+          agents={notFeaturedAgents.items}
         />
       </div>
       <FeaturedAgentsTable
-        agents={agents.agents}
+        agents={agents.items}
         globalActions={[
           {
             component: <Button>Remove</Button>,

diff --git a/autogpt_platform/frontend/src/components/admin/marketplace/actions.ts b/autogpt_platform/frontend/src/components/admin/marketplace/actions.ts
@@ -59,7 +59,7 @@ export async function getFeaturedAgents(
     async () => {
       const api = new ServerSideMarketplaceAPI();
       const featured = await api.getFeaturedAgents(page, pageSize);
-      console.debug(`Getting featured agents ${featured.agents.length}`);
+      console.debug(`Getting featured agents ${featured.items.length}`);
       return featured;
     },
   );
@@ -135,7 +135,7 @@ export async function getNotFeaturedAgents(
     async () => {
       const api = new ServerSideMarketplaceAPI();
       const agents = await api.getNotFeaturedAgents(page, pageSize);
-      console.debug(`Getting not featured agents ${agents.agents.length}`);
+      console.debug(`Getting not featured agents ${agents.items.length}`);
       return agents;
     },
   );

diff --git a/autogpt_platform/frontend/src/lib/marketplace-api/base-client.ts b/autogpt_platform/frontend/src/lib/marketplace-api/base-client.ts
@@ -3,12 +3,14 @@ import {
   AddAgentRequest,
   AgentResponse,
   ListAgentsParams,
-  AgentListResponse,
   AgentDetailResponse,
   AgentWithRank,
   FeaturedAgentResponse,
   UniqueCategoriesResponse,
   AnalyticsEvent,
+  ListResponse,
+  Agent,
+  AgentListResponse,
 } from "./types";
 
 export default class BaseMarketplaceAPI {
@@ -46,7 +48,7 @@ export default class BaseMarketplaceAPI {
   async getTopDownloadedAgents(
     page: number = 1,
     pageSize: number = 10,
-  ): Promise<AgentListResponse> {
+  ): Promise<ListResponse<Agent>> {
     return this._get(
       `/top-downloads/agents?page=${page}&page_size=${pageSize}`,
     );
@@ -55,7 +57,7 @@ export default class BaseMarketplaceAPI {
   async getFeaturedAgents(
     page: number = 1,
     pageSize: number = 10,
-  ): Promise<AgentListResponse> {
+  ): Promise<ListResponse<Agent>> {
     return this._get(`/featured/agents?page=${page}&page_size=${pageSize}`);
   }
 
@@ -67,7 +69,7 @@ export default class BaseMarketplaceAPI {
     descriptionThreshold: number = 60,
     sortBy: string = "rank",
     sortOrder: "asc" | "desc" = "desc",
-  ): Promise<AgentWithRank[]> {
+  ): Promise<ListResponse<AgentWithRank>> {
     const queryParams = new URLSearchParams({
       query,
       page: page.toString(),
@@ -126,7 +128,7 @@ export default class BaseMarketplaceAPI {
     );
   }
 
-  async getAgentSubmissions(): Promise<AgentListResponse> {
+  async getAgentSubmissions(): Promise<ListResponse<AgentResponse>> {
     return this._get("/admin/agent/submissions");
   }
 
@@ -186,7 +188,7 @@ export default class BaseMarketplaceAPI {
   async getNotFeaturedAgents(
     page: number = 1,
     pageSize: number = 10,
-  ): Promise<AgentListResponse> {
+  ): Promise<ListResponse<AgentResponse>> {
     return this._get(
       `/admin/agent/not-featured?page=${page}&page_size=${pageSize}`,
     );

diff --git a/autogpt_platform/frontend/src/lib/marketplace-api/types.ts b/autogpt_platform/frontend/src/lib/marketplace-api/types.ts
@@ -67,7 +67,16 @@ export type AgentWithRank = Agent & {
   rank: number;
 };
 
-export type AgentListResponse = AgentList;
+export type ListResponse<T> = {
+  items: T[];
+  total_count: number;
+  page: number;
+  page_size: number;
+  total_pages: number;
+};
+
+export type AgentListResponse = ListResponse<Agent>;
+export type AgentWithRankListResponse = ListResponse<AgentWithRank>;
 
 export type AgentDetailResponse = AgentDetail;
 

diff --git a/autogpt_platform/market/market/app.py b/autogpt_platform/market/market/app.py
@@ -49,7 +49,8 @@ async def lifespan(app: fastapi.FastAPI):
     yield
     await db_client.disconnect()
 
-docs_url = "/docs" if os.environ.get("APP_ENV") == "local" else None
+
+docs_url = "/docs"
 app = fastapi.FastAPI(
     title="Marketplace API",
     description="AutoGPT Marketplace API is a service that allows users to share AI agents.",