Merge pull request #305 from CSCfi/feature/find-by-name

Filter folder names by name

.devcontainer/devcontainer.json

@@ -17,6 +17,7 @@
 			}
 		},
 		"python.formatting.provider": "black",
+		"python.formatting.blackArgs": ["--line-length", "120"],
 		"python.languageServer": "Pylance",
 		"python.linting.flake8Enabled": true,
 		"python.linting.pylintEnabled": true,

.env.example

@@ -1,33 +1,33 @@
-      # authentication
-      AAI_CLIENT_SECRET=secret_must_be_long
-      AAI_CLIENT_ID=aud2
-      ISS_URL=http://mockauth:8000
-      AUTH_URL=http://localhost:8000/authorize
-      OIDC_URL=http://mockauth:8000
-      AUTH_REFERER=http://mockauth:8000
-      JWK_URL=http://mockauth:8000/keyset
+# authentication
+AAI_CLIENT_SECRET=secret_must_be_long
+AAI_CLIENT_ID=aud2
+ISS_URL=http://mockauth:8000
+AUTH_URL=http://localhost:8000/authorize
+OIDC_URL=http://mockauth:8000
+AUTH_REFERER=http://mockauth:8000
+JWK_URL=http://mockauth:8000/keyset
 
-      # app urls
-      BASE_URL=http://localhost:5430
-      # change to http://frontend:3000 if started using docker-compose for frontend
-      REDIRECT_URL=http://localhost:3000
+# app urls
+BASE_URL=http://localhost:5430
+# change to http://frontend:3000 if started using docker-compose for frontend
+# REDIRECT_URL=http://localhost:3000
 
-      # logging
-      LOG_LEVEL=DEBUG
+# logging
+LOG_LEVEL=DEBUG
 
-      # database
-      MONGO_HOST=database:27017
-      MONGO_DATABASE=default
-      MONGO_AUTHDB=admin
-      MONGO_INITDB_ROOT_PASSWORD=admin
-      MONGO_INITDB_ROOT_USERNAME=admin
-      MONGO_SSL=true
-      MONGO_SSL_CA=/tls/cacert
-      MONGO_SSL_CLIENT_KEY=/tls/key
-      MONGO_SSL_CLIENT_CERT=/tls/cert
+# database
+MONGO_HOST=database:27017
+MONGO_DATABASE=default
+MONGO_AUTHDB=admin
+MONGO_INITDB_ROOT_PASSWORD=admin
+MONGO_INITDB_ROOT_USERNAME=admin
+MONGO_SSL=true
+MONGO_SSL_CA=/tls/cacert
+MONGO_SSL_CLIENT_KEY=/tls/key
+MONGO_SSL_CLIENT_CERT=/tls/cert
 
-      # doi
-      DOI_API=http://mockdoi:8001/dois
-      DOI_PREFIX=10.xxxx
-      DOI_USER=user
-      DOI_KEY=key
+# doi
+DOI_API=http://mockdoi:8001/dois
+DOI_PREFIX=10.xxxx
+DOI_USER=user
+DOI_KEY=key

.github/workflows/int.yml

@@ -29,10 +29,20 @@ jobs:
         docker-compose --env-file .env.example up -d --build 
         sleep 30
   
-    - name: Run Integration test
+    - name: Clear database
       run: |
         python tests/integration/clean_db.py
+      env:
+        MONGO_HOST: localhost:27017
+        MONGO_DATABASE: default
+        MONGO_AUTHDB: admin
+
+    - name: Run Integration test
+      run: |
         python tests/integration/run_tests.py
+      env:
+        BASE_URL: http://localhost:5430
+        ISS_URL: http://localhost:8000
 
     - name: Collect logs from docker
       if: ${{ failure() }}
@@ -58,10 +68,20 @@ jobs:
         docker-compose -f docker-compose-tls.yml --env-file .env.example up -d
         sleep 30
 
-    - name: Run Integration test
+    - name: Clear database
       run: |
         python tests/integration/clean_db.py --tls
+      env:
+        MONGO_HOST: localhost:27017
+        MONGO_DATABASE: default
+        MONGO_AUTHDB: admin
+
+    - name: Run Integration test
+      run: |
         python tests/integration/run_tests.py
+      env:
+        BASE_URL: http://localhost:5430
+        ISS_URL: http://localhost:8000
 
     - name: Collect logs from docker
       if: ${{ failure() }}

Dockerfile-dev

@@ -27,7 +27,7 @@ FROM appbase as local
 #=======================
 
 COPY requirements-dev.txt .
-COPY ./scripts/ ./scripts
+COPY ./scripts/install-hooks.sh ./scripts/install-hooks.sh
 
 RUN pip install --no-cache-dir -r requirements.txt
 RUN pip install --no-cache-dir -r requirements-dev.txt 

docker-compose-tls.yml

@@ -49,6 +49,7 @@ services:
     volumes:
       - data:/data/db
       - ./config:/tls
+      - ./scripts/init_mongo.js:/docker-entrypoint-initdb.d/init_mongo.js:ro
     expose:
       - 27017
     ports:

docker-compose.yml

@@ -42,6 +42,7 @@ services:
       - "MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD}"
     volumes:
       - data:/data/db
+      - ./scripts/init_mongo.js:/docker-entrypoint-initdb.d/init_mongo.js:ro
     expose:
       - 27017
     ports:

docs/specification.yml

@@ -897,7 +897,13 @@ paths:
           name: published
           schema:
             type: string
-          description: Return folders based on the folder published value
+          description: Return folders based on the folder published value. Should be 'true' or 'false'
+        - in: query
+          name: name
+          schema:
+            type: string
+          description: Return folders containing filtered string[s] in their name
+          example: test folder
       responses:
         200:
           description: OK

metadata_backend/api/handlers.py

@@ -1,33 +1,29 @@
 """Handle HTTP methods for server."""
-import ujson
 import json
-import re
 import mimetypes
+import re
 from collections import Counter
+from datetime import date, datetime
+from distutils.util import strtobool
 from math import ceil
 from pathlib import Path
-from typing import Dict, List, Tuple, Union, cast, AsyncGenerator, Any
-from datetime import date, datetime
+from typing import Any, AsyncGenerator, Dict, List, Tuple, Union, cast
 
+import ujson
 from aiohttp import BodyPartReader, web
 from aiohttp.web import Request, Response
-from multidict import CIMultiDict
 from motor.motor_asyncio import AsyncIOMotorClient
-from multidict import MultiDict, MultiDictProxy
+from multidict import CIMultiDict, MultiDict, MultiDictProxy
 from xmlschema import XMLSchemaException
-from distutils.util import strtobool
-
-from .middlewares import decrypt_cookie, get_session
 
-from ..conf.conf import schema_types
+from ..conf.conf import aai_config, publisher, schema_types
+from ..helpers.doi import DOIHandler
 from ..helpers.logger import LOG
 from ..helpers.parser import XMLToJSONParser
 from ..helpers.schema_loader import JSONSchemaLoader, SchemaNotFoundException, XMLSchemaLoader
 from ..helpers.validator import JSONValidator, XMLValidator
-from ..helpers.doi import DOIHandler
-from .operators import FolderOperator, Operator, XMLOperator, UserOperator
-
-from ..conf.conf import aai_config, publisher
+from .middlewares import decrypt_cookie, get_session
+from .operators import FolderOperator, Operator, UserOperator, XMLOperator
 
 
 class RESTAPIHandler:
@@ -689,6 +685,7 @@ async def get_folders(self, req: Request) -> Response:
         """
         page = self._get_page_param(req, "page", 1)
         per_page = self._get_page_param(req, "per_page", 5)
+        sort = {"date": True, "score": False}
         db_client = req.app["db_client"]
 
         user_operator = UserOperator(db_client)
@@ -705,9 +702,15 @@ async def get_folders(self, req: Request) -> Response:
                 reason = "'published' parameter must be either 'true' or 'false'"
                 LOG.error(reason)
                 raise web.HTTPBadRequest(reason=reason)
+        if "name" in req.query:
+            name_param = req.query.get("name", "")
+            if name_param:
+                folder_query = {"$text": {"$search": name_param}}
+            sort["score"] = True
+            sort["date"] = False
 
         folder_operator = FolderOperator(db_client)
-        folders, total_folders = await folder_operator.query_folders(folder_query, page, per_page)
+        folders, total_folders = await folder_operator.query_folders(folder_query, page, per_page, sort)
 
         result = ujson.dumps(
             {

metadata_backend/api/operators.py

@@ -1,18 +1,18 @@
 """Operators for handling database-related operations."""
 import re
+import time
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from uuid import uuid4
-import time
 
 from aiohttp import web
 from dateutil.relativedelta import relativedelta
 from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorCursor
 from multidict import MultiDictProxy
 from pymongo.errors import ConnectionFailure, OperationFailure
 
-from ..conf.conf import query_map, mongo_database
+from ..conf.conf import mongo_database, query_map
 from ..database.db_service import DBService, auto_reconnect
 from ..helpers.logger import LOG
 from ..helpers.parser import XMLToJSONParser
@@ -658,6 +658,7 @@ async def create_folder(self, data: Dict) -> str:
         """
         folder_id = self._generate_folder_id()
         data["folderId"] = folder_id
+        data["text_name"] = " ".join(re.split("[\\W_]", data["name"]))
         data["published"] = False
         data["dateCreated"] = int(time.time())
         data["metadataObjects"] = data["metadataObjects"] if "metadataObjects" in data else []
@@ -677,21 +678,34 @@ async def create_folder(self, data: Dict) -> str:
             LOG.info(f"Inserting folder with id {folder_id} to database succeeded.")
             return folder_id
 
-    async def query_folders(self, query: Dict, page_num: int, page_size: int) -> Tuple[List, int]:
+    async def query_folders(
+        self, query: Dict, page_num: int, page_size: int, sort_param: Optional[dict] = None
+    ) -> Tuple[List, int]:
         """Query database based on url query parameters.
 
         :param query: Dict containing query information
         :param page_num: Page number
         :param page_size: Results per page
+        :param sort_param: Sorting options.
         :returns: Paginated query result
         """
         skips = page_size * (page_num - 1)
+
+        if not sort_param:
+            sort = {"dateCreated": -1}
+        elif sort_param["score"] and not sort_param["date"]:
+            sort = {"score": {"$meta": "textScore"}, "dateCreated": -1}  # type: ignore
+        elif sort_param["score"] and sort_param["date"]:
+            sort = {"dateCreated": -1, "score": {"$meta": "textScore"}}  # type: ignore
+        else:
+            sort = {"dateCreated": -1}
+
         _query = [
             {"$match": query},
-            {"$sort": {"dateCreated": -1}},
+            {"$sort": sort},
             {"$skip": skips},
             {"$limit": page_size},
-            {"$project": {"_id": 0}},
+            {"$project": {"_id": 0, "text_name": 0}},
         ]
         data_raw = await self.db_service.do_aggregate("folder", _query)
 

metadata_backend/helpers/schemas/folders.json

@@ -14,6 +14,10 @@
       "type": "string",
       "title": "Folder Name"
     },
+    "text_name": {
+      "type": "string",
+      "title": "Searchable Folder Name, used for indexing"
+    },
     "description": {
       "type": "string",
       "title": "Folder Description"
@@ -1023,4 +1027,4 @@
     }
   },
   "additionalProperties": false
-}
+}

requirements-dev.in

@@ -1,6 +1,7 @@
+aiofiles  # to run integration tests
 black
 certifi
 flake8
-pip-tools
+pip-tools  # pip depedencies management
 pre-commit
 tox

requirements-dev.txt

@@ -4,6 +4,8 @@
 #
 #    pip-compile requirements-dev.in
 #
+aiofiles==0.8.0
+    # via -r requirements-dev.in
 backports.entry-points-selectable==1.1.1
     # via virtualenv
 black==21.12b0

scripts/init_mongo.js

@@ -0,0 +1,17 @@
+// script to create default database collections and indexes
+// on container start up
+
+db = new Mongo().getDB("default");
+
+db.createCollection('user', { capped: false });
+db.createCollection('folder', { capped: false });
+db.folder.createIndex({ "dateCreated": -1 });
+db.folder.createIndex({ "datePublished": -1 });
+db.folder.createIndex({ "folderId": 1, unique: 1 });
+db.user.createIndex({ "userId": 1, unique: 1 });
+db.folder.createIndex(
+  {
+    text_name: "text",
+  }
+)
+db.folder.getIndexes()