Merge branch 'main' of https://github.com/materialsproject/maggma int…

…o filestore

.github/workflows/testing.yml

@@ -85,7 +85,7 @@ jobs:
       run: |
         pip install -e .
         pytest --cov=maggma --cov-report=xml
-    - uses: codecov/codecov-action@v3.0.0
+    - uses: codecov/codecov-action@v3.1.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./coverage.xml

docs/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [v0.46.1](https://github.com/materialsproject/maggma/tree/v0.46.1) (2022-04-21)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.46.0...v0.46.1)
+
+**Merged pull requests:**
+
+- Prefix `fields` input for read resource key endpoint [\#636](https://github.com/materialsproject/maggma/pull/636) ([munrojm](https://github.com/munrojm))
+
+## [v0.46.0](https://github.com/materialsproject/maggma/tree/v0.46.0) (2022-04-19)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.45.1...v0.46.0)
+
+**Merged pull requests:**
+
+- S3 store and resource additions [\#635](https://github.com/materialsproject/maggma/pull/635) ([munrojm](https://github.com/munrojm))
+
 ## [v0.45.1](https://github.com/materialsproject/maggma/tree/v0.45.1) (2022-04-18)
 
 [Full Changelog](https://github.com/materialsproject/maggma/compare/v0.45.0...v0.45.1)
@@ -50,23 +66,23 @@
 
 ## [v0.44.1](https://github.com/materialsproject/maggma/tree/v0.44.1) (2022-03-08)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.42.0...v0.44.1)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.43.0...v0.44.1)
 
 **Merged pull requests:**
 
 - added localhost test for MongoURIStore [\#595](https://github.com/materialsproject/maggma/pull/595) ([jmmshn](https://github.com/jmmshn))
 
-## [v0.42.0](https://github.com/materialsproject/maggma/tree/v0.42.0) (2022-03-07)
-
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.43.0...v0.42.0)
-
 ## [v0.43.0](https://github.com/materialsproject/maggma/tree/v0.43.0) (2022-03-07)
 
 [Full Changelog](https://github.com/materialsproject/maggma/compare/v0.44.0...v0.43.0)
 
 ## [v0.44.0](https://github.com/materialsproject/maggma/tree/v0.44.0) (2022-03-07)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.41.1...v0.44.0)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.42.0...v0.44.0)
+
+## [v0.42.0](https://github.com/materialsproject/maggma/tree/v0.42.0) (2022-03-07)
+
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.41.1...v0.42.0)
 
 **Merged pull requests:**
 
@@ -146,19 +162,19 @@
 
 ## [v0.36.0](https://github.com/materialsproject/maggma/tree/v0.36.0) (2021-12-06)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.34.0...v0.36.0)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.35.0...v0.36.0)
 
 **Merged pull requests:**
 
 - Added on-disk MongoDB compatible MontyStore [\#514](https://github.com/materialsproject/maggma/pull/514) ([utf](https://github.com/utf))
 
-## [v0.34.0](https://github.com/materialsproject/maggma/tree/v0.34.0) (2021-12-01)
+## [v0.35.0](https://github.com/materialsproject/maggma/tree/v0.35.0) (2021-12-01)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.35.0...v0.34.0)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.34.0...v0.35.0)
 
-## [v0.35.0](https://github.com/materialsproject/maggma/tree/v0.35.0) (2021-12-01)
+## [v0.34.0](https://github.com/materialsproject/maggma/tree/v0.34.0) (2021-12-01)
 
-[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.33.2...v0.35.0)
+[Full Changelog](https://github.com/materialsproject/maggma/compare/v0.33.2...v0.34.0)
 
 **Merged pull requests:**
 

requirements-docs.txt

@@ -1,5 +1,5 @@
-mkdocs==1.2.3
-mkdocs-material==8.2.5
+mkdocs==1.3.0
+mkdocs-material==8.2.11
 mkdocs-minify-plugin==0.5.0
 mkdocstrings==0.18.1
-jinja2<3.1.0
+jinja2<3.2.0

requirements-optional.txt

@@ -1,6 +1,6 @@
 uvicorn==0.17.6
 hvac==0.11.2
 IPython==7.31.1;python_version>"3.6"
-nbformat==5.2.0
-regex==2022.3.15
+nbformat==5.3.0
+regex==2022.4.24
 montydb==2.3.12

requirements-testing.txt

@@ -1,14 +1,14 @@
 pre-commit==2.18.1
-pytest==7.1.1
+pytest==7.1.2
 pytest-asyncio==0.18.3
 pytest-cov==3.0.0
 pytest-mock==3.7.0
 pytest-xdist==2.5.0
-moto==3.1.5
+moto==3.1.6
 pydocstyle==6.1.1
 flake8==4.0.1
 mypy==0.942
 mypy-extensions==0.4.3
 responses<0.21.0
 types-PyYAML==6.0.5
-types-setuptools==57.4.11
+types-setuptools==57.4.14

requirements.txt

@@ -16,4 +16,4 @@ sshtunnel==0.4.0
 msgpack==1.0.3
 msgpack-python==0.5.6
 orjson==3.6.8
-boto3==1.21.22
+boto3==1.21.46

src/maggma/api/models.py

@@ -73,3 +73,18 @@ def default_meta(cls, v, values):
             else:
                 v["total_doc"] = 0
         return v
+
+
+class S3URLDoc(BaseModel):
+
+    """
+    S3 pre-signed URL data returned by the S3 URL resource
+    """
+
+    url: str = Field(
+        ..., description="Pre-signed download URL",
+    )
+
+    requested_datetime: datetime = Field(..., description="Datetime for when URL was requested")
+
+    expiry_datetime: datetime = Field(..., description="Expiry datetime of the URL")

src/maggma/api/resource/__init__.py

@@ -9,3 +9,4 @@
 from maggma.api.resource.post_resource import PostOnlyResource
 from maggma.api.resource.read_resource import ReadOnlyResource, attach_query_ops
 from maggma.api.resource.submission import SubmissionResource
+from maggma.api.resource.s3_url import S3URLResource

src/maggma/api/resource/read_resource.py

@@ -10,7 +10,7 @@
 from maggma.api.query_operator import PaginationQuery, QueryOperator, SparseFieldsQuery
 from maggma.api.resource import Resource, HintScheme, HeaderProcessor
 from maggma.api.resource.utils import attach_query_ops
-from maggma.api.utils import STORE_PARAMS, merge_queries, object_id_serilaization_helper
+from maggma.api.utils import STORE_PARAMS, merge_queries, serialization_helper
 from maggma.core import Store
 from maggma.stores.mongolike import MongoStore
 
@@ -79,10 +79,7 @@ def __init__(
             if query_operators is not None
             else [
                 PaginationQuery(),
-                SparseFieldsQuery(
-                    model,
-                    default_fields=[self.store.key, self.store.last_updated_field],
-                ),
+                SparseFieldsQuery(model, default_fields=[self.store.key, self.store.last_updated_field],),
             ]
         )
 
@@ -105,9 +102,7 @@ def build_get_by_key(self):
         model_name = self.model.__name__
 
         if self.key_fields is None:
-            field_input = SparseFieldsQuery(
-                self.model, [self.store.key, self.store.last_updated_field]
-            ).query
+            field_input = SparseFieldsQuery(self.model, [self.store.key, self.store.last_updated_field]).query
         else:
 
             def field_input():
@@ -116,10 +111,8 @@ def field_input():
         async def get_by_key(
             request: Request,
             response: Response,
-            key: str = Path(
-                ..., alias=key_name, title=f"The {key_name} of the {model_name} to get",
-            ),
-            fields: STORE_PARAMS = Depends(field_input),
+            key: str = Path(..., alias=key_name, title=f"The {key_name} of the {model_name} to get",),
+            _fields: STORE_PARAMS = Depends(field_input),
         ):
             f"""
             Get's a document by the primary key in the store
@@ -132,16 +125,11 @@ async def get_by_key(
             """
             self.store.connect()
 
-            item = [
-                self.store.query_one(
-                    criteria={self.store.key: key}, properties=fields["properties"],
-                )
-            ]
+            item = [self.store.query_one(criteria={self.store.key: key}, properties=_fields["properties"],)]
 
             if item == [None]:
                 raise HTTPException(
-                    status_code=404,
-                    detail=f"Item with {self.store.key} = {key} not found",
+                    status_code=404, detail=f"Item with {self.store.key} = {key} not found",
                 )
 
             for operator in self.query_operators:
@@ -151,7 +139,7 @@ async def get_by_key(
 
             if self.disable_validation:
                 response = Response(  # type: ignore
-                    orjson.dumps(response, default=object_id_serilaization_helper)
+                    orjson.dumps(response, default=serialization_helper)
                 )
 
             if self.header_processor is not None:
@@ -178,14 +166,10 @@ async def search(**queries: Dict[str, STORE_PARAMS]) -> Union[Dict, Response]:
             response: Response = queries.pop("temp_response")  # type: ignore
 
             query_params = [
-                entry
-                for _, i in enumerate(self.query_operators)
-                for entry in signature(i.query).parameters
+                entry for _, i in enumerate(self.query_operators) for entry in signature(i.query).parameters
             ]
 
-            overlap = [
-                key for key in request.query_params.keys() if key not in query_params
-            ]
+            overlap = [key for key in request.query_params.keys() if key not in query_params]
             if any(overlap):
                 if "limit" in overlap or "skip" in overlap:
                     raise HTTPException(
@@ -210,13 +194,7 @@ async def search(**queries: Dict[str, STORE_PARAMS]) -> Union[Dict, Response]:
 
             self.store.connect()
 
-            count = self.store.count(
-                **{
-                    field: query[field]
-                    for field in query
-                    if field in ["criteria", "hint"]
-                }
-            )
+            count = self.store.count(**{field: query[field] for field in query if field in ["criteria", "hint"]})
 
             data = list(self.store.query(**query))
             operator_meta = {}
@@ -231,7 +209,7 @@ async def search(**queries: Dict[str, STORE_PARAMS]) -> Union[Dict, Response]:
 
             if self.disable_validation:
                 response = Response(  # type: ignore
-                    orjson.dumps(response, default=object_id_serilaization_helper)
+                    orjson.dumps(response, default=serialization_helper)
                 )
 
             if self.header_processor is not None: