Use new HTTP API to access the database

.coveragerc

@@ -0,0 +1,4 @@
+[report]
+exclude_lines:
+    pragma: no cover
+    @(?:abc\.)?abstractmethod

.github/workflows/tests.yml

@@ -5,17 +5,15 @@ name: Tests
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - main
   pull_request:
-    branches: [ main ]
-  schedule:
-    - cron: '0 9 * * MON,WED,FRI'
+    branches:
+      - main
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.8

.gitignore

@@ -87,7 +87,7 @@ ipython_config.py
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+.python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -138,4 +138,8 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-.idea/
+.idea/
+
+#pyenv
+Pipfile.lock
+Pipfile

README.md

@@ -56,10 +56,9 @@ At the client object initialization
 You can pass parameters to the Alerce class constructor to set the
 parameters for API connection.
 
-For example using an API on localhost: .. code-block:: python
-
+For example using the ZTF API on localhost:5000 and the DB API on localhost:5050 
 ``` {.sourceCode .python}
-alerce = Alerce(ZTF_API_URL="<http://localhost:5000>")
+alerce = Alerce(ZTF_API_URL="<http://localhost:5000>", ZTF_DB_API_URL="<http://localhost:5050>")
 ```
 
 From a dictionary object
@@ -70,6 +69,7 @@ You can pass parameters to the Alerce class from a dictionary object.
 ``` {.sourceCode .python}
 my_config = {
     "ZTF_API_URL": "http://localhost:5000"
+    "ZTF_DB_API_URL": "http://localhost:5050"
 }
 alerce = Alerce()
 alerce.load_config_from_object(my_config)

alerce/core.py

@@ -1,9 +1,10 @@
 from .search import AlerceSearch
 from .crossmatch import AlerceXmatch
 from .stamps import AlerceStamps
+from .direct import AlerceDirect
 
 
-class Alerce(AlerceSearch, AlerceXmatch, AlerceStamps):
+class Alerce(AlerceSearch, AlerceXmatch, AlerceStamps, AlerceDirect):
     """
     The main client class that has all the methods for accessing the different services.
 
@@ -18,19 +19,4 @@ class Alerce(AlerceSearch, AlerceXmatch, AlerceStamps):
         The url of the ZTF API
     """
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def load_config_from_file(self, path):
-        pass
-
-    def load_config_from_object(self, object):
-        """
-        Sets configuration parameters from a dictionary object.
-
-        Parameters
-        ------------
-        object : dict
-            The dictionary containing the config.
-        """
-        super().load_config_from_object(object)
+    pass

alerce/direct.py

@@ -0,0 +1,36 @@
+from .utils import Client
+
+
+class AlerceDirect(Client):
+
+    """Handles direct request to the database using the available http API"""
+
+    def __init__(self, **kwargs):
+        default_config = {
+            "ZTF_DB_API_URL": "https://api.alerce.online/db/",
+        }
+        default_config.update(kwargs)
+        super().__init__(**default_config)
+
+    def __get_url(self):
+        return f"{self.config['ZTF_DB_API_URL']}"
+
+    def send_query(self, query, format="csv", index=None, sort=None):
+        """Sends the query directly to the API, returning the byte reply directly
+
+        :query: query for the database
+        :format: Format to be returned
+        :index: index if format is pandas
+        :sort: sorting column if format is pandas
+        :returns: byte reply
+
+        """
+        data = {"query": query}
+        q = self._request(
+            "POST",
+            self.__get_url(),
+            data=data,
+            result_format=format,
+            response_format="csv",
+        )
+        return q.result(index, sort)

alerce/exceptions.py

@@ -1,11 +1,20 @@
-def handle_error(response):
+import requests
+
+
+def handle_error(response, response_format="json"):
+    # TODO: The direct API uses code 400 for user input error (bad requests, etc), what should be done here then?
     codes = {-1: APIError, 400: ParseError, 404: ObjectNotFoundError}
-    try:
-        error = response.json().get("errors", {})
-        message = response.json().get("message")
-    except:
-        message = "Unknown API error."
-        error = "Unknown API error."
+    message = "Unknown API error."
+    error = "Unknown API error."
+    if response_format == "json":
+        try:
+            error = response.json().get("errors", {})
+            message = response.json().get("message")
+        except requests.exceptions.JSONDecodeError:
+            pass
+    elif response_format == "csv":
+        error = response.content.decode("utf-8")
+        message = response.content.decode("utf-8")
     code = response.status_code
     data = error
 

alerce/search.py

@@ -1,5 +1,4 @@
-from .exceptions import FormatValidationError, ParseError, handle_error
-from .utils import Result, Client
+from .utils import Client
 
 
 class AlerceSearch(Client):
@@ -30,14 +29,6 @@ def ztf_url(self):
     def __get_url(self, resource, *args):
         return self.ztf_url + self.config["ZTF_ROUTES"][resource] % args
 
-    def __validate_format(self, format):
-        format = format.lower()
-        if not format in self.allowed_formats:
-            raise FormatValidationError(
-                "Format '%s' not in %s" % (format, self.allowed_formats), code=500
-            )
-        return format
-
     def query_objects(self, format="pandas", index=None, sort=None, **kwargs):
         """
         Gets a list of objects filtered by specified parameters.

alerce/utils.py

@@ -1,16 +1,65 @@
-from pandas import DataFrame
+from pandas import DataFrame, read_csv
+from io import StringIO
 from astropy.table import Table
 from .exceptions import handle_error, FormatValidationError
 import requests
+import abc
 
 
-class Result:
-    def __init__(self, json_result, format="json"):
-        self.json_result = json_result
+class Result(abc.ABC):
+    def __init__(self, format="json"):
         self.format = format
 
+    @abc.abstractmethod
+    def to_pandas(self, index=None, sort=None):
+        """Convert the result to a pandas dataframe
+
+        :index: index for the pandas dataframe
+        :sort: sorting column for the dataframe
+        :returns: the processed dataframe
+
+        """
+        pass
+
+    @abc.abstractmethod
+    def to_votable(self):
+        pass
+
+    @abc.abstractmethod
+    def to_json(self):
+        pass
+
+    @abc.abstractmethod
+    def to_csv(self, index=None, sort=None):
+        pass
+
+    def result(self, index=None, sort=None):
+        """Creates the result depending on the arguments and the expected format
+
+        :index: if format is pandas, this is the index column
+        :sort: if format is pandas, this is the sort column
+        :returns: Result in the indicated format
+
+        """
+        if self.format == "pandas":
+            return self.to_pandas(index, sort)
+        elif self.format == "votable":
+            return self.to_votable()
+        elif self.format == "csv":
+            return self.to_csv()
+        elif self.format == "json":
+            return self.to_json()
+        raise ValueError(f"Unrecognized format '{self.format}'")
+
+
+class ResultJson(Result):
+    """Object that holds a json type result"""
+
+    def __init__(self, json_result, **kwargs):
+        self.json_result = json_result
+        super().__init__(**kwargs)
+
     def to_pandas(self, index=None, sort=None):
-        dataframe = None
         if isinstance(self.json_result, list):
             dataframe = DataFrame(self.json_result)
         else:
@@ -27,21 +76,64 @@ def to_votable(self):
     def to_json(self):
         return self.json_result
 
-    def result(self, index=None, sort=None):
-        if self.format == "json":
-            return self.to_json()
-        if self.format == "pandas":
-            return self.to_pandas(index, sort)
-        if self.format == "votable":
-            return self.to_votable()
+    def to_csv(self):
+        df = self.to_pandas()
+        return df.to_csv(index=False)
+
+
+class ResultCsv(Result):
+
+    """Object that holds a csv type result"""
+
+    def __init__(self, csv_result_byte, **kwargs):
+        self.csv_result = csv_result_byte.decode("utf-8")
+        self.data = [x.split(",") for x in self.csv_result.split("\n")[1:-1]]
+
+        self.columns = [x for x in self.csv_result.split("\n")[0].split(",")]
+        super().__init__(**kwargs)
+
+    def _rename_duplicates(self):
+        counts = {}
+        columns = self.columns.copy()
+        for i, column in enumerate(columns):
+            if column in counts:
+                counts[column] += 1
+                columns[i] = f"{column}_{counts[column] - 1}"
+            else:
+                counts[column] = 1
+        return columns
+
+    def to_pandas(self, index=None, sort=None):
+        dataframe = read_csv(StringIO(self.csv_result))
+        if sort:
+            dataframe.sort_values(sort, inplace=True)
+        if index:
+            dataframe.set_index(index, inplace=True)
+        return dataframe
+
+    def to_votable(self):
+        # TODO: Check if renaming the columns doesn't cause problems to the user
+        columns = self._rename_duplicates()
+        df = read_csv(StringIO(self.csv_result), names=columns, skiprows=1)
+        df = df.convert_dtypes()
+        table = Table.from_pandas(df)
+        return table
+
+    def to_json(self):
+        columns = self._rename_duplicates()
+        df = read_csv(StringIO(self.csv_result), names=columns, skiprows=1)
+        return df.to_json(orient="records")
+
+    def to_csv(self):
+        return self.csv_result
 
 
 class Client:
     def __init__(self, **kwargs):
         self.session = requests.Session()
         self.config = {}
         self.config.update(kwargs)
-        self.allowed_formats = ["pandas", "votable", "json"]
+        self.allowed_formats = ["pandas", "votable", "json", "csv"]
 
     def load_config_from_file(self, path):
         pass
@@ -58,13 +150,23 @@ def _validate_format(self, format):
         return format
 
     def _request(
-        self, method, url, params=None, response_field=None, result_format="json"
+        self,
+        method,
+        url,
+        params=None,
+        data=None,
+        response_field=None,
+        result_format="json",
+        response_format="json",
     ):
         result_format = self._validate_format(result_format)
-        resp = self.session.request(method, url, params=params)
 
+        resp = self.session.request(method, url, params=params, data=data)
         if resp.status_code >= 400:
-            handle_error(resp)
-        if response_field and result_format != "json":
-            return Result(resp.json()[response_field], format=result_format)
-        return Result(resp.json(), format=result_format)
+            handle_error(resp, response_format)
+
+        if response_format == "csv":
+            return ResultCsv(resp, format=result_format)
+        if response_field and result_format != "json" and result_format != "csv":
+            return ResultJson(resp.json()[response_field], format=result_format)
+        return ResultJson(resp.json(), format=result_format)