Use new HTTP API to access the database

.gitignore

@@ -138,4 +138,8 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-.idea/
+.idea/
+
+#pyenv
+Pipfile.lock
+Pipfile

README.md

@@ -56,10 +56,10 @@ At the client object initialization
 You can pass parameters to the Alerce class constructor to set the
 parameters for API connection.
 
-For example using an API on localhost: .. code-block:: python
-
+For example using the ZTF API on localhost:5000 and the DB API on localhost:5050 
 ``` {.sourceCode .python}
 alerce = Alerce(ZTF_API_URL="<http://localhost:5000>")
+alerce = Alerce(ZTF_DB_API_URL="<http://localhost:5050>")
 ```
 
 From a dictionary object
@@ -70,6 +70,7 @@ You can pass parameters to the Alerce class from a dictionary object.
 ``` {.sourceCode .python}
 my_config = {
     "ZTF_API_URL": "http://localhost:5000"
+    "ZTF_DB_API_URL": "http://localhost:5050"
 }
 alerce = Alerce()
 alerce.load_config_from_object(my_config)

alerce/core.py

@@ -1,9 +1,10 @@
 from .search import AlerceSearch
 from .crossmatch import AlerceXmatch
 from .stamps import AlerceStamps
+from .direct import AlerceDirect
 
 
-class Alerce(AlerceSearch, AlerceXmatch, AlerceStamps):
+class Alerce(AlerceSearch, AlerceXmatch, AlerceStamps, AlerceDirect):
     """
     The main client class that has all the methods for accessing the different services.
 

alerce/direct.py

@@ -0,0 +1,33 @@
+from .exceptions import FormatValidationError, ParseError, handle_error
+from .utils import Result, Client
+
+class AlerceDirect(Client):
+
+    """Handles direct request to the database using the available http API"""
+
+    def __init__(self, **kwargs):
+        default_config = {
+            "ZTF_DB_API_URL": "https://api.alerce.online/db/",
+        }
+        default_config.update(kwargs)
+        super().__init__(**default_config)
+
+    def __get_url(self):
+        return f"{self.config['ZTF_DB_API_URL']}"
+
+    def send_query(self, query, format='csv', index=None, sort=None):
+        """Sends the query directly to the API, returning the byte reply directly
+
+        :query: query for the database
+        :format: Format to be returned
+        :index: index if format is pandas
+        :sort: sorting column if format is pandas
+        :returns: byte reply
+
+        """
+        data = {'query': query}
+        q = self._request(
+            "POST", self.__get_url(), data=data, result_format=format, response_format='csv'
+        )
+        return q.result(index, sort)
+

alerce/exceptions.py

@@ -1,11 +1,19 @@
-def handle_error(response):
+def handle_error(response, response_format='json'):
+    print(response)
+    # TODO: The direct API uses code 400 for user input error (bad requests, etc), what should be done here then?
     codes = {-1: APIError, 400: ParseError, 404: ObjectNotFoundError}
-    try:
-        error = response.json().get("errors", {})
-        message = response.json().get("message")
-    except:
-        message = "Unknown API error."
-        error = "Unknown API error."
+    message = "Unknown API error."
+    error = "Unknown API error."
+    if response_format=='json':
+        try:
+            error = response.json().get("errors", {})
+            message = response.json().get("message")
+        except:
+            message = "Unknown API error."
+            error = "Unknown API error."
+    elif response_format=='csv':
+        error = response.content.decode('utf-8')
+        message = response.content.decode('utf-8')
     code = response.status_code
     data = error
 

alerce/utils.py

@@ -1,22 +1,69 @@
-from pandas import DataFrame
+from pandas import DataFrame, read_csv
+from io import StringIO
 from astropy.table import Table
 from .exceptions import handle_error, FormatValidationError
 import requests
+import abc
 
 
 class Result:
-    def __init__(self, json_result, format="json"):
-        self.json_result = json_result
+    def __init__(self, format="json"):
         self.format = format
 
+    @abc.abstractmethod
+    def to_pandas(self, index=None, sort=None):
+        """Convert the result to a pandas dataframe
+
+        :index: index for the pandas dataframe
+        :sort: sorting column for the dataframe
+        :returns: the processed dataframe
+
+        """
+        pass
+
+    @abc.abstractmethod
+    def to_votable(self):
+        pass
+
+    @abc.abstractmethod
+    def to_json(self):
+       pass
+
+    @abc.abstractmethod
+    def to_csv(self, index=None, sort=None):
+        pass
+
+    def result(self, index=None, sort=None):
+        """Creates the result depending on the arguments and the expected format
+
+        :index: if format is pandas, this is the index column
+        :sort: if format is pandas, this is the sort column
+        :returns: Result in the indicated formar
+
+        """
+        if self.format == "pandas":
+            return self.to_pandas(index, sort)
+        if self.format == "votable":
+            return self.to_votable()
+        if self.format == "csv":
+            return self.to_csv()
+        if self.format == "json":
+            return self.to_json()
+
+class ResultJson(Result):
+    """Object that holds a json type result"""
+    def __init__(self, json_result, **kwargs):
+        self.json_result = json_result
+        super().__init__(**kwargs)
+        print(self.format)
+
     def to_pandas(self, index=None, sort=None):
         dataframe = None
         if isinstance(self.json_result, list):
             dataframe = DataFrame(self.json_result)
         else:
             dataframe = DataFrame([self.json_result])
-        if sort:
-            dataframe.sort_values(sort, inplace=True)
+        if sort: dataframe.sort_values(sort, inplace=True)
         if index:
             dataframe.set_index(index, inplace=True)
         return dataframe
@@ -27,21 +74,63 @@ def to_votable(self):
     def to_json(self):
         return self.json_result
 
-    def result(self, index=None, sort=None):
-        if self.format == "json":
-            return self.to_json()
-        if self.format == "pandas":
-            return self.to_pandas(index, sort)
-        if self.format == "votable":
-            return self.to_votable()
+    def to_csv(self):
+        df = self.to_pandas()
+        return df.to_csv(index=False)
+
+
+class ResultCsv(Result):
+
+    """Object that holds a csv type result"""
+
+    def __init__(self, csv_result_byte, **kwargs):
+        self.csv_result = csv_result_byte.decode('utf-8')
+        self.data = [x.split(',') for x in self.csv_result.split('\n')[1:-1]]
+
+        self.columns = [x for x in self.csv_result.split('\n')[0].split(',')]
+        super().__init__(**kwargs)
 
+    def _rename_duplicates(self):
+        counts = {}
+        columns = self.columns.copy()
+        for i, column in enumerate(columns):
+            if column in counts:
+                counts[column] += 1
+                columns[i] = f"{column}_{counts[column] - 1}"
+            else:
+                counts[column] = 1
+        return columns
+
+
+    def to_pandas(self, index=None, sort=None):
+        dataframe = read_csv(StringIO(self.csv_result))
+        if sort: dataframe.sort_values(sort, inplace=True)
+        if index:
+            dataframe.set_index(index, inplace=True)
+        return dataframe
+
+    def to_votable(self):
+        # TODO: Check if renaming the columns doesn't cause problems to the user
+        columns = self._rename_duplicates()
+        df = read_csv(StringIO(self.csv_result), names=columns, skiprows=1)
+        df = df.convert_dtypes()
+        table = Table.from_pandas(df)
+        return table
+
+    def to_json(self):
+        columns = self._rename_duplicates()
+        df = read_csv(StringIO(self.csv_result), names=columns, skiprows=1)
+        return df.to_json(orient='records')
+
+    def to_csv(self):
+        return self.csv_result
 
 class Client:
     def __init__(self, **kwargs):
         self.session = requests.Session()
         self.config = {}
         self.config.update(kwargs)
-        self.allowed_formats = ["pandas", "votable", "json"]
+        self.allowed_formats = ["pandas", "votable", "json", "csv"]
 
     def load_config_from_file(self, path):
         pass
@@ -58,13 +147,16 @@ def _validate_format(self, format):
         return format
 
     def _request(
-        self, method, url, params=None, response_field=None, result_format="json"
+        self, method, url, params=None, data=None, response_field=None, result_format="json", response_format="json"
     ):
         result_format = self._validate_format(result_format)
-        resp = self.session.request(method, url, params=params)
 
+        resp = self.session.request(method, url, params=params, data=data)
         if resp.status_code >= 400:
-            handle_error(resp)
-        if response_field and result_format != "json":
-            return Result(resp.json()[response_field], format=result_format)
-        return Result(resp.json(), format=result_format)
+            handle_error(resp, response_format)
+
+        if response_format == 'csv':
+            return ResultCsv(resp, format=result_format)
+        if response_field and result_format != "json" and result_format != 'csv':
+            return ResultJson(resp.json()[response_field], format=result_format)
+        return ResultJson(resp.json(), format=result_format)