diff --git a/README.md b/README.md index 3f9ed307..7ada1bb1 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ A locally-focused workflow (local development, local execution) with the CLI may - [`lean cloud object-store get`](#lean-cloud-object-store-get) - [`lean cloud object-store list`](#lean-cloud-object-store-list) - [`lean cloud object-store ls`](#lean-cloud-object-store-ls) +- [`lean cloud object-store properties`](#lean-cloud-object-store-properties) - [`lean cloud object-store set`](#lean-cloud-object-store-set) - [`lean cloud optimize`](#lean-cloud-optimize) - [`lean cloud pull`](#lean-cloud-pull) @@ -112,6 +113,7 @@ A locally-focused workflow (local development, local execution) with the CLI may - [`lean object-store get`](#lean-object-store-get) - [`lean object-store list`](#lean-object-store-list) - [`lean object-store ls`](#lean-object-store-ls) +- [`lean object-store properties`](#lean-object-store-properties) - [`lean object-store set`](#lean-object-store-set) - [`lean optimize`](#lean-optimize) - [`lean project-create`](#lean-project-create) @@ -450,6 +452,8 @@ Usage: lean cloud object-store delete [OPTIONS] KEY Delete a value from the organization's cloud object store. + :param key: The desired key name to delete. + Options: --verbose Enable debug logging --help Show this message and exit. @@ -459,16 +463,20 @@ _See code: [lean/commands/cloud/object_store/delete.py](lean/commands/cloud/obje ### `lean cloud object-store get` -Get a value from the organization's cloud object store. +Download an object store value to disk from the organization's cloud object store. ``` -Usage: lean cloud object-store get [OPTIONS] KEY +Usage: lean cloud object-store get [OPTIONS] [KEY]... + + Download an object store value to disk from the organization's cloud object store. - Get a value from the organization's cloud object store. + :param key: The desired key to fetch, multiple can be provided. Options: - --verbose Enable debug logging - --help Show this message and exit. + --destination-folder TEXT The destination folder to download the object store values, if not provided will use to + current directory + --verbose Enable debug logging + --help Show this message and exit. ``` _See code: [lean/commands/cloud/object_store/get.py](lean/commands/cloud/object_store/get.py)_ @@ -482,6 +490,8 @@ Usage: lean cloud object-store list [OPTIONS] [KEY] List all values for the given root key in the organization's cloud object store. + :param key: The desired root key to list. + Options: --verbose Enable debug logging --help Show this message and exit. @@ -498,6 +508,8 @@ Usage: lean cloud object-store ls [OPTIONS] [KEY] List all values for the given root key in the organization's cloud object store. + :param key: The desired root key to list. + Options: --verbose Enable debug logging --help Show this message and exit. @@ -505,6 +517,24 @@ Options: _See code: [lean/commands/cloud/object_store/ls.py](lean/commands/cloud/object_store/ls.py)_ +### `lean cloud object-store properties` + +Get a value properties from the organization's cloud object store. + +``` +Usage: lean cloud object-store properties [OPTIONS] KEY + + Get a value properties from the organization's cloud object store. + + :param key: The desired key to fetch the properties for. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/cloud/object_store/properties.py](lean/commands/cloud/object_store/properties.py)_ + ### `lean cloud object-store set` Sets the data to the given key in the organization's cloud object store. @@ -1427,6 +1457,22 @@ Options: _See code: [lean/commands/object_store/ls.py](lean/commands/object_store/ls.py)_ +### `lean object-store properties` + +Opens the local storage directory in the file explorer. + +``` +Usage: lean object-store properties [OPTIONS] + + Opens the local storage directory in the file explorer. + +Options: + --verbose Enable debug logging + --help Show this message and exit. +``` + +_See code: [lean/commands/object_store/properties.py](lean/commands/object_store/properties.py)_ + ### `lean object-store set` Opens the local storage directory in the file explorer. diff --git a/lean/commands/cloud/object_store/__init__.py b/lean/commands/cloud/object_store/__init__.py index 8d4e7297..2f31f91e 100644 --- a/lean/commands/cloud/object_store/__init__.py +++ b/lean/commands/cloud/object_store/__init__.py @@ -16,9 +16,10 @@ from lean.commands.cloud.object_store.set import set from lean.commands.cloud.object_store.list import list from lean.commands.cloud.object_store.delete import delete +from lean.commands.cloud.object_store.properties import properties object_store.add_command(get) object_store.add_command(set) object_store.add_command(list) object_store.add_command(delete) - +object_store.add_command(properties) diff --git a/lean/commands/cloud/object_store/delete.py b/lean/commands/cloud/object_store/delete.py index 4d2633f2..fa67d9ff 100644 --- a/lean/commands/cloud/object_store/delete.py +++ b/lean/commands/cloud/object_store/delete.py @@ -22,8 +22,9 @@ def delete(key: str) -> str: """ Delete a value from the organization's cloud object store. - + + :param key: The desired key name to delete. """ organization_id = container.organization_manager.try_get_working_organization_id() api_client = container.api_client - api_client.object_store.delete(key, organization_id) \ No newline at end of file + api_client.object_store.delete(key, organization_id) diff --git a/lean/commands/cloud/object_store/get.py b/lean/commands/cloud/object_store/get.py index 951a277b..b2a03477 100644 --- a/lean/commands/cloud/object_store/get.py +++ b/lean/commands/cloud/object_store/get.py @@ -11,45 +11,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -from click import command, argument +from uuid import uuid4 +from os import path, getcwd, unlink, mkdir + +from click import command, option, argument from lean.click import LeanCommand from lean.container import container @command(cls=LeanCommand) -@argument("key", type=str) -def get(key: str) -> str: +@argument("key", type=str, nargs=-1) +@option("--destination-folder", type=str, default="", + help=f"The destination folder to download the object store values," + f" if not provided will use to current directory") +def get(key: [str], destination_folder: str): """ - Get a value from the organization's cloud object store. + Download an object store value to disk from the organization's cloud object store. + :param key: The desired key to fetch, multiple can be provided. """ organization_id = container.organization_manager.try_get_working_organization_id() api_client = container.api_client logger = container.logger - data = api_client.object_store.get(key, organization_id) - - try: - headers = ["size", "modified", "key", "preview"] - display_headers = ["Bytes", "Modified", "Filename", "Preview"] - data_row = [] - for header in headers: - if header == "preview": - value = str(data["metadata"].get(header, "N/A")) - data_row.append(_clean_up_preview(value)) - else: - value = str(data["metadata"].get(header, "")) - data_row.append(value) - all_rows = [display_headers] + [data_row] - column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] - for row in all_rows: - logger.info(" ".join(value.ljust(width) for value, width in zip(row, column_widths))) - except KeyError as e: - logger.error(f"Key {key} not found.") - except Exception as e: - logger.error(f"Error: {e}") - - -def _clean_up_preview(preview: str) -> str: - return preview.rstrip()[:10] + logger.info(f"Fetching object store download url") + url = api_client.object_store.get(key, organization_id, logger) + if not destination_folder: + destination_folder = getcwd() + + if not path.exists(destination_folder): + mkdir(destination_folder) + + temp_file = path.join(destination_folder, f"{str(uuid4())}.zip") + + with logger.transient_progress() as progress: + progress.add_task(f"Start downloading keys into {temp_file}:", total=None) + logger.debug(f"Downloading: {url}") + + api_client.data.download_url(url, temp_file, lambda advance: None) + + logger.info(f"Unzipping object store keys values into: '{destination_folder}'") + from zipfile import ZipFile + with ZipFile(temp_file, 'r') as zip_ref: + zip_ref.extractall(destination_folder) + if path.exists(temp_file): + logger.debug(f"Deleting temp file: '{temp_file}'") + unlink(temp_file) diff --git a/lean/commands/cloud/object_store/list.py b/lean/commands/cloud/object_store/list.py index 83a08b05..7de4fe5e 100644 --- a/lean/commands/cloud/object_store/list.py +++ b/lean/commands/cloud/object_store/list.py @@ -19,9 +19,11 @@ @object_store.command(cls=LeanCommand, name="list", aliases=["ls"]) @argument("key", type=str, default="/") -def list(key: str) -> str: +def list(key: str): """ List all values for the given root key in the organization's cloud object store. + + :param key: The desired root key to list. """ organization_id = container.organization_manager.try_get_working_organization_id() api_client = container.api_client @@ -41,4 +43,4 @@ def list(key: str) -> str: except KeyError as e: logger.error(f"Key {key} not found.") except Exception as e: - logger.error(f"Error: {e}") \ No newline at end of file + logger.error(f"Error: {e}") diff --git a/lean/commands/cloud/object_store/properties.py b/lean/commands/cloud/object_store/properties.py new file mode 100644 index 00000000..3e42ac57 --- /dev/null +++ b/lean/commands/cloud/object_store/properties.py @@ -0,0 +1,54 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command, argument +from lean.click import LeanCommand +from lean.container import container + + +@command(cls=LeanCommand) +@argument("key", type=str) +def properties(key: str): + """ + Get a value properties from the organization's cloud object store. + + :param key: The desired key to fetch the properties for. + """ + organization_id = container.organization_manager.try_get_working_organization_id() + api_client = container.api_client + logger = container.logger + data = api_client.object_store.properties(key, organization_id) + + try: + headers = ["size", "modified", "key", "preview"] + display_headers = ["Bytes", "Modified", "Filename", "Preview"] + data_row = [] + for header in headers: + if header == "preview": + value = str(data["metadata"].get(header, "N/A")) + data_row.append(_clean_up_preview(value)) + else: + value = str(data["metadata"].get(header, "")) + data_row.append(value) + all_rows = [display_headers] + [data_row] + column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] + for row in all_rows: + logger.info(" ".join(value.ljust(width) for value, width in zip(row, column_widths))) + except KeyError as e: + logger.error(f"Key {key} not found.") + except Exception as e: + logger.error(f"Error: {e}") + + +def _clean_up_preview(preview: str) -> str: + return preview.rstrip()[:10] diff --git a/lean/commands/cloud/object_store/set.py b/lean/commands/cloud/object_store/set.py index 6cfd7279..5794af05 100644 --- a/lean/commands/cloud/object_store/set.py +++ b/lean/commands/cloud/object_store/set.py @@ -31,4 +31,4 @@ def set(key: str, path: Path) -> None: api_client = container.api_client with open(path, "rb") as file: bytes_data: bytes = file.read() - api_client.object_store.set(key, bytes_data, organization_id) \ No newline at end of file + api_client.object_store.set(key, bytes_data, organization_id) diff --git a/lean/commands/object_store/__init__.py b/lean/commands/object_store/__init__.py index d6108b87..ec091e26 100644 --- a/lean/commands/object_store/__init__.py +++ b/lean/commands/object_store/__init__.py @@ -16,9 +16,10 @@ from lean.commands.object_store.set import set from lean.commands.object_store.list import list from lean.commands.object_store.delete import delete +from lean.commands.object_store.properties import properties object_store.add_command(get) object_store.add_command(set) object_store.add_command(list) object_store.add_command(delete) - +object_store.add_command(properties) diff --git a/lean/commands/object_store/properties.py b/lean/commands/object_store/properties.py new file mode 100644 index 00000000..1e7cdb95 --- /dev/null +++ b/lean/commands/object_store/properties.py @@ -0,0 +1,26 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from click import command +from lean.click import LeanCommand +from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer + + +@command(cls=LeanCommand) +def properties() -> str: + """ + Opens the local storage directory in the file explorer. + """ + open_storage_directory_in_explorer(container.lean_config_manager) diff --git a/lean/components/api/backtest_client.py b/lean/components/api/backtest_client.py index 4b49ff70..1f3abc6d 100644 --- a/lean/components/api/backtest_client.py +++ b/lean/components/api/backtest_client.py @@ -11,10 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List - from lean.components.api.api_client import * -from lean.models.api import QCBacktest, QCBacktestReport +from lean.models.api import QCBacktest class BacktestClient: @@ -41,18 +39,6 @@ def get(self, project_id: int, backtest_id: str) -> QCBacktest: return QCBacktest(**data["backtest"]) - def get_all(self, project_id: int) -> List[QCBacktest]: - """Returns all backtests in a project. - - :param project_id: the id of the project to retrieve the backtests of - :return: the backtests in the specified project - """ - data = self._api.get("backtests/read", { - "projectId": project_id - }) - - return [QCBacktest(**backtest) for backtest in data["backtests"]] - def create(self, project_id: int, compile_id: str, name: str) -> QCBacktest: """Creates a new backtest. @@ -71,35 +57,6 @@ def create(self, project_id: int, compile_id: str, name: str) -> QCBacktest: return QCBacktest(**data["backtest"]) - def get_report(self, project_id: int, backtest_id: str) -> QCBacktestReport: - """Returns the report of a backtest. - - :param project_id: the id of the project the backtest belongs to - :param backtest_id: the id of the backtest to retrieve the report of - :return: the report of the specified backtest - """ - data = self._api.post("backtests/read/report", { - "projectId": project_id, - "backtestId": backtest_id - }) - - return QCBacktestReport(**data) - - def update(self, project_id: int, backtest_id: str, name: str, note: str) -> None: - """Updates an existing backtest. - - :param project_id: the id of the project the backtest belongs to - :param backtest_id: the id of the backtest to update - :param name: the new name to assign to the backtest - :param note: the new note to assign to the backtest - """ - self._api.post("backtests/update", { - "projectId": project_id, - "backtestId": backtest_id, - "name": name, - "note": note - }) - def delete(self, project_id: int, backtest_id: str) -> None: """Deletes an existing backtest. diff --git a/lean/components/api/compile_client.py b/lean/components/api/compile_client.py index 88dd7c55..ccbb1371 100644 --- a/lean/components/api/compile_client.py +++ b/lean/components/api/compile_client.py @@ -12,7 +12,7 @@ # limitations under the License. from lean.components.api.api_client import * -from lean.models.api import QCCompileWithLogs, QCCompileWithParameters +from lean.models.api import QCCompileWithLogs, QCCompile class CompileClient: @@ -39,7 +39,7 @@ def get(self, project_id: int, compile_id: str) -> QCCompileWithLogs: return QCCompileWithLogs(**data) - def create(self, project_id: int) -> QCCompileWithParameters: + def create(self, project_id: int) -> QCCompile: """Creates a new compile. :param project_id: the id of the project to create a compile for @@ -49,4 +49,4 @@ def create(self, project_id: int) -> QCCompileWithParameters: "projectId": project_id }) - return QCCompileWithParameters(**data) + return QCCompile(**data) diff --git a/lean/components/api/data_client.py b/lean/components/api/data_client.py index cbdece2f..60355fab 100644 --- a/lean/components/api/data_client.py +++ b/lean/components/api/data_client.py @@ -39,18 +39,26 @@ def download_file(self, relative_file_path: str, organization_id: str, :param local_filename: the final local path where the data file will be stored :param progress_callback: the download progress callback """ - from tempfile import NamedTemporaryFile - from shutil import move - from os import path, makedirs - data = self._api.post("data/read", { "format": "link", "filePath": relative_file_path, "organizationId": organization_id }) + self.download_url(data["link"], local_filename, progress_callback) + + def download_url(self, url: str, local_filename: str, progress_callback: Callable[[float], None]) -> None: + """Downloads the content of a downloadable file. + :param url: the url to download + :param local_filename: the final local path where the data file will be stored + :param progress_callback: the download progress callback + """ + from tempfile import NamedTemporaryFile + from shutil import move + from os import path, makedirs + # we stream the data into a temporary file and later move it to it's final location - with self._http_client.get(data["link"], stream=True) as r: + with self._http_client.get(url, stream=True) as r: r.raise_for_status() total_size = 0 try: diff --git a/lean/components/api/node_client.py b/lean/components/api/node_client.py index 5b4986a7..775eb3ae 100644 --- a/lean/components/api/node_client.py +++ b/lean/components/api/node_client.py @@ -12,7 +12,7 @@ # limitations under the License. from lean.components.api.api_client import * -from lean.models.api import QCNode, QCNodeList +from lean.models.api import QCNodeList class NodeClient: @@ -36,55 +36,3 @@ def get_all(self, organization_id: str) -> QCNodeList: }) return QCNodeList(**data) - - def create(self, organization_id: str, name: str, sku: str) -> QCNode: - """Creates a new node. - - :param organization_id: the id of the organization to create the node in - :param name: the name of the node to create - :param sku: the sku of the node to create - :return: the created node - """ - data = self._api.post("nodes/create", { - "organizationId": organization_id, - "name": name, - "sku": sku - }) - - return QCNode(**data["node"]) - - def update(self, organization_id: str, node_id: str, new_name: str) -> None: - """Updates an existing node. - - :param organization_id: the id of the organization the node belongs to - :param node_id: the id of the node to update - :param new_name: the new name to assign to the node - :return: the updated node - """ - self._api.post("nodes/update", { - "organizationId": organization_id, - "nodeId": node_id, - "name": new_name - }) - - def delete(self, organization_id: str, node_id: str) -> None: - """Deletes an existing node. - - :param organization_id: the id of the organization the node belongs to - :param node_id: the id of the node to delete - """ - self._api.post("nodes/delete", { - "organizationId": organization_id, - "nodeId": node_id - }) - - def stop(self, organization_id: str, node_id: str) -> None: - """Stops the current activity on a node. - - :param organization_id: the id of the organization the node belongs to - :param node_id: the id of the node to stop the current activity for - """ - self._api.post("nodes/stop", { - "organizationId": organization_id, - "nodeId": node_id - }) diff --git a/lean/components/api/object_store_client.py b/lean/components/api/object_store_client.py index fb43c216..76cd6bb5 100644 --- a/lean/components/api/object_store_client.py +++ b/lean/components/api/object_store_client.py @@ -12,6 +12,7 @@ # limitations under the License. from lean.components.api.api_client import * +from lean.components.util.logger import Logger class ObjectStoreClient: @@ -24,28 +25,65 @@ def __init__(self, api_client: 'APIClient') -> None: """ self._api = api_client - def get(self, key: str, organization_id: str) -> str: + def properties(self, key: str, organization_id: str) -> str: """Returns the details of key from the object store. - :param key: key of the object to retrieve :param organization_id: the id of the organization who's object store to retrieve from :return: the details of the specified object """ - payload = { "organizationId": organization_id, "key": key, } - data = self._api.post("object/get", payload) + return self._api.post("object/properties", payload) - return data + def get(self, keys: [str], organization_id: str, logger: Logger) -> str: + """Will fetch an url to download the requested keys + :param keys: keys of the object to retrieve + :param organization_id: the id of the organization who's object store to retrieve from + :param logger: the logger instance to use + :return: the url to download the requested keys + """ - def set(self, key: str, objectData: bytes, organization_id: str) -> None: - """Sets the given key in the Object Store. + payload = { + "organizationId": organization_id, + "keys": keys, + } + data = self._api.post("object/get", payload) + if "url" in data and data["url"]: + # we got the url right away + return data["url"] + + job_id = data["jobId"] + if job_id: + from time import sleep + payload = { + "organizationId": organization_id, + "jobId": job_id, + } + + with logger.transient_progress() as progress: + progress.add_task("Waiting for files to be ready for download:", total=None) + + # we will retry up to 5 min to get the url + retry_count = 0 + while not data["url"]: + sleep(3) + data = self._api.post("object/get", payload) + retry_count = retry_count + 1 + if retry_count > ((60 * 5) / 3): + raise TimeoutError(f"Timeout waiting for object store job id {job_id}, please contact support") + else: + raise ValueError("JobId was not found in API response") + + return data["url"] + + def set(self, key: str, object_data: bytes, organization_id: str) -> None: + """Sets the given key in the Object Store. :param key: key of the object to set - :param objectData: the data to set + :param object_data: the data to set :param organization_id: the id of the organization who's object store to set data in """ payload = { @@ -53,7 +91,7 @@ def set(self, key: str, objectData: bytes, organization_id: str) -> None: "key": key } extra_options = { - "files": {"objectData": objectData} + "files": {"objectData": object_data} } self._api.post("object/set", payload, False, extra_options) @@ -72,7 +110,7 @@ def list(self, path: str, organization_id: str) -> str: data = self._api.post("object/list", payload) return data - + def delete(self, key: str, organization_id: str) -> None: """Deletes the given key from the Object Store. @@ -84,4 +122,4 @@ def delete(self, key: str, organization_id: str) -> None: "key": key } - self._api.post("object/delete", payload) \ No newline at end of file + self._api.post("object/delete", payload) diff --git a/lean/components/cloud/cloud_runner.py b/lean/components/cloud/cloud_runner.py index 2bc6fbb2..cb980377 100644 --- a/lean/components/cloud/cloud_runner.py +++ b/lean/components/cloud/cloud_runner.py @@ -126,22 +126,6 @@ def compile_project(self, project: QCProject) -> QCCompileWithLogs: created_compile = self._api_client.compiles.create(project.projectId) - # Log the parameters reported in the compile - parameters = [] - parameter_count = 0 - - for parameter_container in created_compile.parameters: - for parameter in parameter_container.parameters: - parameters.append(f"- {parameter_container.file}:{parameter.line} :: {parameter.type}") - parameter_count += int(parameter.type.split(" ")[0]) - - if parameter_count > 0: - self._logger.info(f"Detected parameters ({parameter_count}):") - for parameter in parameters: - self._logger.info(parameter) - else: - self._logger.info("Detected parameters: none") - finished_compile = self._task_manager.poll( make_request=lambda: self._api_client.compiles.get(project.projectId, created_compile.compileId), is_done=lambda data: data.state in [QCCompileState.BuildSuccess, QCCompileState.BuildError] diff --git a/lean/components/util/logger.py b/lean/components/util/logger.py index 631bae95..49face72 100644 --- a/lean/components/util/logger.py +++ b/lean/components/util/logger.py @@ -56,6 +56,18 @@ def error(self, message: Any) -> None: """ self._console.print(message, style="red") + def transient_progress(self, prefix: str = ""): + """Creates a Progress instance. + + :param prefix: the text to show before the bar (defaults to a blank string) + :return: a Progress instance which can be used to display progress bars + """ + from rich.progress import BarColumn, Progress, TextColumn + progress = Progress(TextColumn(prefix), BarColumn(), + console=self._console, transient=True) + progress.start() + return progress + def progress(self, prefix: str = "", suffix: str = "{task.percentage:0.0f}%"): """Creates a Progress instance. diff --git a/lean/models/api.py b/lean/models/api.py index 32a3c0c0..3033815c 100644 --- a/lean/models/api.py +++ b/lean/models/api.py @@ -126,32 +126,19 @@ class QCMinimalFile(WrappedBaseModel): modified: datetime -class QCCompileParameter(WrappedBaseModel): - line: int - type: str - - -class QCCompileParameterContainer(WrappedBaseModel): - file: str - parameters: List[QCCompileParameter] - - class QCCompileState(str, Enum): InQueue = "InQueue" BuildSuccess = "BuildSuccess" BuildError = "BuildError" -class QCCompileWithLogs(WrappedBaseModel): +class QCCompile(WrappedBaseModel): compileId: str state: QCCompileState - logs: List[str] -class QCCompileWithParameters(WrappedBaseModel): - compileId: str - state: QCCompileState - parameters: List[QCCompileParameterContainer] +class QCCompileWithLogs(QCCompile): + logs: List[str] class QCBacktest(WrappedBaseModel): @@ -238,10 +225,6 @@ def get_statistics_table(self): return table -class QCBacktestReport(WrappedBaseModel): - report: str - - class QCNodePrice(WrappedBaseModel): monthly: int yearly: int diff --git a/tests/commands/cloud/object_store/test_get.py b/tests/commands/cloud/object_store/test_get.py index 37ba75ce..bd5b0a4c 100644 --- a/tests/commands/cloud/object_store/test_get.py +++ b/tests/commands/cloud/object_store/test_get.py @@ -10,7 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import zipfile +from os import mkdir +from pathlib import Path from unittest import mock from click.testing import CliRunner @@ -20,11 +22,17 @@ from tests.conftest import initialize_container -def test_get_gets_value_when_key_is_given() -> None: +@mock.patch("zipfile.ZipFile") +def test_get_gets_value_when_key_is_given(mock_zipfile) -> None: + archive = mock.Mock() + mocked_read = mock.Mock() + archive.return_value.read = mocked_read + mock_zipfile.return_value.__enter__ = archive + api_client = mock.Mock() api_client.is_authenticated.return_value = True initialize_container(api_client_to_use=api_client) result = CliRunner().invoke(lean, ["cloud", "object-store", "get", "test-key"]) assert result.exit_code == 0 - container.api_client.object_store.get.assert_called_once_with('test-key', 'abc') + container.api_client.object_store.get.assert_called_once_with(('test-key',), 'abc', container.logger) diff --git a/tests/components/api/test_clients.py b/tests/components/api/test_clients.py index 22ba1d34..5ea171d1 100644 --- a/tests/components/api/test_clients.py +++ b/tests/components/api/test_clients.py @@ -203,11 +203,6 @@ def test_backtest_crud() -> None: break sleep(1) - # Ensure we have a backtest node to run on - backtest_nodes = node_client.get_all(project.organizationId).backtest - if all(node.busy for node in backtest_nodes): - node_client.stop(project.organizationId, backtest_nodes[0].id) - # Test a backtest can be started backtest_name = f"Test Backtest {datetime.now()}" created_backtest = backtest_client.create(project.projectId, created_compile.compileId, backtest_name) @@ -226,22 +221,9 @@ def test_backtest_crud() -> None: assert retrieved_backtest.backtestId == created_backtest.backtestId assert retrieved_backtest.name == backtest_name - # Test the backtest can be updated - backtest_client.update(project.projectId, created_backtest.backtestId, backtest_name + "2", "This is a note") - retrieved_backtest = backtest_client.get(project.projectId, created_backtest.backtestId) - - assert retrieved_backtest.backtestId == created_backtest.backtestId - assert retrieved_backtest.name == backtest_name + "2" - assert retrieved_backtest.note == "This is a note" - # Test the backtest can be deleted backtest_client.delete(project.projectId, created_backtest.backtestId) - # Test the backtest is really deleted - backtests = backtest_client.get_all(project.projectId) - - assert not any([backtest.backtestId == created_backtest.backtestId for backtest in backtests]) - def test_live_client_get_all_parses_response() -> None: api_client = create_api_client() @@ -253,20 +235,6 @@ def test_live_client_get_all_parses_response() -> None: live_client.get_all() -def test_node_client_get_all_parses_response() -> None: - api_client = create_api_client() - account_client = AccountClient(api_client) - node_client = NodeClient(api_client) - - # Retrieve the organization details - organization = account_client.get_organization() - - # Test nodes can be retrieved - # All we can do here is make sure NodeClient can parse the response - # Tests aren't supposed to trigger actions which cost money, so we can't create a node here - node_client.get_all(organization.organizationId) - - def test_account_client_get_organization() -> None: api_client = create_api_client() account_client = AccountClient(api_client)