From 96876a1f228e36957ad22c93bc35395da06cfd45 Mon Sep 17 00:00:00 2001 From: Paras Jain Date: Fri, 28 Jan 2022 15:21:34 -0800 Subject: [PATCH] Migrate from loguru to print (#104) * Migrate from loguru to print * Update dependencies for logger --- nb/01-04-2021_plot_udp_throughput.ipynb | 2 +- nb/01-21-2022_visualize_pareto.ipynb | 2 +- nb/11-16-2021_plot_latency.ipynb | 2 +- nb/11-16-2021_plot_throughput.ipynb | 2 +- nb/11-22-2021_plot_throughput_time.ipynb | 2 +- nb/11-22-2021_throughput_solver.ipynb | 2 +- nb/12-02-2021_gcp_premium_standard.ipynb | 2 +- nb/12-11-2021_get_transfer_cost.ipynb | 2 +- nb/12-12-2021_visualize_pareto.ipynb | 2 +- scripts/requirements-gateway.txt | 2 +- setup.py | 2 +- skylark/benchmark/network/latency.py | 2 +- skylark/benchmark/network/throughput.py | 2 +- skylark/benchmark/network/traceroute.py | 2 +- .../replicate/benchmark_triangles.py | 2 +- skylark/benchmark/replicate/test_direct.py | 2 +- skylark/benchmark/stop_all_instances.py | 2 +- skylark/benchmark/utils.py | 2 +- skylark/chunk.py | 2 +- skylark/cli/cli.py | 8 +++--- skylark/cli/cli_aws.py | 8 ++---- skylark/cli/cli_azure.py | 6 +---- skylark/cli/cli_helper.py | 2 +- skylark/cli/cli_solver.py | 6 +---- skylark/compute/aws/aws_cloud_provider.py | 2 +- skylark/compute/aws/aws_server.py | 2 +- skylark/compute/azure/azure_cloud_provider.py | 2 +- skylark/compute/gcp/gcp_cloud_provider.py | 2 +- skylark/compute/server.py | 8 +++--- skylark/gateway/chunk_store.py | 2 +- skylark/gateway/gateway_daemon.py | 24 +++-------------- skylark/gateway/gateway_daemon_api.py | 17 ++---------- skylark/gateway/gateway_receiver.py | 2 +- skylark/gateway/gateway_sender.py | 2 +- skylark/obj_store/azure_interface.py | 10 ++----- skylark/replicate/replication_plan.py | 2 +- skylark/replicate/replicator_client.py | 2 +- skylark/replicate/solver.py | 2 +- skylark/test/test_gateway_daemon.py | 2 +- skylark/test/test_replicator_client.py | 6 ++--- skylark/utils/logger.py | 27 +++++++++++++++++++ skylark/utils/utils.py | 2 +- 42 files changed, 79 insertions(+), 105 deletions(-) create mode 100644 skylark/utils/logger.py diff --git a/nb/01-04-2021_plot_udp_throughput.ipynb b/nb/01-04-2021_plot_udp_throughput.ipynb index cc248cba0..87a569c32 100644 --- a/nb/01-04-2021_plot_udp_throughput.ipynb +++ b/nb/01-04-2021_plot_udp_throughput.ipynb @@ -19,7 +19,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "\n", "import pickle\n", "import json\n", diff --git a/nb/01-21-2022_visualize_pareto.ipynb b/nb/01-21-2022_visualize_pareto.ipynb index aa75a6939..dbe0cc20d 100644 --- a/nb/01-21-2022_visualize_pareto.ipynb +++ b/nb/01-21-2022_visualize_pareto.ipynb @@ -10,7 +10,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "\n", "import seaborn as sns\n", "\n", diff --git a/nb/11-16-2021_plot_latency.ipynb b/nb/11-16-2021_plot_latency.ipynb index edf158885..2980c0825 100644 --- a/nb/11-16-2021_plot_latency.ipynb +++ b/nb/11-16-2021_plot_latency.ipynb @@ -19,7 +19,7 @@ "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "\n", "import pickle\n", "import json\n", diff --git a/nb/11-16-2021_plot_throughput.ipynb b/nb/11-16-2021_plot_throughput.ipynb index b1e0ec8e4..1f83a90ee 100644 --- a/nb/11-16-2021_plot_throughput.ipynb +++ b/nb/11-16-2021_plot_throughput.ipynb @@ -19,7 +19,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "\n", "import pickle\n", "import json\n", diff --git a/nb/11-22-2021_plot_throughput_time.ipynb b/nb/11-22-2021_plot_throughput_time.ipynb index dceb28a0b..36fa2e099 100644 --- a/nb/11-22-2021_plot_throughput_time.ipynb +++ b/nb/11-22-2021_plot_throughput_time.ipynb @@ -19,7 +19,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "from tqdm import tqdm\n", "\n", "import pickle\n", diff --git a/nb/11-22-2021_throughput_solver.ipynb b/nb/11-22-2021_throughput_solver.ipynb index e812719c8..b9a8ee7f7 100644 --- a/nb/11-22-2021_throughput_solver.ipynb +++ b/nb/11-22-2021_throughput_solver.ipynb @@ -20,7 +20,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "from tqdm import tqdm\n", "\n", "import pickle\n", diff --git a/nb/12-02-2021_gcp_premium_standard.ipynb b/nb/12-02-2021_gcp_premium_standard.ipynb index 5bc265db1..14389a6cf 100644 --- a/nb/12-02-2021_gcp_premium_standard.ipynb +++ b/nb/12-02-2021_gcp_premium_standard.ipynb @@ -19,7 +19,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "\n", "import pickle\n", "import json\n", diff --git a/nb/12-11-2021_get_transfer_cost.ipynb b/nb/12-11-2021_get_transfer_cost.ipynb index 72ca329fa..e620bced7 100644 --- a/nb/12-11-2021_get_transfer_cost.ipynb +++ b/nb/12-11-2021_get_transfer_cost.ipynb @@ -19,7 +19,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "\n", "import pickle\n", "import requests\n", diff --git a/nb/12-12-2021_visualize_pareto.ipynb b/nb/12-12-2021_visualize_pareto.ipynb index 368b9604d..849330d2b 100644 --- a/nb/12-12-2021_visualize_pareto.ipynb +++ b/nb/12-12-2021_visualize_pareto.ipynb @@ -20,7 +20,7 @@ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from loguru import logger\n", + "from skylark.utils import logger\n", "\n", "import seaborn as sns\n", "\n", diff --git a/scripts/requirements-gateway.txt b/scripts/requirements-gateway.txt index 8326ed42c..842c33a79 100644 --- a/scripts/requirements-gateway.txt +++ b/scripts/requirements-gateway.txt @@ -9,8 +9,8 @@ flask google-api-python-client google-cloud-compute google-cloud-storage -loguru setproctitle +termcolor tqdm werkzeug oslo.concurrency diff --git a/setup.py b/setup.py index 73c5ccdd0..1ee6eb440 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,8 @@ "google-api-python-client", "google-cloud-compute", "google-cloud-storage", - "loguru", "setproctitle", + "termcolor", "tqdm", "werkzeug", ], diff --git a/skylark/benchmark/network/latency.py b/skylark/benchmark/network/latency.py index 0640498f3..21bc8d157 100644 --- a/skylark/benchmark/network/latency.py +++ b/skylark/benchmark/network/latency.py @@ -3,7 +3,7 @@ import re from typing import List, Tuple -from loguru import logger +from skylark.utils import logger from tqdm import tqdm from skylark import skylark_root diff --git a/skylark/benchmark/network/throughput.py b/skylark/benchmark/network/throughput.py index 5e720d973..0c4d7b5aa 100644 --- a/skylark/benchmark/network/throughput.py +++ b/skylark/benchmark/network/throughput.py @@ -5,7 +5,7 @@ import sys import os -from loguru import logger +from skylark.utils import logger from tqdm import tqdm import questionary diff --git a/skylark/benchmark/network/traceroute.py b/skylark/benchmark/network/traceroute.py index ef4cb7727..7462aa128 100644 --- a/skylark/benchmark/network/traceroute.py +++ b/skylark/benchmark/network/traceroute.py @@ -3,7 +3,7 @@ import json from typing import List, Tuple -from loguru import logger +from skylark.utils import logger from tqdm import tqdm from skylark import skylark_root diff --git a/skylark/benchmark/replicate/benchmark_triangles.py b/skylark/benchmark/replicate/benchmark_triangles.py index 635422e6d..dbdedcc0b 100644 --- a/skylark/benchmark/replicate/benchmark_triangles.py +++ b/skylark/benchmark/replicate/benchmark_triangles.py @@ -4,7 +4,7 @@ from pathlib import Path import typer -from loguru import logger +from skylark.utils import logger from skylark import GB, MB, skylark_root from skylark.replicate.replication_plan import ReplicationJob, ReplicationTopology from skylark.replicate.replicator_client import ReplicatorClient diff --git a/skylark/benchmark/replicate/test_direct.py b/skylark/benchmark/replicate/test_direct.py index 40abc1ddf..e623a255c 100644 --- a/skylark/benchmark/replicate/test_direct.py +++ b/skylark/benchmark/replicate/test_direct.py @@ -3,7 +3,7 @@ import time from datetime import datetime -from loguru import logger +from skylark.utils import logger from skylark import skylark_root from skylark.benchmark.utils import provision diff --git a/skylark/benchmark/stop_all_instances.py b/skylark/benchmark/stop_all_instances.py index f1de52b5a..0836e3ffd 100644 --- a/skylark/benchmark/stop_all_instances.py +++ b/skylark/benchmark/stop_all_instances.py @@ -1,6 +1,6 @@ import argparse -from loguru import logger +from skylark.utils import logger from tqdm import tqdm from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider diff --git a/skylark/benchmark/utils.py b/skylark/benchmark/utils.py index 86e1eafdd..3c104fe87 100644 --- a/skylark/benchmark/utils.py +++ b/skylark/benchmark/utils.py @@ -1,7 +1,7 @@ import sys from typing import Dict, Iterable, List, Optional, Tuple -from loguru import logger +from skylark.utils import logger from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider from skylark.compute.aws.aws_server import AWSServer from skylark.compute.azure.azure_cloud_provider import AzureCloudProvider diff --git a/skylark/chunk.py b/skylark/chunk.py index c7eaa8c14..6508d4da3 100644 --- a/skylark/chunk.py +++ b/skylark/chunk.py @@ -4,7 +4,7 @@ from enum import Enum, auto from typing import Dict, List, Optional -from loguru import logger +from skylark.utils import logger @dataclass diff --git a/skylark/cli/cli.py b/skylark/cli/cli.py index 29476a5f0..11e3792dc 100644 --- a/skylark/cli/cli.py +++ b/skylark/cli/cli.py @@ -25,11 +25,13 @@ import skylark.cli.cli_solver import skylark.cli.experiments import typer -from loguru import logger +from skylark.utils import logger from skylark import GB, MB, config_file, print_header from skylark.cli.cli_helper import ( check_ulimit, + copy_azure_local, copy_gcs_local, + copy_local_azure, copy_local_gcs, copy_local_local, copy_local_s3, @@ -51,10 +53,6 @@ app.add_typer(skylark.cli.cli_azure.app, name="azure") app.add_typer(skylark.cli.cli_solver.app, name="solver") -# config logger -logger.remove() -logger.add(sys.stderr, format="{function:>20}:{line:<3} | {message}", enqueue=True) - @app.command() def ls(directory: str): diff --git a/skylark/cli/cli_aws.py b/skylark/cli/cli_aws.py index f1456b9ca..c9b4ae7f4 100644 --- a/skylark/cli/cli_aws.py +++ b/skylark/cli/cli_aws.py @@ -9,17 +9,13 @@ import questionary import typer -from loguru import logger +from skylark.utils import logger from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider from skylark.compute.aws.aws_server import AWSServer from skylark.utils.utils import do_parallel app = typer.Typer(name="skylark-aws") -# config logger -logger.remove() -logger.add(sys.stderr, format="{function:>20}:{line:<3} | {message}", colorize=True, enqueue=True) - @app.command() def vcpu_limits(quota_code="L-1216C47A"): @@ -52,7 +48,7 @@ def ssh(region: Optional[str] = None): proc = subprocess.Popen(split(f"ssh -i {str(instance.local_keyfile)} ubuntu@{instance.public_ip()}")) proc.wait() else: - logger.secho(f"No instance selected", fg="red") + typer.secho(f"No instance selected", fg="red") if __name__ == "__main__": diff --git a/skylark/cli/cli_azure.py b/skylark/cli/cli_azure.py index 028152fab..1c2596265 100644 --- a/skylark/cli/cli_azure.py +++ b/skylark/cli/cli_azure.py @@ -10,17 +10,13 @@ import typer from azure.identity import DefaultAzureCredential from azure.mgmt.compute import ComputeManagementClient -from loguru import logger +from skylark.utils import logger from skylark.cli.cli_helper import load_config from skylark.compute.azure.azure_cloud_provider import AzureCloudProvider from skylark.utils.utils import do_parallel app = typer.Typer(name="skylark-azure") -# config logger -logger.remove() -logger.add(sys.stderr, format="{function:>20}:{line:<3} | {message}", colorize=True, enqueue=True) - @app.command() def get_valid_skus( diff --git a/skylark/cli/cli_helper.py b/skylark/cli/cli_helper.py index 03d04d951..1907c0bc9 100644 --- a/skylark/cli/cli_helper.py +++ b/skylark/cli/cli_helper.py @@ -9,7 +9,7 @@ from typing import Dict, List, Optional import typer -from loguru import logger +from skylark.utils import logger from skylark import config_file from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider from skylark.compute.azure.azure_cloud_provider import AzureCloudProvider diff --git a/skylark/cli/cli_solver.py b/skylark/cli/cli_solver.py index 402af495c..61fd8db83 100644 --- a/skylark/cli/cli_solver.py +++ b/skylark/cli/cli_solver.py @@ -8,16 +8,12 @@ import cvxpy as cp import typer -from loguru import logger +from skylark.utils import logger from skylark.replicate.solver import ThroughputProblem, ThroughputSolverILP from skylark import skylark_root app = typer.Typer(name="skylark-solver") -# config logger -logger.remove() -logger.add(sys.stderr, format="{function:>20}:{line:<3} | {message}", colorize=True, enqueue=True) - def choose_solver(): try: diff --git a/skylark/compute/aws/aws_cloud_provider.py b/skylark/compute/aws/aws_cloud_provider.py index e175d0ecb..cee5d323c 100644 --- a/skylark/compute/aws/aws_cloud_provider.py +++ b/skylark/compute/aws/aws_cloud_provider.py @@ -5,7 +5,7 @@ import botocore import pandas as pd -from loguru import logger +from skylark.utils import logger from oslo_concurrency import lockutils from skylark import skylark_root diff --git a/skylark/compute/aws/aws_server.py b/skylark/compute/aws/aws_server.py index ee2c8164f..069401c55 100644 --- a/skylark/compute/aws/aws_server.py +++ b/skylark/compute/aws/aws_server.py @@ -5,7 +5,7 @@ import boto3 import paramiko -from loguru import logger +from skylark.utils import logger from oslo_concurrency import lockutils from skylark import key_root diff --git a/skylark/compute/azure/azure_cloud_provider.py b/skylark/compute/azure/azure_cloud_provider.py index a21340976..cac1d3fb8 100644 --- a/skylark/compute/azure/azure_cloud_provider.py +++ b/skylark/compute/azure/azure_cloud_provider.py @@ -4,7 +4,7 @@ from typing import List, Optional import paramiko -from loguru import logger +from skylark.utils import logger from skylark import key_root from skylark.compute.azure.azure_server import AzureServer from skylark.compute.cloud_providers import CloudProvider diff --git a/skylark/compute/gcp/gcp_cloud_provider.py b/skylark/compute/gcp/gcp_cloud_provider.py index 5fac1c443..310f29603 100644 --- a/skylark/compute/gcp/gcp_cloud_provider.py +++ b/skylark/compute/gcp/gcp_cloud_provider.py @@ -6,7 +6,7 @@ import googleapiclient import paramiko -from loguru import logger +from skylark.utils import logger from oslo_concurrency import lockutils from skylark import key_root diff --git a/skylark/compute/server.py b/skylark/compute/server.py index d6e83211b..7c620043d 100644 --- a/skylark/compute/server.py +++ b/skylark/compute/server.py @@ -7,7 +7,7 @@ from typing import Dict import requests -from loguru import logger +from skylark.utils import logger from skylark.compute.utils import make_dozzle_command, make_netdata_command from skylark.utils.utils import PathLike, Timer, wait_for @@ -268,11 +268,11 @@ def start_gateway( # todo add other launch flags for gateway daemon logger.debug(desc_prefix + f": Starting gateway container {gateway_docker_image}") docker_run_flags = f"-d --rm --log-driver=local --ipc=host --network=host --ulimit nofile={1024 * 1024} {docker_envs}" - gateway_daemon_cmd = f"python -u /pkg/skylark/gateway/gateway_daemon.py --debug --chunk-dir /dev/shm/skylark/chunks --outgoing-ports '{json.dumps(outgoing_ports)}' --region {self.region_tag}" + gateway_daemon_cmd = f"python -u /pkg/skylark/gateway/gateway_daemon.py --chunk-dir /dev/shm/skylark/chunks --outgoing-ports '{json.dumps(outgoing_ports)}' --region {self.region_tag}" docker_launch_cmd = f"sudo docker run {docker_run_flags} --name skylark_gateway {gateway_docker_image} {gateway_daemon_cmd}" start_out, start_err = self.run_command(docker_launch_cmd) - logger.debug(desc_prefix + f": Gateway started {start_out}") - assert not start_err.strip(), f"Error starting gateway: {start_err}" + logger.debug(desc_prefix + f": Gateway started {start_out.strip()}") + assert not start_err.strip(), f"Error starting gateway: {start_err.strip()}" gateway_container_hash = start_out.strip().split("\n")[-1][:12] self.gateway_api_url = f"http://{self.public_ip()}:8080/api/v1" self.gateway_log_viewer_url = f"http://{self.public_ip()}:8888/container/{gateway_container_hash}" diff --git a/skylark/gateway/chunk_store.py b/skylark/gateway/chunk_store.py index c774814ce..7091bfc4f 100644 --- a/skylark/gateway/chunk_store.py +++ b/skylark/gateway/chunk_store.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Dict, List, Optional -from loguru import logger +from skylark.utils import logger from skylark.chunk import ChunkRequest, ChunkState diff --git a/skylark/gateway/gateway_daemon.py b/skylark/gateway/gateway_daemon.py index 31ec979fc..cd06b4251 100644 --- a/skylark/gateway/gateway_daemon.py +++ b/skylark/gateway/gateway_daemon.py @@ -12,7 +12,7 @@ from typing import Dict, Optional import setproctitle -from loguru import logger +from skylark.utils import logger from skylark import MB, print_header from skylark.chunk import ChunkState from skylark.gateway.chunk_store import ChunkStore @@ -32,15 +32,7 @@ def __init__( outgoing_ports: Dict[str, int], chunk_dir: PathLike, max_incoming_ports=64, - debug=False, - log_dir: Optional[PathLike] = None, ): - if log_dir is not None: - log_dir = Path(log_dir) - log_dir.mkdir(exist_ok=True) - logger.remove() - logger.add(log_dir / "gateway_daemon.log", rotation="10 MB", enqueue=True) - logger.add(sys.stderr, colorize=True, format="{function:>15}:{line:<3} {level:<8} {message}", level="DEBUG", enqueue=True) self.region = region self.chunk_store = ChunkStore(chunk_dir) self.gateway_receiver = GatewayReceiver(chunk_store=self.chunk_store, max_pending_chunks=max_incoming_ports) @@ -49,9 +41,7 @@ def __init__( # API server atexit.register(self.cleanup) - self.api_server = GatewayDaemonAPI( - self.chunk_store, self.gateway_receiver, debug=debug, log_dir=log_dir, daemon_cleanup_handler=self.cleanup - ) + self.api_server = GatewayDaemonAPI(self.chunk_store, self.gateway_receiver, daemon_cleanup_handler=self.cleanup) self.api_server.start() logger.info(f"Gateway daemon API started at {self.api_server.url}") @@ -177,15 +167,7 @@ def fn(chunk_req, src_region, src_bucket): "--outgoing-ports", type=str, required=True, help="JSON encoded path mapping destination ip to number of outgoing ports" ) parser.add_argument("--chunk-dir", type=Path, default="/dev/shm/skylark/chunks", help="Directory to store chunks") - parser.add_argument("--debug", action="store_true", help="Enable debug mode for Flask") - parser.add_argument("--log-dir", type=Path, default=Path("/var/log/skylark"), help="Directory to write logs to") args = parser.parse_args() - daemon = GatewayDaemon( - region=args.region, - outgoing_ports=json.loads(args.outgoing_ports), - chunk_dir=args.chunk_dir, - debug=args.debug, - log_dir=Path(args.log_dir), - ) + daemon = GatewayDaemon(region=args.region, outgoing_ports=json.loads(args.outgoing_ports), chunk_dir=args.chunk_dir) daemon.run() diff --git a/skylark/gateway/gateway_daemon_api.py b/skylark/gateway/gateway_daemon_api.py index ad048cb16..521705e6e 100644 --- a/skylark/gateway/gateway_daemon_api.py +++ b/skylark/gateway/gateway_daemon_api.py @@ -5,7 +5,7 @@ from pathlib import Path from flask import Flask, jsonify, request -from loguru import logger +from skylark.utils import logger from skylark import MB from skylark.chunk import ChunkRequest, ChunkState from skylark.gateway.chunk_store import ChunkStore @@ -33,8 +33,6 @@ def __init__( gateway_receiver: GatewayReceiver, host="0.0.0.0", port=8080, - debug=False, - log_dir=None, daemon_cleanup_handler=None, ): super().__init__() @@ -49,18 +47,7 @@ def __init__( self.register_request_routes() # make server - self.log_dir = log_dir - if log_dir is not None: - log_dir = Path(log_dir) - handler = logging.handlers.RotatingFileHandler(log_dir / "gateway_daemon_api.log", maxBytes=1 * MB) - logging.getLogger("werkzeug").addHandler(handler) - if debug: - self.app.config["JSONIFY_PRETTYPRINT_REGULAR"] = True - self.app.config["TESTING"] = True - logging.getLogger("werkzeug").addHandler(logging.StreamHandler()) - logging.getLogger("werkzeug").setLevel(logging.DEBUG) - else: - logging.getLogger("werkzeug").setLevel(logging.WARNING) + logging.getLogger("werkzeug").setLevel(logging.WARNING) self.server = make_server(host, port, self.app, threaded=True) self.url = "http://{}:{}".format(host, port) diff --git a/skylark/gateway/gateway_receiver.py b/skylark/gateway/gateway_receiver.py index 855cc8cd6..1ff21833e 100644 --- a/skylark/gateway/gateway_receiver.py +++ b/skylark/gateway/gateway_receiver.py @@ -7,7 +7,7 @@ from typing import Tuple import setproctitle -from loguru import logger +from skylark.utils import logger from skylark import GB, MB from skylark.chunk import WireProtocolHeader from skylark.gateway.chunk_store import ChunkStore diff --git a/skylark/gateway/gateway_sender.py b/skylark/gateway/gateway_sender.py index 7e360af64..60a2d11b7 100644 --- a/skylark/gateway/gateway_sender.py +++ b/skylark/gateway/gateway_sender.py @@ -5,7 +5,7 @@ import requests import setproctitle -from loguru import logger +from skylark.utils import logger from skylark import MB from skylark.chunk import ChunkRequest from skylark.gateway.chunk_store import ChunkStore diff --git a/skylark/obj_store/azure_interface.py b/skylark/obj_store/azure_interface.py index 61872037d..a3ae21b36 100644 --- a/skylark/obj_store/azure_interface.py +++ b/skylark/obj_store/azure_interface.py @@ -98,8 +98,8 @@ def get_obj_metadata(self, obj_name): # Not Tested blob_client = self.blob_service_client.get_blob_client(container=self.container_name, blob=obj_name) try: return blob_client.get_blob_properties() - except ResourceNotFoundError: - typer.secho("No blob found.") + except ResourceNotFoundError as e: + raise NoSuchObjectException(f"Object {obj_name} does not exist, or you do not have permission to access it") from e def get_obj_size(self, obj_name): return self.get_obj_metadata(obj_name).size @@ -112,12 +112,6 @@ def exists(self, obj_name): except ResourceNotFoundError: return False - """ - stream = blob_client.download_blob() - for chunk in stream.chunks(): - # Reading data in chunks to avoid loading all into memory at once - """ - def download_object(self, src_object_name, dst_file_path) -> Future: src_object_name, dst_file_path = str(src_object_name), str(dst_file_path) src_object_name = src_object_name if src_object_name[0] != "/" else src_object_name diff --git a/skylark/replicate/replication_plan.py b/skylark/replicate/replication_plan.py index 386bebe3f..2f91161e5 100644 --- a/skylark/replicate/replication_plan.py +++ b/skylark/replicate/replication_plan.py @@ -4,7 +4,7 @@ import shutil from typing import Dict, List, Optional, Set, Tuple -from loguru import logger +from skylark.utils import logger import graphviz as gv from skylark import MB from skylark.chunk import ChunkRequest diff --git a/skylark/replicate/replicator_client.py b/skylark/replicate/replicator_client.py index fa1b4c1b5..f2fab1804 100644 --- a/skylark/replicate/replicator_client.py +++ b/skylark/replicate/replicator_client.py @@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Tuple import requests -from loguru import logger +from skylark.utils import logger from tqdm import tqdm import pandas as pd from skylark import GB, KB, MB diff --git a/skylark/replicate/solver.py b/skylark/replicate/solver.py index cc4679c8c..01189eb3a 100644 --- a/skylark/replicate/solver.py +++ b/skylark/replicate/solver.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd -from loguru import logger +from skylark.utils import logger from skylark import GB from skylark.compute.cloud_providers import CloudProvider diff --git a/skylark/test/test_gateway_daemon.py b/skylark/test/test_gateway_daemon.py index ee24d5d81..41b2f93e2 100644 --- a/skylark/test/test_gateway_daemon.py +++ b/skylark/test/test_gateway_daemon.py @@ -1,6 +1,6 @@ from pathlib import Path -from loguru import logger +from skylark.utils import logger from skylark.chunk import Chunk, ChunkRequest, ChunkRequestHop, ChunkState from skylark.gateway.gateway_daemon import GatewayDaemon diff --git a/skylark/test/test_replicator_client.py b/skylark/test/test_replicator_client.py index 8441dc81f..5cac163f0 100644 --- a/skylark/test/test_replicator_client.py +++ b/skylark/test/test_replicator_client.py @@ -1,6 +1,6 @@ import argparse -from loguru import logger +from skylark.utils import logger from skylark import GB, MB, print_header import tempfile @@ -88,9 +88,7 @@ def main(args): obj_store_interface_dst.create_bucket() if not args.skip_upload: - # todo implement object store support - # pass - print("Not skipping upload...", src_bucket, dst_bucket) + logger.info(f"Not skipping upload, source bucket is {src_bucket}, destination bucket is {dst_bucket}") # TODO: fix this to get the key instead of S3Object matching_src_keys = list([obj.key for obj in obj_store_interface_src.list_objects(prefix=args.key_prefix)]) diff --git a/skylark/utils/logger.py b/skylark/utils/logger.py new file mode 100644 index 000000000..df061264b --- /dev/null +++ b/skylark/utils/logger.py @@ -0,0 +1,27 @@ +from datetime import datetime +import sys +from functools import partial + +import termcolor + + +def log(msg, LEVEL="INFO", color="white", *args, **kwargs): + if args or kwargs: + msg = msg.format(*args, **kwargs) + level_prefix = ("[" + LEVEL.upper() + "] ").ljust(7) + time = datetime.now().strftime("%H:%M:%S") + print(f"{time} {level_prefix} {termcolor.colored(msg, color)}", flush=True, file=sys.stderr) + + +debug = partial(log, LEVEL="DEBUG", color="cyan") +info = partial(log, LEVEL="INFO", color="white") +warn = partial(log, LEVEL="WARN", color="yellow") +warning = partial(log, LEVEL="WARN", color="yellow") +error = partial(log, LEVEL="ERROR", color="red") + + +def exception(msg, *args, **kwargs): + error(f"Exception: {msg}", *args, **kwargs) + import traceback + + traceback.print_exc() diff --git a/skylark/utils/utils.py b/skylark/utils/utils.py index f82609c8e..3842a4473 100644 --- a/skylark/utils/utils.py +++ b/skylark/utils/utils.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Union -from loguru import logger +from skylark.utils import logger from tqdm import tqdm PathLike = Union[str, Path]