Migrate from loguru to print (#104)

* Migrate from loguru to print

* Update dependencies for logger

nb/01-04-2021_plot_udp_throughput.ipynb

@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

nb/01-21-2022_visualize_pareto.ipynb

@@ -10,7 +10,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import seaborn as sns\n",
     "\n",

nb/11-16-2021_plot_latency.ipynb

@@ -19,7 +19,7 @@
    "source": [
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

nb/11-16-2021_plot_throughput.ipynb

@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

nb/11-22-2021_plot_throughput_time.ipynb

@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "from tqdm import tqdm\n",
     "\n",
     "import pickle\n",

nb/11-22-2021_throughput_solver.ipynb

@@ -20,7 +20,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "from tqdm import tqdm\n",
     "\n",
     "import pickle\n",

nb/12-02-2021_gcp_premium_standard.ipynb

@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

nb/12-11-2021_get_transfer_cost.ipynb

@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import requests\n",

nb/12-12-2021_visualize_pareto.ipynb

@@ -20,7 +20,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import seaborn as sns\n",
     "\n",

scripts/requirements-gateway.txt

@@ -9,8 +9,8 @@ flask
 google-api-python-client
 google-cloud-compute
 google-cloud-storage
-loguru
 setproctitle
+termcolor
 tqdm
 werkzeug
 oslo.concurrency

setup.py

@@ -18,8 +18,8 @@
         "google-api-python-client",
         "google-cloud-compute",
         "google-cloud-storage",
-        "loguru",
         "setproctitle",
+        "termcolor",
         "tqdm",
         "werkzeug",
     ],

skylark/benchmark/network/latency.py

@@ -3,7 +3,7 @@
 import re
 from typing import List, Tuple
 
-from loguru import logger
+from skylark.utils import logger
 from tqdm import tqdm
 
 from skylark import skylark_root

skylark/benchmark/network/throughput.py

@@ -5,7 +5,7 @@
 import sys
 import os
 
-from loguru import logger
+from skylark.utils import logger
 from tqdm import tqdm
 import questionary
 

skylark/benchmark/network/traceroute.py

@@ -3,7 +3,7 @@
 import json
 from typing import List, Tuple
 
-from loguru import logger
+from skylark.utils import logger
 from tqdm import tqdm
 
 from skylark import skylark_root

skylark/benchmark/replicate/benchmark_triangles.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 
 import typer
-from loguru import logger
+from skylark.utils import logger
 from skylark import GB, MB, skylark_root
 from skylark.replicate.replication_plan import ReplicationJob, ReplicationTopology
 from skylark.replicate.replicator_client import ReplicatorClient

skylark/benchmark/replicate/test_direct.py

@@ -3,7 +3,7 @@
 import time
 from datetime import datetime
 
-from loguru import logger
+from skylark.utils import logger
 
 from skylark import skylark_root
 from skylark.benchmark.utils import provision

skylark/benchmark/stop_all_instances.py

@@ -1,6 +1,6 @@
 import argparse
 
-from loguru import logger
+from skylark.utils import logger
 from tqdm import tqdm
 
 from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider

skylark/benchmark/utils.py

@@ -1,7 +1,7 @@
 import sys
 from typing import Dict, Iterable, List, Optional, Tuple
 
-from loguru import logger
+from skylark.utils import logger
 from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider
 from skylark.compute.aws.aws_server import AWSServer
 from skylark.compute.azure.azure_cloud_provider import AzureCloudProvider

skylark/chunk.py

@@ -4,7 +4,7 @@
 from enum import Enum, auto
 from typing import Dict, List, Optional
 
-from loguru import logger
+from skylark.utils import logger
 
 
 @dataclass

skylark/cli/cli.py

@@ -25,11 +25,13 @@
 import skylark.cli.cli_solver
 import skylark.cli.experiments
 import typer
-from loguru import logger
+from skylark.utils import logger
 from skylark import GB, MB, config_file, print_header
 from skylark.cli.cli_helper import (
     check_ulimit,
+    copy_azure_local,
     copy_gcs_local,
+    copy_local_azure,
     copy_local_gcs,
     copy_local_local,
     copy_local_s3,
@@ -51,10 +53,6 @@
 app.add_typer(skylark.cli.cli_azure.app, name="azure")
 app.add_typer(skylark.cli.cli_solver.app, name="solver")
 
-# config logger
-logger.remove()
-logger.add(sys.stderr, format="{function:>20}:{line:<3} | <level>{message}</level>", enqueue=True)
-
 
 @app.command()
 def ls(directory: str):

skylark/cli/cli_aws.py

@@ -9,17 +9,13 @@
 import questionary
 
 import typer
-from loguru import logger
+from skylark.utils import logger
 from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider
 from skylark.compute.aws.aws_server import AWSServer
 from skylark.utils.utils import do_parallel
 
 app = typer.Typer(name="skylark-aws")
 
-# config logger
-logger.remove()
-logger.add(sys.stderr, format="{function:>20}:{line:<3} | <level>{message}</level>", colorize=True, enqueue=True)
-
 
 @app.command()
 def vcpu_limits(quota_code="L-1216C47A"):
@@ -52,7 +48,7 @@ def ssh(region: Optional[str] = None):
         proc = subprocess.Popen(split(f"ssh -i {str(instance.local_keyfile)} ubuntu@{instance.public_ip()}"))
         proc.wait()
     else:
-        logger.secho(f"No instance selected", fg="red")
+        typer.secho(f"No instance selected", fg="red")
 
 
 if __name__ == "__main__":

skylark/cli/cli_azure.py

@@ -10,17 +10,13 @@
 import typer
 from azure.identity import DefaultAzureCredential
 from azure.mgmt.compute import ComputeManagementClient
-from loguru import logger
+from skylark.utils import logger
 from skylark.cli.cli_helper import load_config
 from skylark.compute.azure.azure_cloud_provider import AzureCloudProvider
 from skylark.utils.utils import do_parallel
 
 app = typer.Typer(name="skylark-azure")
 
-# config logger
-logger.remove()
-logger.add(sys.stderr, format="{function:>20}:{line:<3} | <level>{message}</level>", colorize=True, enqueue=True)
-
 
 @app.command()
 def get_valid_skus(

skylark/cli/cli_helper.py

@@ -9,7 +9,7 @@
 from typing import Dict, List, Optional
 
 import typer
-from loguru import logger
+from skylark.utils import logger
 from skylark import config_file
 from skylark.compute.aws.aws_cloud_provider import AWSCloudProvider
 from skylark.compute.azure.azure_cloud_provider import AzureCloudProvider

skylark/cli/cli_solver.py

@@ -8,16 +8,12 @@
 import cvxpy as cp
 
 import typer
-from loguru import logger
+from skylark.utils import logger
 from skylark.replicate.solver import ThroughputProblem, ThroughputSolverILP
 from skylark import skylark_root
 
 app = typer.Typer(name="skylark-solver")
 
-# config logger
-logger.remove()
-logger.add(sys.stderr, format="{function:>20}:{line:<3} | <level>{message}</level>", colorize=True, enqueue=True)
-
 
 def choose_solver():
     try:

skylark/compute/aws/aws_cloud_provider.py

@@ -5,7 +5,7 @@
 
 import botocore
 import pandas as pd
-from loguru import logger
+from skylark.utils import logger
 
 from oslo_concurrency import lockutils
 from skylark import skylark_root

skylark/compute/aws/aws_server.py

@@ -5,7 +5,7 @@
 
 import boto3
 import paramiko
-from loguru import logger
+from skylark.utils import logger
 
 from oslo_concurrency import lockutils
 from skylark import key_root

skylark/compute/azure/azure_cloud_provider.py

@@ -4,7 +4,7 @@
 from typing import List, Optional
 
 import paramiko
-from loguru import logger
+from skylark.utils import logger
 from skylark import key_root
 from skylark.compute.azure.azure_server import AzureServer
 from skylark.compute.cloud_providers import CloudProvider

skylark/compute/gcp/gcp_cloud_provider.py

@@ -6,7 +6,7 @@
 
 import googleapiclient
 import paramiko
-from loguru import logger
+from skylark.utils import logger
 
 from oslo_concurrency import lockutils
 from skylark import key_root

skylark/compute/server.py

@@ -7,7 +7,7 @@
 from typing import Dict
 
 import requests
-from loguru import logger
+from skylark.utils import logger
 from skylark.compute.utils import make_dozzle_command, make_netdata_command
 from skylark.utils.utils import PathLike, Timer, wait_for
 
@@ -268,11 +268,11 @@ def start_gateway(
         # todo add other launch flags for gateway daemon
         logger.debug(desc_prefix + f": Starting gateway container {gateway_docker_image}")
         docker_run_flags = f"-d --rm --log-driver=local --ipc=host --network=host --ulimit nofile={1024 * 1024} {docker_envs}"
-        gateway_daemon_cmd = f"python -u /pkg/skylark/gateway/gateway_daemon.py --debug --chunk-dir /dev/shm/skylark/chunks --outgoing-ports '{json.dumps(outgoing_ports)}' --region {self.region_tag}"
+        gateway_daemon_cmd = f"python -u /pkg/skylark/gateway/gateway_daemon.py --chunk-dir /dev/shm/skylark/chunks --outgoing-ports '{json.dumps(outgoing_ports)}' --region {self.region_tag}"
         docker_launch_cmd = f"sudo docker run {docker_run_flags} --name skylark_gateway {gateway_docker_image} {gateway_daemon_cmd}"
         start_out, start_err = self.run_command(docker_launch_cmd)
-        logger.debug(desc_prefix + f": Gateway started {start_out}")
-        assert not start_err.strip(), f"Error starting gateway: {start_err}"
+        logger.debug(desc_prefix + f": Gateway started {start_out.strip()}")
+        assert not start_err.strip(), f"Error starting gateway: {start_err.strip()}"
         gateway_container_hash = start_out.strip().split("\n")[-1][:12]
         self.gateway_api_url = f"http://{self.public_ip()}:8080/api/v1"
         self.gateway_log_viewer_url = f"http://{self.public_ip()}:8888/container/{gateway_container_hash}"

skylark/gateway/chunk_store.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from loguru import logger
+from skylark.utils import logger
 from skylark.chunk import ChunkRequest, ChunkState
 
 

skylark/gateway/gateway_daemon.py

@@ -12,7 +12,7 @@
 from typing import Dict, Optional
 
 import setproctitle
-from loguru import logger
+from skylark.utils import logger
 from skylark import MB, print_header
 from skylark.chunk import ChunkState
 from skylark.gateway.chunk_store import ChunkStore
@@ -32,15 +32,7 @@ def __init__(
         outgoing_ports: Dict[str, int],
         chunk_dir: PathLike,
         max_incoming_ports=64,
-        debug=False,
-        log_dir: Optional[PathLike] = None,
     ):
-        if log_dir is not None:
-            log_dir = Path(log_dir)
-            log_dir.mkdir(exist_ok=True)
-            logger.remove()
-            logger.add(log_dir / "gateway_daemon.log", rotation="10 MB", enqueue=True)
-            logger.add(sys.stderr, colorize=True, format="{function:>15}:{line:<3} {level:<8} {message}", level="DEBUG", enqueue=True)
         self.region = region
         self.chunk_store = ChunkStore(chunk_dir)
         self.gateway_receiver = GatewayReceiver(chunk_store=self.chunk_store, max_pending_chunks=max_incoming_ports)
@@ -49,9 +41,7 @@ def __init__(
 
         # API server
         atexit.register(self.cleanup)
-        self.api_server = GatewayDaemonAPI(
-            self.chunk_store, self.gateway_receiver, debug=debug, log_dir=log_dir, daemon_cleanup_handler=self.cleanup
-        )
+        self.api_server = GatewayDaemonAPI(self.chunk_store, self.gateway_receiver, daemon_cleanup_handler=self.cleanup)
         self.api_server.start()
         logger.info(f"Gateway daemon API started at {self.api_server.url}")
 
@@ -177,15 +167,7 @@ def fn(chunk_req, src_region, src_bucket):
         "--outgoing-ports", type=str, required=True, help="JSON encoded path mapping destination ip to number of outgoing ports"
     )
     parser.add_argument("--chunk-dir", type=Path, default="/dev/shm/skylark/chunks", help="Directory to store chunks")
-    parser.add_argument("--debug", action="store_true", help="Enable debug mode for Flask")
-    parser.add_argument("--log-dir", type=Path, default=Path("/var/log/skylark"), help="Directory to write logs to")
     args = parser.parse_args()
 
-    daemon = GatewayDaemon(
-        region=args.region,
-        outgoing_ports=json.loads(args.outgoing_ports),
-        chunk_dir=args.chunk_dir,
-        debug=args.debug,
-        log_dir=Path(args.log_dir),
-    )
+    daemon = GatewayDaemon(region=args.region, outgoing_ports=json.loads(args.outgoing_ports), chunk_dir=args.chunk_dir)
     daemon.run()