Merge branch 'main' into dev/paras/solver2

skyplane-project · Jan 30, 2022 · 25133f3 · 25133f3
2 parents ed9c160 + aab2d6c
commit 25133f3
Show file tree

Hide file tree

Showing 58 changed files with 8,757 additions and 1,351 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,26 @@
 # syntax=docker/dockerfile:1
 FROM python:3.8-slim
+
+# increase number of open files and concurrent TCP connections
+RUN echo 'net.ipv4.ip_local_port_range = 12000 65535' >> /etc/sysctl.conf
+RUN echo 'fs.file-max = 1048576' >> /etc/sysctl.conf
+RUN mkdir -p /etc/security/
+RUN echo '*                soft    nofile          1048576' >> /etc/security/limits.conf
+RUN echo '*                hard    nofile          1048576' >> /etc/security/limits.conf
+RUN echo 'root             soft    nofile          1048576' >> /etc/security/limits.conf
+RUN echo 'root             hard    nofile          1048576' >> /etc/security/limits.conf
+
+# install apt packages
+RUN --mount=type=cache,target=/var/cache/apt apt update \
+    && apt-get install --no-install-recommends -y git wget ca-certificates build-essential rsync \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# install gateway
 COPY scripts/requirements-gateway.txt /tmp/requirements-gateway.txt
 RUN --mount=type=cache,target=/root/.cache/pip pip install --no-cache-dir --compile -r /tmp/requirements-gateway.txt && rm -r /tmp/requirements-gateway.txt
 
 WORKDIR /pkg
 COPY . .
 RUN pip install -e .
-CMD ["python", "skylark/gateway/gateway_daemon.py"]
+CMD ["python", "skylark/gateway/gateway_daemon.py"]
diff --git a/nb/01-04-2021_plot_udp_throughput.ipynb b/nb/01-04-2021_plot_udp_throughput.ipynb
@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

diff --git a/nb/11-16-2021_plot_latency.ipynb b/nb/11-16-2021_plot_latency.ipynb
@@ -19,7 +19,7 @@
    "source": [
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

diff --git a/nb/11-16-2021_plot_throughput.ipynb b/nb/11-16-2021_plot_throughput.ipynb
@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

diff --git a/nb/11-22-2021_plot_throughput_time.ipynb b/nb/11-22-2021_plot_throughput_time.ipynb
@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "from tqdm import tqdm\n",
     "\n",
     "import pickle\n",

diff --git a/nb/11-22-2021_throughput_solver.ipynb b/nb/11-22-2021_throughput_solver.ipynb
@@ -20,7 +20,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "from tqdm import tqdm\n",
     "\n",
     "import pickle\n",

diff --git a/nb/12-02-2021_gcp_premium_standard.ipynb b/nb/12-02-2021_gcp_premium_standard.ipynb
@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import json\n",

diff --git a/nb/12-11-2021_get_transfer_cost.ipynb b/nb/12-11-2021_get_transfer_cost.ipynb
@@ -19,7 +19,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import pickle\n",
     "import requests\n",

diff --git a/nb/12-12-2021_visualize_pareto.ipynb b/nb/12-12-2021_visualize_pareto.ipynb
@@ -20,7 +20,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "from loguru import logger\n",
+    "from skylark.utils import logger\n",
     "\n",
     "import seaborn as sns\n",
     "\n",

diff --git a/profiles/throughput.csv b/profiles/throughput.csv
diff --git a/scripts/experiments/num_connections/benchmark_num_connections.sh b/scripts/experiments/num_connections/benchmark_num_connections.sh
@@ -20,7 +20,7 @@ function benchmark_config {
     NUM_GATEWAYS=$2
     # N_CHUNKS_PER_CONNECTION * NUM_CONNECTIONS
     N_CHUNKS=$((N_CHUNKS_PER_CONNECTION * NUM_CONNECTIONS))
-    echo "skylark replicate-random $SRC_REGION $DST_REGION $INTER_REGION --chunk-size-mb $CHUNK_SIZE_MB --n-chunks $N_CHUNKS --num-gateways $NUM_GATEWAYS --num-outgoing-connections $NUM_CONNECTIONS --no-reuse-gateways --no-serve-web-dashboard"
+    echo "skylark replicate-random $SRC_REGION $DST_REGION $INTER_REGION --chunk-size-mb $CHUNK_SIZE_MB --n-chunks $N_CHUNKS --num-gateways $NUM_GATEWAYS --num-outgoing-connections $NUM_CONNECTIONS --no-reuse-gateways"
 }
 
 # log function with message argument

diff --git a/scripts/experiments/num_gateways/benchmark_num_gateways.sh b/scripts/experiments/num_gateways/benchmark_num_gateways.sh
@@ -20,7 +20,7 @@ function benchmark_config {
     NUM_GATEWAYS=$2
     N_CHUNKS=$((N_CHUNKS_PER_CONNECTION * NUM_CONNECTIONS * NUM_GATEWAYS))
     set -x
-    skylark replicate-random $SRC_REGION $DST_REGION $INTER_REGION --chunk-size-mb $CHUNK_SIZE_MB --n-chunks $N_CHUNKS --num-gateways $NUM_GATEWAYS --num-outgoing-connections $NUM_CONNECTIONS --no-serve-web-dashboard --aws-instance-class m5.4xlarge
+    skylark replicate-random $SRC_REGION $DST_REGION $INTER_REGION --chunk-size-mb $CHUNK_SIZE_MB --n-chunks $N_CHUNKS --num-gateways $NUM_GATEWAYS --num-outgoing-connections $NUM_CONNECTIONS --aws-instance-class m5.4xlarge
     set +x
 }
 

diff --git a/scripts/requirements-gateway.txt b/scripts/requirements-gateway.txt
@@ -2,13 +2,15 @@ azure-mgmt-resource
 azure-mgmt-compute
 azure-mgmt-network
 azure-identity
+azure-storage-blob
 awscrt
 boto3
 flask
 google-api-python-client
 google-cloud-compute
-loguru
+google-cloud-storage
 setproctitle
+termcolor
 tqdm
 werkzeug
 oslo.concurrency

diff --git a/scripts/test_gateway.sh b/scripts/test_gateway.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 set -x
 sudo DOCKER_BUILDKIT=1 docker build -t gateway_test .
-sudo docker run --rm --ipc=host --network=host --name=skylark_gateway gateway_test python /pkg/skylark/gateway/gateway_daemon.py --chunk-dir /dev/shm/skylark_test/chunks
+sudo docker run --rm --ipc=host --network=host --name=skylark_gateway gateway_test python /pkg/skylark/gateway/gateway_daemon.py --chunk-dir /dev/shm/skylark_test/chunks --region aws:us-east-1 --outgoing-ports "{}"
diff --git a/setup.py b/setup.py
@@ -11,13 +11,15 @@
         "azure-mgmt-compute",
         "azure-mgmt-network",
         "azure-identity",
+        "azure-storage-blob>=12.0.0",
         "awscrt",
         "boto3",
         "flask",
         "google-api-python-client",
         "google-cloud-compute",
-        "loguru",
+        "google-cloud-storage",
         "setproctitle",
+        "termcolor",
         "tqdm",
         "werkzeug",
     ],
@@ -39,9 +41,5 @@
         ],
         "test": ["black", "ipython", "jupyter_console", "pytest", "pytype"],
     },
-    entry_points={
-        "console_scripts": [
-            "skylark=skylark.cli.cli:app",
-        ]
-    },
+    entry_points={"console_scripts": ["skylark=skylark.cli.cli:app"]},
 )
diff --git a/skylark/benchmark/network/latency.py b/skylark/benchmark/network/latency.py
@@ -3,7 +3,7 @@
 import re
 from typing import List, Tuple
 
-from loguru import logger
+from skylark.utils import logger
 from tqdm import tqdm
 
 from skylark import skylark_root

diff --git a/skylark/benchmark/network/throughput.py b/skylark/benchmark/network/throughput.py
diff --git a/skylark/benchmark/network/traceroute.py b/skylark/benchmark/network/traceroute.py
@@ -3,7 +3,7 @@
 import json
 from typing import List, Tuple
 
-from loguru import logger
+from skylark.utils import logger
 from tqdm import tqdm
 
 from skylark import skylark_root