Preliminary support for on-prem transfers with HDFS and POSIX path support

.github/workflows/pytest.yml

@@ -87,6 +87,8 @@ jobs:
         run: |
           poetry install -E gateway -E solver -E aws -E azure -E gcp
           poetry run pip install -r requirements-dev.txt
+          poetry run sudo apt install default-jdk
+          poetry run wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.0/hadoop-3.3.0.tar.gz -P /tmp && tar -xzf /tmp/hadoop-3.3.0.tar.gz -C /tmp && sudo mv /tmp/hadoop-3.3.0 /usr/local/hadoop && rm /tmp/hadoop-3.3.0.tar.gz
         if: steps.cache.outputs.cache-hit != 'true'
       - name: Run cloud tests
         env:
@@ -148,10 +150,13 @@ jobs:
         run: |
           poetry config virtualenvs.in-project false
           poetry config virtualenvs.path ~/.virtualenvs
+
       - name: Install Dependencies
         run: |
           poetry install -E gateway -E solver -E aws -E azure -E gcp
           poetry run pip install -r requirements-dev.txt
+          poetry run sudo apt install default-jdk
+          poetry run wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.0/hadoop-3.3.0.tar.gz -P /tmp && tar -xzf /tmp/hadoop-3.3.0.tar.gz -C /tmp && sudo mv /tmp/hadoop-3.3.0 /usr/local/hadoop && rm /tmp/hadoop-3.3.0.tar.gz
         if: steps.cache.outputs.cache-hit != 'true'
       - id: 'auth'
         uses: 'google-github-actions/auth@v0'

Dockerfile

@@ -2,11 +2,25 @@
 FROM python:3.11-slim
 
 # install apt packages
-RUN --mount=type=cache,target=/var/cache/apt apt update \
-    && apt-get install --no-install-recommends -y curl ca-certificates stunnel4 gcc libc-dev \
+RUN --mount=type=cache,target=/var/cache/apt apt-get update \
+    && apt-get install --no-install-recommends -y curl ca-certificates stunnel4 gcc libc-dev wget \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+#install HDFS Onprem Packages
+RUN apt-get update && \
+    apt-get install -y openjdk-11-jdk && \
+    apt-get clean 
+
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
+
+RUN wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.0/hadoop-3.3.0.tar.gz -P /tmp \
+    && tar -xzf /tmp/hadoop-3.3.0.tar.gz -C /tmp \
+    && mv /tmp/hadoop-3.3.0 /usr/local/hadoop \
+    && rm /tmp/hadoop-3.3.0.tar.gz
+
+ENV HADOOP_HOME /usr/local/hadoop
+
 # configure stunnel
 RUN mkdir -p /etc/stunnel \
     && openssl genrsa -out key.pem 2048 \
@@ -31,6 +45,10 @@ RUN (echo 'net.ipv4.ip_local_port_range = 12000 65535' >> /etc/sysctl.conf) \
 
 # install gateway
 COPY scripts/requirements-gateway.txt /tmp/requirements-gateway.txt
+
+#Onprem: Install Hostname Resolution for HDFS
+COPY scripts/on_prem/hostname /tmp/hostname
+
 RUN --mount=type=cache,target=/root/.cache/pip pip3 install --no-cache-dir -r /tmp/requirements-gateway.txt && rm -r /tmp/requirements-gateway.txt
 
 WORKDIR /pkg

docs/index.rst

@@ -24,6 +24,8 @@ Contents
    quickstart
    configure
    architecture
+   performance_stats_collection
+   on-prem_setup
    faq
 
 .. toctree::

docs/installation.rst

@@ -57,3 +57,33 @@ Now, you can initialize Skyplane with your desired cloud providers. Skyplane aut
 
    ---> Setup cloud provider connectors:
    $ skyplane init
+
+
+Transferring Data
+-------------------
+
+We're ready to use Skyplane! Let's use `skyplane cp` to copy files from AWS to GCP:
+
+.. code-block:: bash
+
+   ---> 🎸 Ready to rock and roll! Copy some files:
+   $ skyplane cp -r s3://... gs://...
+
+To transfer only new objects, you can instead use `skyplane sync`: 
+
+.. code-block:: bash
+
+   ---> Copy only diff
+   $ skyplane sync s3://... gs://...
+
+To transfer from local disk or HDFS cluster, you can use `skyplane cp` as well:
+
+(Note: On-Prem require additional setup. Please navigate to the `On-Prem` section for more details)
+
+.. code-block:: bash
+
+   ---> Copy from local disk
+   $ skyplane cp -r /path/to/local/file gs://...
+
+   ---> Copy from HDFS
+   $ skyplane cp -r hdfs://... gs://...

docs/on-prem_setup.md

@@ -0,0 +1,41 @@
+# On-Prem Transfers
+
+Currently Skyplane supports On-prem from local disk, NFS, HDFS to cloud storages. 
+
+## HDFS Setup
+
+Skyplane utilizes Pyarrow and libhdfs for HDFS connection. 
+
+**Transfer from HDFS requires prior Hadoop and Java installation.**
+
+* Please refer to [Pyarrow HDFS documentation](https://arrow.apache.org/docs/python/filesystems.html#hadoop-distributed-file-system-hdfs) for necessary environment variable setup.
+
+Note that the cluster needs to communicate to the Skyplane gateway. Please change the incoming firewall for the clusters to allow traffic from Skyplane. 
+
+### Resolving HDFS Datanodes
+
+A file called `hostname` is under the `./skyplane/scripts/on_prem` folder. This file will be used for hostname/datanode IP resolution. This is for datanode's internal IP resolution.
+
+* Copy the hostname/Internal IP for each datanode and the external ip for the corresponding datanode to the file.
+
+* The hostname after writing all the required information should look like this. 
+```text 
+<External IP>   <Datanodes' Hostname or Internal IP>
+```
+
+
+
+### Testing the Transfer
+
+Now you can test running `skyplane cp` to transfer from local disk or HDFS cluster to any cloud storages.
+
+
+```bash
+
+   ---> Copy from local disk
+   $ skyplane cp -r /path/to/local/file gs://...
+
+   ---> Copy from HDFS
+   $ skyplane cp -r hdfs://... gs://...
+
+```