feat(devbox): Bootstrap code for agent.

agent/cli.py

@@ -63,6 +63,7 @@ def ping_server(password: str):
 def config(name, user, workers, proxy_ip=None, sentry_dsn=None):
     config = {
         "benches_directory": f"/home/{user}/benches",
+        "devboxes_directory": f"/home/{user}/devboxes",
         "name": name,
         "tls_directory": f"/home/{user}/agent/tls",
         "nginx_directory": f"/home/{user}/agent/nginx",

agent/devbox.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+from agent.base import Base
+from agent.job import step
+
+if TYPE_CHECKING:
+    from agent.server import Server
+
+
+class Devbox(Base):
+    def __init__(
+        self,
+        devbox_name: str,
+        server: Server,
+        vnc_password: str | None = None,
+        codeserver_password: str | None = None,
+        websockify_port: int | None = None,
+        vnc_port: int | None = None,
+        codeserver_port: int | None = None,
+        browser_port: int | None = None,
+    ):
+        self.devbox_name = devbox_name
+        self.server = server
+        self.directory = os.path.join(self.server.devboxes_directory, devbox_name)
+        self.websockify_port = websockify_port
+        self.vnc_port = vnc_port
+        self.codeserver_port = codeserver_port
+        self.browser_port = browser_port
+        self.vnc_password = vnc_password
+        self.codeserver_password = codeserver_password
+        self.job = None
+        self.step = None
+        self.status = None
+
+    @property
+    def job_record(self):
+        return self.server.job_record
+
+    @property
+    def step_record(self):
+        return self.server.step_record
+
+    @step_record.setter
+    def step_record(self, value):
+        self.server.step_record = value
+
+    @step("Devbox Setup NGINX")
+    def setup_nginx(self, is_devbox=False):
+        from filelock import FileLock
+
+        with FileLock(os.path.join(self.directory, "nginx.config.lock")):
+            self.generate_nginx_config()
+        return self.server._reload_nginx()
+
+    def generate_nginx_config(self):
+        config = {
+            "devbox_name": self.devbox_name,
+            "websockify_port": self.websockify_port,
+            "codeserver_port": self.codeserver_port,
+            "browser_port": self.browser_port,
+        }
+        nginx_config = os.path.join(self.directory, "nginx.conf")
+
+        self.server._render_template("devbox/nginx.conf.jinja2", config, nginx_config)
+
+    @step("Create Devbox Database Volume")
+    def create_devbox_database_volume(self):
+        command = f"docker volume create {self.devbox_name}_db-data"
+        return self.execute(command)
+
+    @step("Create Devbox Home Volume")
+    def create_devbox_home_volume(self):
+        command = f"docker volume create {self.devbox_name}_home"
+        return self.execute(command)
+
+    @step("Run Devbox")
+    def run_devbox(self):
+        command = (
+            f'docker run --security-opt="no-new-privileges=false" -d --rm --name {self.devbox_name} '
+            f"-p {self.websockify_port}:6969 "
+            f"-p {self.codeserver_port}:8443 "
+            f"-p {self.vnc_port}:5901 "
+            f"-p {self.browser_port}:8000 "
+            f"-v {self.devbox_name}_db-data:/var/lib/mysql "
+            f'-e PASSWORD="{self.codeserver_password}" '
+            f'-e VNC_PASSWORD="{self.vnc_password}" '
+            f"-v {self.devbox_name}_home:/home/frappe "
+            "arunmathaisk/devbox-image:latest"
+        )
+        return self.execute(command)
+
+    @step("Stop Devbox")
+    def stop_devbox(self):
+        self.execute(command=f"docker stop {self.devbox_name}")
+        return self.execute(f"docker rm -f {self.devbox_name}")
+
+    def get_devbox_status(self):
+        command = f"docker inspect --format='{{{{.State.Status}}}}' {self.devbox_name}"
+        # We pass the --rm flag thus cant do inspect on non existing container name or container id
+        return self.execute(command, non_zero_throw=False)

agent/server.py

@@ -4,6 +4,7 @@
 import os
 import platform
 import shutil
+import socket
 import tempfile
 import time
 from contextlib import suppress
@@ -15,6 +16,7 @@
 
 from agent.base import AgentException, Base
 from agent.bench import Bench
+from agent.devbox import Devbox
 from agent.exceptions import BenchNotExistsException
 from agent.job import Job, Step, job, step
 from agent.patch_handler import run_patches
@@ -27,6 +29,7 @@ def __init__(self, directory=None):
         self.config_file = os.path.join(self.directory, "config.json")
         self.name = self.config["name"]
         self.benches_directory = self.config["benches_directory"]
+        self.devboxes_directory = self.config["devboxes_directory"]
         self.archived_directory = os.path.join(os.path.dirname(self.benches_directory), "archived")
         self.nginx_directory = self.config["nginx_directory"]
         self.hosts_directory = os.path.join(self.nginx_directory, "hosts")
@@ -763,3 +766,86 @@ def wildcards(self) -> list[str]:
             if "*" in host:
                 wildcards.append(host.strip("*."))
         return wildcards
+
+    def find_available_ports(self, num_ports, starting_port=49152, ending_port=65535):
+        reserved_ports = [22, 80, 443, 3306]  # List of reserved ports
+
+        available_ports = []
+        current_port = starting_port
+
+        while len(available_ports) < num_ports and current_port <= ending_port:
+            if current_port not in reserved_ports:
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                    try:
+                        s.bind(("0.0.0.0", current_port))
+                        available_ports.append(current_port)
+                    except Exception:
+                        pass
+            current_port += 1
+
+        return available_ports
+
+    @step("Initialize Devbox")
+    def devbox_init(self, devbox_name):
+        devboxes_directory = os.path.join(self.devboxes_directory, devbox_name)
+        os.mkdir(devboxes_directory)
+
+    @job("New Devbox", priority="low")
+    def new_devbox(self, devbox_name, vnc_password, codeserver_password):
+        ports = self.find_available_ports(num_ports=4)
+        websockify_port, vnc_port, codeserver_port, browser_port = ports
+        self.devbox_init(devbox_name=devbox_name)
+        devbox = Devbox(
+            devbox_name=devbox_name,
+            server=self,
+            vnc_password=vnc_password,
+            codeserver_password=codeserver_password,
+            websockify_port=websockify_port,
+            vnc_port=vnc_port,
+            codeserver_port=codeserver_port,
+            browser_port=browser_port,
+        )
+        devbox.create_devbox_database_volume()
+        devbox.create_devbox_home_volume()
+        devbox.run_devbox()
+        devbox.setup_nginx()
+        return {
+            "message": {
+                "websockify_port": devbox.websockify_port,
+                "vnc_port": devbox.vnc_port,
+                "codeserver_port": devbox.codeserver_port,
+                "browser_port": devbox.browser_port,
+            }
+        }
+
+    @job("Start Devbox", priority="low")
+    def start_devbox(
+        self,
+        devbox_name,
+        vnc_password,
+        codeserver_password,
+        websockify_port,
+        vnc_port,
+        codeserver_port,
+        browser_port,
+    ):
+        devbox = Devbox(
+            devbox_name=devbox_name,
+            server=self,
+            vnc_password=vnc_password,
+            codeserver_password=codeserver_password,
+            websockify_port=websockify_port,
+            vnc_port=vnc_port,
+            codeserver_port=codeserver_port,
+            browser_port=browser_port,
+        )
+        devbox.run_devbox()
+
+    @job("Stop Devbox", priority="low")
+    def stop_devbox(self, devbox_name):
+        devbox = Devbox(devbox_name=devbox_name, server=self)
+        devbox.stop_devbox()
+
+    def get_devbox_status(self, devbox_name):
+        devbox = Devbox(devbox_name=devbox_name, server=self)
+        return devbox.get_devbox_status()

agent/templates/devbox/nginx.conf.jinja2

@@ -0,0 +1,61 @@
+    server {
+        listen 80;
+        server_name {{ devbox_name }};
+
+        location / {
+            proxy_set_header Host $host;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Origin $scheme://$http_host;   # Dynamic scheme
+            proxy_set_header Connection "upgrade";          # Quotes added
+            proxy_set_header Accept-Encoding gzip;
+            proxy_pass http://127.0.0.1:{{websockify_port}};
+
+            # Disable buffering for WebSocket
+            proxy_buffering off;
+
+            # Increase timeout for WebSocket
+            proxy_read_timeout 86400;
+            proxy_send_timeout 86400;
+        }
+
+        location /websockify {
+            proxy_set_header Host $host;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Origin $scheme://$http_host;   # Already dynamic here
+            proxy_set_header Connection "upgrade";          # Quotes added
+            proxy_pass http://127.0.0.1:{{websockify_port}};
+            proxy_redirect off;
+            # Disable buffering for WebSocket
+            proxy_buffering off;
+
+            # Increase timeout for WebSocket
+            proxy_read_timeout 86400;
+            proxy_send_timeout 86400;
+        }
+
+        location /code/ {
+            rewrite ^/code(/.*)$ $1 break;
+            proxy_pass http://127.0.0.1:{{codeserver_port}};
+            proxy_set_header Host $host;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection upgrade;
+            proxy_set_header Accept-Encoding gzip;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+        location /browser {
+            rewrite ^/browser(/.*)$ $1 break;
+            proxy_pass http://127.0.0.1:{{browser_port}};
+            proxy_set_header Host $host;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection upgrade;
+            proxy_set_header Accept-Encoding gzip;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+    }
+    }

agent/templates/nginx/nginx.conf.jinja2

@@ -83,4 +83,5 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 	include /home/frappe/benches/*/nginx.conf;
+	include /home/frappe/devboxes/*/nginx.conf;
 }

agent/templates/proxy/nginx.conf.jinja2

@@ -156,6 +156,59 @@ server {
 		proxy_pass $upstream_server_hash;
 	}
 
+	location /websockify {
+		proxy_http_version 1.1;
+
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Host $host;
+		proxy_set_header X-Real-IP $remote_addr;
+		proxy_set_header X-Forwarded-For $remote_addr;
+		proxy_set_header X-Forwarded-Proto $scheme;
+		proxy_set_header Connection "upgrade";
+
+		more_set_headers "X-Proxy-Upstream: $upstream_server_hash";
+
+		proxy_pass $upstream_server_hash;
+	}
+
+	location /code {
+		proxy_http_version 1.1;
+
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Host $host;
+		proxy_set_header X-Real-IP $remote_addr;
+		proxy_set_header X-Forwarded-For $remote_addr;
+		proxy_set_header X-Forwarded-Proto $scheme;
+		proxy_set_header Connection "upgrade";
+
+		more_set_headers "X-Proxy-Upstream: $upstream_server_hash";
+
+		proxy_pass $upstream_server_hash;
+	}
+
+	location /browser {
+		proxy_http_version 1.1;
+
+		proxy_read_timeout 600;
+		proxy_buffering off;
+		proxy_buffer_size 128k;
+		proxy_buffers 100 128k;
+
+		proxy_set_header Host $host;
+		proxy_set_header X-Real-IP $remote_addr;
+		proxy_set_header X-Forwarded-For $remote_addr;
+		proxy_set_header X-Forwarded-Proto $scheme;
+
+		more_set_headers "X-Proxy-Upstream: $upstream_server_hash";
+
+		more_set_headers "Access-Control-Allow-Origin: $access_control_allow_origin";
+		more_set_headers "Access-Control-Allow-Headers: $access_control_allow_headers";
+		more_set_headers "Access-Control-Allow-Credentials: $access_control_allow_credentials";
+		more_set_headers "Access-Control-Allow-Methods: $access_control_allow_methods";
+
+		proxy_pass $upstream_server_hash;
+	}
+
 	location / {
 		proxy_http_version 1.1;
 

agent/web.py

@@ -1310,3 +1310,45 @@ def recover_update_inplace(bench: str):
         request.json.get("image"),
     )
     return {"job": job}
+
+
+@application.route("/devboxes", methods=["POST"])
+def new_devbox():
+    data = request.json
+    devbox_name = data.get("devbox_name")
+    vnc_password = data.get("vnc_password")
+    codeserver_password = data.get("codeserver_password")
+    job = Server().new_devbox(
+        devbox_name=devbox_name, vnc_password=vnc_password, codeserver_password=codeserver_password
+    )
+    return {"job": job}
+
+
+@application.route("/devboxes/<string:devbox_name>/start", methods=["POST"])
+def start_devbox(devbox_name: str):
+    data = request.json
+    job = Server().start_devbox(
+        devbox_name=devbox_name,
+        vnc_password=data.get("vnc_password"),
+        codeserver_password=data.get("codeserver_password"),
+        websockify_port=data.get("websockify_port"),
+        vnc_port=data.get("vnc_port"),
+        codeserver_port=data.get("codeserver_port"),
+        browser_port=data.get("browser_port"),
+    )
+    return {"job": job}
+
+
+@application.route("/devboxes/<string:devbox_name>/stop", methods=["POST"])
+def stop_devbox(devbox_name: str):
+    job = Server().stop_devbox(devbox_name=devbox_name)
+    return {"job": job}
+
+
+@application.route("/devboxes/<string:devbox_name>/status", methods=["POST"])
+def get_devbox_status(devbox_name: str):
+    result = Server().get_devbox_status(devbox_name=devbox_name)
+    result["start"] = result["start"].isoformat()
+    result["end"] = result["end"].isoformat()
+    result["duration"] = result["duration"].total_seconds()
+    return jsonify(result)