From fc2ecea2cd41ba55e97232fb8385c34ec5390491 Mon Sep 17 00:00:00 2001 From: Anan Zhuang Date: Wed, 29 Mar 2023 00:00:58 +0000 Subject: [PATCH] Add docker files and instructions for debugging Selenium functional tests This PR introduces new Docker files to enable debugging of Selenium functional tests for Docker users. It configures a VNC viewer for real-time browser interaction monitoring during test execution. Additionally, a new section is added to the documentation detailing the process of running and debugging Selenium functional tests using Docker and a VNC viewer. Issue Resolve https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3700 Signed-off-by: Anan Zhuang --- dev-tools/install-docker-dev.sh | 16 ++++- docs/docker-dev/Dockerfile.selenium | 70 +++++++++++++++++++++ docs/docker-dev/docker-compose.selenium.yml | 24 +++++++ docs/docker-dev/docker-dev-setup-manual.md | 59 +++++++++++++---- docs/docker-dev/start-vnc.sh | 15 +++++ 5 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 docs/docker-dev/Dockerfile.selenium create mode 100644 docs/docker-dev/docker-compose.selenium.yml create mode 100644 docs/docker-dev/start-vnc.sh diff --git a/dev-tools/install-docker-dev.sh b/dev-tools/install-docker-dev.sh index 0d40d12afed6..39badf9029cc 100644 --- a/dev-tools/install-docker-dev.sh +++ b/dev-tools/install-docker-dev.sh @@ -21,8 +21,22 @@ osd_do_copy_dev_docker_files(){ mkdir -p "$INSTALL_DIR" osd_download -s "$ENTRYPOINT_SRC" -o "./$INSTALL_DIR/entrypoint.sh" osd_download -s "$DOCKER_COMPOSE_SRC" -o "./$INSTALL_DIR/docker-compose.yml" + + if [ "$1" = "--ftr" ]; then + printf "run ftr" + local START_VNC_SRC + START_VNC_SRC="https://raw.githubusercontent.com/your-repo/your-branch/start-vnc.sh" + local DOCKERFILE_SELENIUM_SRC + DOCKERFILE_SELENIUM_SRC="https://raw.githubusercontent.com/your-repo/your-branch/Dockerfile.selenium" + local DOCKER_COMPOSE_SELENIUM_SRC + DOCKER_COMPOSE_SELENIUM_SRC="https://raw.githubusercontent.com/your-repo/your-branch/docker-compose.selenium.yml" + + osd_download -s "$START_VNC_SRC" -o "./$INSTALL_DIR/start-vnc.sh" + osd_download -s "$DOCKERFILE_SELENIUM_SRC" -o "./$INSTALL_DIR/Dockerfile.selenium" + osd_download -s "$DOCKER_COMPOSE_SELENIUM_SRC" -o "./$INSTALL_DIR/docker-compose.selenium.yml" + fi } -osd_do_copy_dev_docker_files +osd_do_copy_dev_docker_files $1 } # this ensures the entire script is downloaded # diff --git a/docs/docker-dev/Dockerfile.selenium b/docs/docker-dev/Dockerfile.selenium new file mode 100644 index 000000000000..059b2d74afa5 --- /dev/null +++ b/docs/docker-dev/Dockerfile.selenium @@ -0,0 +1,70 @@ +FROM abbyhu/opensearch-dashboards-dev:latest + +# Switch to root user +USER root + +# Install the locales package +# Uncomment the en_US.UTF-8 UTF-8 line in the sytstem /etc/locale.gen file +# Then generate the locales and update the system locale to en_US.UTF-8 +# Install all other requested packages +RUN apt-get update && \ + apt-get install -y locales && \ + sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=en_US.UTF-8 && \ + apt-get install -y xvfb x11vnc openbox lxde-core lxterminal wget apt-transport-https sudo + +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Create the LXTerminal configuration directory and set the encoding +RUN mkdir -p /etc/xdg/lxterminal && \ + echo '[General]' >> /etc/xdg/lxterminal/lxterminal.conf && \ + echo 'encoding=UTF-8' >> /etc/xdg/lxterminal/lxterminal.conf + +# Specify the version of Chrome that matches the version of chromedriver in the package.json. +#ARG CHROME_VERSION=107.0.5304.121-1 + +## Install Google Chrome version 107 +#RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - && \ +# echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' > /etc/apt/sources.list.d/google-chrome.list && \ +# apt-get update && \ +# wget -O /tmp/chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROME_VERSION}_amd64.deb && \ +# apt-get install -y /tmp/chrome.deb --no-install-recommends && \ +# rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install Google Chrome +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ + echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' > /etc/apt/sources.list.d/google-chrome.list && \ + apt-get update && \ + apt-get install -y google-chrome-stable && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Create the directory and set the ownership for osd-dev user +RUN mkdir -p /docker-workspace/OpenSearch-Dashboards/.opensearch && \ + chown -R osd-dev /docker-workspace/OpenSearch-Dashboards/.opensearch + +COPY entrypoint.sh /entrypoint.sh +COPY start-vnc.sh /start-vnc.sh + +RUN chmod +x /entrypoint.sh /start-vnc.sh + +ENTRYPOINT ["/entrypoint.sh"] + +# Create a Google Chrome desktop file with the specified launch options. +# Currently Google Chrome is not available in the menu of your VNC Viewer session. +# To enable that, you need to open the terminal and run: +# google-chrome --no-sandbox --disable-gpu --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 +# This part is added to automate this process by creating a desktop file for Google Chrome. +RUN echo '[Desktop Entry]\n\ +Version=1.0\n\ +Name=Google Chrome\n\ +GenericName=Web Browser\n\ +Comment=Access the Internet\n\ +Exec=google-chrome --no-sandbox --disable-gpu --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222 %U\n\ +Terminal=false\n\ +Icon=google-chrome\n\ +Type=Application\n\ +Categories=Network;WebBrowser;\n\ +MimeType=text/html;text/xml;application/xhtml_xml;x-scheme-handler/http;x-scheme-handler/https;'\ +> /usr/share/applications/google-chrome.desktop diff --git a/docs/docker-dev/docker-compose.selenium.yml b/docs/docker-dev/docker-compose.selenium.yml new file mode 100644 index 000000000000..c0ed2bafcaa4 --- /dev/null +++ b/docs/docker-dev/docker-compose.selenium.yml @@ -0,0 +1,24 @@ +version: '3' +services: + selenium-test: + build: + context: . + dockerfile: Dockerfile.selenium + container_name: selenium-test + depends_on: + - dev-env + environment: + - DISPLAY=:99 + volumes: + - /dev/shm:/dev/shm + - .:/workspace + networks: + - opensearch-net + ports: + - 5900:5900 + entrypoint: ["/start-vnc.sh"] +volumes: + opensearch-data: + osd-dev: +networks: + opensearch-net: diff --git a/docs/docker-dev/docker-dev-setup-manual.md b/docs/docker-dev/docker-dev-setup-manual.md index 53b80e9ff633..80236a79847a 100644 --- a/docs/docker-dev/docker-dev-setup-manual.md +++ b/docs/docker-dev/docker-dev-setup-manual.md @@ -1,18 +1,18 @@ # Docker Development Environment Setup The following instructions demonstrate how to set up a development environment for OpenSearch Dashboards using Docker. It utilizes tools such as `Docker` and `VS Code`, and users should be familiar with the basic usages of them. Users will be able to develop and run the application inside VS Code without additional configurations. -1. Install [Docker](https://docs.docker.com/get-docker/) if not already installed. +1. Install [Docker](https://docs.docker.com/get-docker/) if not already installed. * Make sure that Docker daemon is running. (For windows and macos，you need to have [Docker Desktop](https://docs.docker.com/desktop/), or its alternatives, such as [Finch](https://github.com/runfinch/finch)) 2. In the terminal, run the command below. - * This should create a folder named `opensearch-dashboards-docker-dev` and it should contain two files: `docker-compose.yml` and `entrypoint.sh`. + * This should create a folder named `opensearch-dashboards-docker-dev` and it should contain two files: `docker-compose.yml` and `entrypoint.sh`. * Here is the link to the installer script: `https://raw.githubusercontent.com/opensearch-project/OpenSearch-Dashboards/main/dev-tools/install-docker-dev.sh` if needed. ```bash curl -o- https://raw.githubusercontent.com/opensearch-project/OpenSearch-Dashboards/main/dev-tools/install-docker-dev.sh | bash ``` -3. Open VS Code or [install it](https://code.visualstudio.com/download), if it's not already installed. +3. Open VS Code or [install it](https://code.visualstudio.com/download), if it's not already installed. * Make sure VS Code has the extensions `Dev Containers` and `Docker` installed. If not, go to `Extensions` tab, search and install them. 4. Under the Discover tab, click `Open Folder`, and open the `opensearch-dashboards-docker-dev` folder that we just created. @@ -21,22 +21,22 @@ curl -o- https://raw.githubusercontent.com/opensearch-project/OpenSearch-Dashboa * If fork repo has not been created: Go to [OpenSearch Dashboards github page](https://github.com/opensearch-project/OpenSearch-Dashboards) and under fork, select create a new fork, and then copy the https link of the fork url and use it in the above command. The command needs to be re-run every time it re-start the docker compose file in a new terminal. ```bash export REPO_URL=[insert your fork repo url here] -``` - -6. Run the `docker-compose.yml` file in the background by typing: +``` + +6. Run the `docker-compose.yml` file in the background by typing: ```bash docker compose up -d --build ``` -7. Under the `Docker` tab in VS Code, verify that there are two containers running: `opensearchproject/opensearch:latest` and `abbyhu/opensearch-dashboards-dev:latest`. - * This can also be verified by using the command line: +7. Under the `Docker` tab in VS Code, verify that there are two containers running: `opensearchproject/opensearch:latest` and `abbyhu/opensearch-dashboards-dev:latest`. + * This can also be verified by using the command line: ```bash docker ps ``` -8. Right-click `abbyhu/opensearch-dashboards-dev:latest`, and select `Attach Visual Studio Code`. - * This will ssh into the container and you will be able to view and edit the files using VS Code as the code editor. - * If you do not wish to use VS Code as the code editor, the alternative way of ssh into the container is by using the command below: +8. Right-click `abbyhu/opensearch-dashboards-dev:latest`, and select `Attach Visual Studio Code`. + * This will ssh into the container and you will be able to view and edit the files using VS Code as the code editor. + * If you do not wish to use VS Code as the code editor, the alternative way of ssh into the container is by using the command below: ```bash docker exec -it dev-env /bin/bash ``` @@ -48,7 +48,7 @@ docker compose up -d --build yarn start:docker ``` -11. Now that OpenSearch Dashboards is running, you should be able to see a log line similar to `[info][server][OpenSearchDashboards][http] http server running at http://0.0.0.0:5603/dog`. +11. Now that OpenSearch Dashboards is running, you should be able to see a log line similar to `[info][server][OpenSearchDashboards][http] http server running at http://0.0.0.0:5603/dog`. * The last three letters `dog` are randomly generated every time we start dashboards. 12. Wait for the optimizer to run, which takes about 100s - 200s. Once the optimizer is finished running, it will show a line such as `[success][@osd/optimizer] 48 bundles compiled successfully after 204.9 sec, watching for changes`. @@ -57,3 +57,38 @@ yarn start:docker * Files are constantly watched, so when you make code changes, OpenSearch Dashboards will rebuild and restart automatically. Refresh the link in the browser and the new changes should be applied. 14. `Git` is already configured in the `entrypoint.sh` file, and the remote is already tracking the fork repository. You can start contributing by creating your branch off the main, and commit your first PR! + +# Debug Functional Tests +This section explains how to run Selenium functional tests using Docker and debug the tests through browser interaction. + +1. Install a VNC viewer, such as [RealVNC](https://www.realvnc.com/en/connect/download/viewer/), if you haven't already. This will allow you to view and interact with the browser running the Selenium tests. The following steps are using RealVNC as a setup example. + +2. Make sure you have completed steps 1-5 in the [Docker Development Environment Setup](#docker-development-environment-setup). Now, ensure you have 5 files, `docker-compose.yml`, `docker-compose.selenium.yml`, `Dockerfile.selenium`, `entrypoint.sh` and `start-vnc.sh`. You can also download them by running the installer script. This time you need to pass a parameter `--ftr`. + +```bash +curl -o- https://raw.githubusercontent.com/opensearch-project/OpenSearch-Dashboards/main/dev-tools/install-docker-dev.sh | bash -s -- --ftr +``` + +3. In the terminal, run the following commands to stop any running containers, build the new configuration, and start the containers with the Selenium setup: + +```bash +docker-compose -f docker-compose.yml -f docker-compose.selenium.yml down +docker-compose -f docker-compose.yml -f docker-compose.selenium.yml build +docker-compose -f docker-compose.yml -f docker-compose.selenium.yml up -d +``` + +4. Under the Docker tab in VS Code, you should see three containers running: opensearchproject/opensearch:latest, abbyhu/opensearch-dashboards-dev:latest, and selenium-test. + +5. Right-click `selenium-test`, and select `Attach Visual Studio Code`. This step is the same as step 8 in the Docker Development Environment Setup. + +6. Connect to the VNC server running in the Docker container. First, launch the VNC viewer, then enter `localhost:5900` at the top and press return. You will see a popup window labeled `Unencrypted connection`; click continue. Enter the password (the default password is 123), which is configured in `start-vnc.sh`. + +7. After entering the VNC viewer, click the menu at the bottom left. Then click `Internet` to ensure Google works. Also, click `System Tools` and select `LXTerminal` to verify you can type in the terminal. The default terminal path should be `/docker-workspace/OpenSearch-Dashboards`. + +8. Run OpenSearch and OpenSearch Dashboards. First, start OpenSearch by executing `yarn opensearch snapshot` in one terminal. In a second terminal, run `yarn start --no-base-path` to start OpenSearch Dashboards on port 5601. Before running any functional tests, make sure OpenSearch Dashboards is running on port 5601. This process may take 2-10 mins, as it requires all the bundles to be assembled. Please be patient. However, this only occurs the first time you run it; afterward, everything is cached, significantly reducing the startup time for OpenSearch Dashboards. The initial bundling speed is dependent on your hardware limitations and VNC settings. Feel free to make adjustments to improve the process as needed. + +9. Open a separate terminal and update Chromedriver by executing `node scripts/upgrade_chromedriver.js`, followed by `yarn osd bootstrap`. This will update the Chromedriver version within the node_modules directory, allowing you to use a compatible version with your installed Google Chrome. + +10. To execute a single functional test, use the following example: `node scripts/functional_tests.js --config test/functional/config.js --include ciGroup9`. This command runs all the functional tests within ciGroup9. + +11. The Selenium tests will be executed in the browser, viewable through the VNC viewer. You can monitor the test progress in real-time. diff --git a/docs/docker-dev/start-vnc.sh b/docs/docker-dev/start-vnc.sh new file mode 100644 index 000000000000..4101d46db5f1 --- /dev/null +++ b/docs/docker-dev/start-vnc.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Start the VNC server +export DISPLAY=:99 + +mkdir -p /home/.vnc + +Xvfb :99 -screen 0 1024x768x16 & + +x11vnc -forever -display :99 -rfbport 5900 -passwd 123 -bg -o /home/.vnc/x11vnc.log + +sudo -u osd-dev startlxde & + +# Keep the container running +tail -f /dev/null