diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 2f3a818..9f6c9f3 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -7,16 +7,28 @@ on: jobs: coverage: runs-on: ubuntu-latest + env: + VOLUME_BASE_PATH: "/tmp/capy" + AVAILABLE_VOLUMES: "1,2,3" steps: - uses: actions/checkout@v3 with: submodules: true + - uses: actions/setup-java@v3 with: distribution: 'adopt' java-version: '11' + + - name: Create dir structure + run: chmod +x ./.setup-vol-dir && ./.setup-vol-dir $VOLUME_BASE_PATH + + - name: Start services + run: docker-compose up -d + - name: Test run: chmod +x ./gradlew && ./gradlew testCodeCoverageReport + - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 27cc40d..f6ad659 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -11,22 +11,35 @@ jobs: - uses: actions/checkout@v3 with: submodules: true + - uses: actions/setup-java@v3 with: distribution: 'adopt' java-version: '11' + - name: Build - run: chmod +x ./gradlew && ./gradlew build + run: chmod +x ./gradlew && ./gradlew build -x test test: runs-on: ubuntu-latest + env: + VOLUME_BASE_PATH: "/tmp/capy" + AVAILABLE_VOLUMES: "1,2,3" steps: - uses: actions/checkout@v3 with: submodules: true + - uses: actions/setup-java@v3 with: distribution: 'adopt' java-version: '11' + + - name: Create dir structure + run: chmod +x ./.setup-vol-dir && ./.setup-vol-dir $VOLUME_BASE_PATH + + - name: Start services + run: docker-compose up -d + - name: Test run: chmod +x ./gradlew && ./gradlew test diff --git a/.gitignore b/.gitignore index f1c47b1..533a278 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,13 @@ # Ignore Gradle build output directory build +# npm node_modules + +# JDTLS +bin +.settings +.classpath +.project +/workspace +/*/workspace diff --git a/.setup-vol-dir b/.setup-vol-dir new file mode 100755 index 0000000..0336627 --- /dev/null +++ b/.setup-vol-dir @@ -0,0 +1,11 @@ +#!/usr/bin/env sh +# +# Helper to create the directory structure required for Worker service +# $1 -> Base Path + +mkdir -p $1/files/volume1 +mkdir -p $1/files/volume2 +mkdir -p $1/files/volume3 +mkdir -p $1/backups/volume1 +mkdir -p $1/backups/volume2 +mkdir -p $1/backups/volume3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 83019c6..6c05ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### 0.0.5 (2023-09-27) + + +### Features + +* Upload file & Testing ([#14](https://github.com/hawks-atlanta/worker-java/issues/14)) ([757794b](https://github.com/hawks-atlanta/worker-java/commit/757794bf7f570ab425b775f3c8c3f0e165642524)) + +### 0.0.4 (2023-09-13) + ### 0.0.3 (2023-09-11) ### 0.0.2 (2023-08-29) diff --git a/CLI.md b/CLI.md new file mode 100644 index 0000000..2141175 --- /dev/null +++ b/CLI.md @@ -0,0 +1,11 @@ +# CLI + +This document describes how to use the service as a CLI tool. + +## Environment variables + +| Variable | Description | Example | +|--------------------|-----------------------------------------------|-------------------------| +| `METADATA_BASEURL` | Metadata service host | `http://127.0.0.1:8082` | +| `VOLUME_BASE_PATH` | The path where file and backup volumes lie in | `/tmp/store` | +| `VOLUME_COUNT` | How many volumes there is | `3` | diff --git a/Dockerfile b/Dockerfile index f354834..5a3d748 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,13 +5,23 @@ FROM adoptopenjdk/openjdk11:x86_64-alpine-jdk-11.0.20_8-slim as builder COPY . /src WORKDIR /src -RUN ./gradlew build +RUN ./gradlew build -x test # runner FROM adoptopenjdk/openjdk11:x86_64-alpine-jre-11.0.20_8 COPY --from=builder /src/app/build/libs/app-all.jar /opt/app.jar -EXPOSE 8080/tcp + +# create directory structure + +COPY --from=builder /src/.setup-vol-dir /opt/.setup-vol-dir +RUN /opt/.setup-vol-dir /var/capy/store +RUN rm /opt/.setup-vol-dir + +EXPOSE 1099/tcp +ENV METADATA_BASEURL "http://127.0.0.1:8082/api/v1" +ENV VOLUME_BASE_PATH "/var/capy/store" +ENV AVAILABLE_VOLUMES "1,2,3" CMD ["java", "-jar", "/opt/app.jar"] diff --git a/README.md b/README.md index 9d0e8b9..bbce0ff 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ git submodule update --init - (Optional) Use the **gradle wrapper script** (`./gradlew`) for all `gradle` commands. For example: ```sh - ./gradlew build + ./gradlew run ``` - (Optional) Use the provided `nix-shell` to get into a shell with all required dependecies [[install Nix](https://nixos.org/download)]. @@ -41,7 +41,18 @@ gradle run ### Run tests ```sh -gradle test +gradle test # only run tests +gradle testCodeCoverageReport # run tests & generate coverage + +# rerun tests +gradle cleanTest test +gradle cleanTest testCodeCoverageReport +``` + +See test results +```sh +app/build/reports/tests/test/index.html # general +app/build/reports/jacoco/testCodeCoverageReport/html/index.html # coverage ``` ### Format diff --git a/app/build.gradle b/app/build.gradle index 562e18e..68fa050 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -1,3 +1,6 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent + plugins { // Apply the application plugin to add support for building a CLI application in Java. id 'application' @@ -25,7 +28,11 @@ dependencies { implementation 'com.google.guava:guava:31.1-jre' // Use subproject - implementation project(':capyfile.rmi.interfaces') + implementation project(':capyfile.rmi') + + // JSON library + // https://mvnrepository.com/artifact/org.json/json + implementation 'org.json:json:20230618' } // Apply a specific Java toolchain to ease working on different environments. @@ -41,6 +48,31 @@ application { } tasks.named('test') { - // Use JUnit Platform for unit tests. useJUnitPlatform() + + testLogging { + // set options for log level LIFECYCLE + events TestLogEvent.FAILED, + TestLogEvent.PASSED, + TestLogEvent.SKIPPED, + TestLogEvent.STANDARD_ERROR + TestLogEvent.STANDARD_OUT + exceptionFormat TestExceptionFormat.FULL + showExceptions true + showCauses true + showStackTraces true + + // set options for log level DEBUG and INFO + debug { + events TestLogEvent.STARTED, + TestLogEvent.FAILED, + TestLogEvent.PASSED, + TestLogEvent.SKIPPED, + TestLogEvent.STANDARD_ERROR, + TestLogEvent.STANDARD_OUT + exceptionFormat TestExceptionFormat.FULL + } + info.events = debug.events + info.exceptionFormat = debug.exceptionFormat + } } diff --git a/app/src/main/java/worker/App.java b/app/src/main/java/worker/App.java index 5b78953..873f269 100644 --- a/app/src/main/java/worker/App.java +++ b/app/src/main/java/worker/App.java @@ -1,16 +1,12 @@ package worker; -import capyfile.rmi.*; -import java.rmi.NotBoundException; -import java.rmi.RemoteException; -import java.rmi.registry.LocateRegistry; -import java.rmi.registry.Registry; -import java.rmi.server.UnicastRemoteObject; +import worker.config.Config; public class App { public static void main (String[] args) { + Config.initializeFromEnv (); System.out.println ("Worker: Serving RMI"); try { @@ -19,7 +15,7 @@ public static void main (String[] args) server.createStubAndBind (); } catch (Exception e) { - System.out.println (e); + e.printStackTrace (); } } } diff --git a/app/src/main/java/worker/WorkerServiceImpl.java b/app/src/main/java/worker/WorkerServiceImpl.java index 43fa291..565303c 100644 --- a/app/src/main/java/worker/WorkerServiceImpl.java +++ b/app/src/main/java/worker/WorkerServiceImpl.java @@ -1,12 +1,14 @@ -package capyfile.rmi; +package worker; -import capyfile.rmi.interfaces.File; -import capyfile.rmi.interfaces.FileDownload; -import capyfile.rmi.interfaces.IWorkerService; +import capyfile.rmi.DownloadFileArgs; +import capyfile.rmi.File; +import capyfile.rmi.IWorkerService; +import capyfile.rmi.UploadFileArgs; import java.rmi.RemoteException; import java.rmi.registry.LocateRegistry; import java.rmi.registry.Registry; import java.rmi.server.UnicastRemoteObject; +import worker.file.ThreadUploadFile; public class WorkerServiceImpl implements IWorkerService { @@ -18,15 +20,23 @@ public void createStubAndBind () throws RemoteException (IWorkerService)UnicastRemoteObject.exportObject ((IWorkerService)this, 0); // RMI registry - Registry registry = LocateRegistry.createRegistry (1900); + Registry registry = LocateRegistry.createRegistry (1099); // bind remote object registry.rebind ("WorkerService", stub); } - public void uploadFile (File upload) throws RemoteException { return; } + public void uploadFile (UploadFileArgs args) throws RemoteException + { + // delegate to thread + + ThreadUploadFile thread = new ThreadUploadFile (args.uuid, args.contents); + thread.start (); + + return; + } - public File downloadFile (FileDownload download) throws RemoteException + public File downloadFile (DownloadFileArgs args) throws RemoteException { File file = new File ("----", null); return file; diff --git a/app/src/main/java/worker/config/Config.java b/app/src/main/java/worker/config/Config.java new file mode 100644 index 0000000..88595f4 --- /dev/null +++ b/app/src/main/java/worker/config/Config.java @@ -0,0 +1,35 @@ +package worker.config; + +import java.util.Arrays; + +public class Config +{ + private static String metadataBaseUrl = ""; + private static String volumeBasePath = ""; + private static int[] availableVolumes = {}; + + public static String getMetadataBaseUrl () { return metadataBaseUrl; } + public static String getVolumeBasePath () { return volumeBasePath; } + public static int[] getAvailableVolumes () { return availableVolumes; } + private static String getEnv (String env, String def) + { + return System.getenv ().getOrDefault (env, def); + } + + public static void initializeFromEnv () + { + Config.metadataBaseUrl = getEnv ("METADATA_BASEURL", "http://127.0.0.1:8082/api/v1"); + Config.volumeBasePath = getEnv ("VOLUME_BASE_PATH", "/var/capy/store"); + String aVols = getEnv ("AVAILABLE_VOLUMES", ""); + + try { + Config.availableVolumes = + Arrays.stream (aVols.split (",")).mapToInt (Integer::parseInt).toArray (); + if (availableVolumes.length == 0) { + throw new RuntimeException ("Invalid var AVAILABLE_VOLUMES: No volumes specified"); + } + } catch (Exception e) { + throw new RuntimeException ("Invalid var AVAILABLE_VOLUMES: Couldn't parse"); + } + } +} diff --git a/app/src/main/java/worker/file/ThreadUploadFile.java b/app/src/main/java/worker/file/ThreadUploadFile.java new file mode 100644 index 0000000..e12037d --- /dev/null +++ b/app/src/main/java/worker/file/ThreadUploadFile.java @@ -0,0 +1,68 @@ +package worker.file; + +import java.io.File; +import java.io.FileOutputStream; +import worker.config.Config; +import worker.services.MetadataService; + +public class ThreadUploadFile extends Thread +{ + private static final int READY_ATTEMPTS = 5; + private static final long READY_TIMEOUT = 5000; + String fileUUID; + byte[] contents; + + public ThreadUploadFile (String fileUUID, byte[] contents) + { + this.fileUUID = fileUUID; + this.contents = contents; + } + + public void run () + { + // pick volume to save file + + int[] vols = Config.getAvailableVolumes (); + int volume = vols[(int)(Math.random () * vols.length)]; + + String filePath = String.format ( + "%1$s/files/volume%2$d/%3$s", Config.getVolumeBasePath (), volume, this.fileUUID); + String backupPath = String.format ( + "%1$s/backups/volume%2$d/%3$s", Config.getVolumeBasePath (), volume, this.fileUUID); + + // write + + if (!writeFile (filePath) || !writeFile (backupPath)) { + return; // couldn't write + } + + // mark file as ready + + for (int i = 0; i < READY_ATTEMPTS; i++) { + if (MetadataService.fileReady (fileUUID, volume)) { + break; + } + try { + sleep (READY_TIMEOUT); + } catch (Exception e) { + e.printStackTrace (); + } + } + } + + private boolean writeFile (String path) + { + // write file + + File outputFile = new File (path); + + try (FileOutputStream outputStream = new FileOutputStream (outputFile)) { + outputStream.write (this.contents); + } catch (Exception e) { + e.printStackTrace (); + return false; + } + + return true; + } +} diff --git a/app/src/main/java/worker/services/MetadataService.java b/app/src/main/java/worker/services/MetadataService.java new file mode 100644 index 0000000..3beea47 --- /dev/null +++ b/app/src/main/java/worker/services/MetadataService.java @@ -0,0 +1,95 @@ +package worker.services; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.util.UUID; +import org.json.JSONObject; +import worker.config.Config; + +public class MetadataService +{ + public static boolean fileReady (String fileUUID, int volume) + { + JSONObject body = new JSONObject (); + body.put ("volume", String.valueOf (volume)); + + String uri = String.format ("%s/files/ready/%s", Config.getMetadataBaseUrl (), fileUUID); + + // PUT request + + try { + HttpResponse res = HttpClient.newHttpClient ().send ( + HttpRequest.newBuilder () + .uri (URI.create (uri)) + .PUT (BodyPublishers.ofString (body.toString ())) + .build (), + HttpResponse.BodyHandlers.ofString ()); + + if (res.statusCode () == 204) { + return true; + } + } catch (Exception e) { + e.printStackTrace (); + } + + return false; + } + + public static boolean checkReady (String fileUUID) + { + String uri = String.format ("%s/files/metadata/%s", Config.getMetadataBaseUrl (), fileUUID); + + // GET + + try { + HttpResponse res = HttpClient.newHttpClient ().send ( + HttpRequest.newBuilder ().uri (URI.create (uri)).GET ().build (), + HttpResponse.BodyHandlers.ofString ()); + + if (res.statusCode () == 200) { + return true; + } + } catch (Exception e) { + e.printStackTrace (); + } + + return false; + } + + public static UUID + saveFile (UUID userUUID, UUID directoryUUID, String filetype, String filename, int filesize) + throws Exception + { + + JSONObject body = new JSONObject (); + body.put ("userUUID", userUUID); + body.put ("parentUUID", directoryUUID == null ? JSONObject.NULL : directoryUUID); + body.put ("fileType", "archive"); + body.put ("fileName", filename); + body.put ("fileExtension", filetype == null ? JSONObject.NULL : filetype); + body.put ("fileSize", filesize); + + // post + + try { + HttpResponse res = HttpClient.newHttpClient ().send ( + HttpRequest.newBuilder () + .uri (URI.create (Config.getMetadataBaseUrl () + "/files")) + .POST (BodyPublishers.ofString (body.toString ())) + .build (), + HttpResponse.BodyHandlers.ofString ()); + + JSONObject resBody = new JSONObject (res.body ()); + if (res.statusCode () == 201) { + return UUID.fromString (resBody.getString ("uuid")); + } + } catch (Exception e) { + e.printStackTrace (); + } + + throw new Exception ("Couldn't save file metadata"); + } +} diff --git a/app/src/test/java/worker/TestUtil.java b/app/src/test/java/worker/TestUtil.java new file mode 100644 index 0000000..4303ffd --- /dev/null +++ b/app/src/test/java/worker/TestUtil.java @@ -0,0 +1,23 @@ +package worker; +import java.lang.reflect.Field; +import java.util.Map; +import java.util.Random; + +public class TestUtil +{ + public static byte[] randomBytes (int size) + { + byte[] buff = new byte[size]; + (new Random ()).nextBytes (buff); + return buff; + } + + @SuppressWarnings ({ "unchecked" }) + public static void updateEnv (String name, String val) throws ReflectiveOperationException + { + Map env = System.getenv (); + Field field = env.getClass ().getDeclaredField ("m"); + field.setAccessible (true); + ((Map)field.get (env)).put (name, val); + } +} diff --git a/app/src/test/java/worker/UTConfig.java b/app/src/test/java/worker/UTConfig.java new file mode 100644 index 0000000..979a13d --- /dev/null +++ b/app/src/test/java/worker/UTConfig.java @@ -0,0 +1,36 @@ +package worker; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.Test; +import worker.config.Config; + +class UTConfig +{ + @Test void configFromEnv () + { + + try { + Config.initializeFromEnv (); + } catch (Exception e) { + fail ("Couldn't initialize vars"); + } + + try { + TestUtil.updateEnv ("AVAILABLE_VOLUMES", ","); + Config.initializeFromEnv (); + fail (); + } catch (Exception e) { + assertTrue (e instanceof RuntimeException, "Should throw"); + } + + try { + TestUtil.updateEnv ("AVAILABLE_VOLUMES", ""); + Config.initializeFromEnv (); + fail (); + } catch (Exception e) { + assertTrue (e instanceof RuntimeException, "Should throw"); + } + } +} diff --git a/app/src/test/java/worker/UTFileIO.java b/app/src/test/java/worker/UTFileIO.java new file mode 100644 index 0000000..464be1c --- /dev/null +++ b/app/src/test/java/worker/UTFileIO.java @@ -0,0 +1,50 @@ +package worker; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import capyfile.rmi.UploadFileArgs; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import worker.config.Config; +import worker.services.MetadataService; + +class UTFileIO +{ + @BeforeEach void setup () + { + try { + TestUtil.updateEnv ("AVAILABLE_VOLUMES", "1,2,3"); + } catch (Exception e) { + e.printStackTrace (); + } + Config.initializeFromEnv (); + } + + @Test void uploadFile () + { + WorkerServiceImpl server = new WorkerServiceImpl (); + + try { + // get file uuid + + String uuid = + MetadataService + .saveFile (UUID.randomUUID (), null, "txt", UUID.randomUUID ().toString (), 32) + .toString (); + + UploadFileArgs req = new UploadFileArgs (uuid, TestUtil.randomBytes (32)); + server.uploadFile (req); + + // wait until it's written + + Thread.sleep (2_000); + assertTrue (MetadataService.checkReady (req.uuid), () -> "File is ready"); + return; + } catch (Exception e) { + e.printStackTrace (); + } + fail ("Couldn't check file"); + } +} diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..3d8cae1 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,51 @@ +version: '3.1' + +networks: + net: + +services: + metadata: + image: ghcr.io/hawks-atlanta/metadata-scala:latest + container_name: metadata + restart: on-failure + depends_on: + - postgres-db + ports: + - "127.0.0.1:8082:8080" + environment: + DATABASE_HOST: postgres-db + DATABASE_PORT: 5432 + DATABASE_NAME: metadatadb + DATABASE_USER: username + DATABASE_PASSWORD: password + networks: + - net + + postgres-db: + image: postgres:latest + container_name: postgres-db + restart: on-failure + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_DB=metadatadb + - POSTGRES_PASSWORD=password + - POSTGRES_USER=username + networks: + - net + + postgres-admin: + image: adminer + ports: + - "127.0.0.1:5050:8080" + environment: + ADMINER_DESIGN: hever + ADMINER_DEFAULT_SERVER: postgres-db + ADMINER_DEFAULT_USER: username + ADMINER_DEFAULT_PASSWORD: password + ADMINER_DEFAULT_TYPE: postgresql + ADMINER_DEFAULT_PORT: 5432 + ADMINER_DEFAULT_DB: metadatadb + networks: + - net + diff --git a/package-lock.json b/package-lock.json index 4873208..33ae3b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "devDependencies": { "standard-version": "^9.5.0" }, - "version": "0.0.3" + "version": "0.0.5" }, "node_modules/@babel/code-frame": { "version": "7.22.13", @@ -2099,5 +2099,5 @@ } } }, - "version": "0.0.3" + "version": "0.0.5" } diff --git a/package.json b/package.json index 3900ffb..741c616 100644 --- a/package.json +++ b/package.json @@ -2,5 +2,5 @@ "devDependencies": { "standard-version": "^9.5.0" }, - "version": "0.0.3" + "version": "0.0.5" } diff --git a/rmi b/rmi index 68560b7..ca6f035 160000 --- a/rmi +++ b/rmi @@ -1 +1 @@ -Subproject commit 68560b7fd64a875585d2cb974537738f578b5437 +Subproject commit ca6f0358311bb9ba54c250a9aa8db79b468df665 diff --git a/settings.gradle b/settings.gradle index 0c51127..fa34ce6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,8 +5,8 @@ plugins { rootProject.name = 'worker' -include ('app', ':capyfile.rmi.interfaces') +include ('app', ':capyfile.rmi') // Include subproject -project (':capyfile.rmi.interfaces').projectDir = file ('./rmi/lib/') +project (':capyfile.rmi').projectDir = file ('./rmi/lib/')