Skip to content

Commit

Permalink
feat: download file (#46)
Browse files Browse the repository at this point in the history
* feat: createFile handler

* fix: Use byte primitive

* chore: Remove RMI demo from App.java

* chore: apply clang-format

* feat: Abstract rmi connection

* chore: Update rmi lib

* chore: Utils package

* feat: Auth abstraction

* feat: Authenticate jwt on upload file

* feat: Get user's UUID

* feat: Get file mimetype from bytes

* fix: Make metadata & auth services use a single postgres instance

* feat: Pre declare userUUID

* fix: Get userUUID from token

* docs: Update env vars default values

* feat(upload-file): save metadata

* fix: ManagerRMI interface import

* chore: Remove debug msgs

* refactor: Switch SOAP data types sufix to prefix

* chore: Remove debug msgs

* chore: Update RMI submodule

* refactor: Redesign request / response types

* fix: Adapt current service implementation with new interface design

* docs: Add open api specification

* docs: Add API definition to README.md

* docs: Improve CLI.md examples

* refactor: Redesign request / response types

* chore: Update python sample client

* chore: Update submodule

* chore: update submodule

* fix: Typo in request type

* feat(upload-file): Add file size limits

* chore: Use error instead of success

* fix: Simplify http requests

* refactor: External services: Managers to Services

* refactor: Separate SOAP request handlers as controllers

* chore: Add http codes to auth method

* feat: Add http codes to Metadata save file

* feat: Better http code handling

* feat: Add ResStatus generic downcast support

* feat: Validate field types

* feat: Adapt getUserUUID method to a ResStatus response

* test: Test util to tweak configurations

* fix: controller exit codes

* test: Authorization service abstraction

* test: FileIO upload-file

* docs: Update README & API spec

* test: show realtime std, err on tests execution

* feat: update python client demo

* chore: Update docker compose with worker service & adminer

* docs: Update default MetadataURI env var

* chore: Use worker release image

* ci: Fix workflows: testing, coverage

* fix: Do not assume api sufix

* fix: Exclude tests on Dockerfile build

* chore: Update RMI types to use UUID

* feat(ServiceMetadata): canRead wrapper added

* feat(ServiceMetadata): getFileMetadata wrapper added

* chore: update fileDownload res/req types

* feat: fileDownload service implementation

* chore: Add RMI Streaming library

* feat(fileDownload): Add 404 code case

* fix: Login return code

* test: Add new ASCII generator

* test: Update uploadFile test to save relevant data for other tests

* test: Add fileDownload tests

* chore: Remove debug messages & unclutter test output

* fix: Add workaround issue Metadata#74

* chore: increase buffer size for data streaming

* chore: update submodule

* feat: Update demo python SOAP client

* fix: Nullable extension result

* fix: Give time for file to finish uploading before next test

* ci: run test workflows on PRs against main

* chore: Improve cli client (#55)

* refactor: Make cli client reusable
---------

Co-authored-by: woynert <[email protected]>

---------

Co-authored-by: Pedro Andrés Chaparro Quintero <[email protected]>
  • Loading branch information
Woynert and PedroChaparro authored Oct 3, 2023
1 parent a14b096 commit 0d2eaa1
Show file tree
Hide file tree
Showing 23 changed files with 497 additions and 75 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ jobs:

- uses: actions/setup-java@v3
with:
distribution: 'adopt'
java-version: '11'
distribution: "adopt"
java-version: "11"

- name: Set up docker environment
run: docker compose up -d
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/format.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Format

on:
pull_request:
branches: ["dev"]
branches: ["dev", "main"]

jobs:
formatter:
Expand All @@ -14,4 +14,3 @@ jobs:

- name: Clang check
run: chmod +x ./format.sh && ./format.sh clang-check

2 changes: 1 addition & 1 deletion .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ jobs:
release_name: Release ${{ needs.versioning.outputs.version }}
draft: false
prerelease: false

build-docker:
runs-on: ubuntu-latest
permissions:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tagging.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ jobs:
- name: Changelog
run: 'npx standard-version --message "[ci skip] chore(release): %s"'
- name: Push changes
run: git push --follow-tags --force origin dev
run: git push --follow-tags --force origin dev
12 changes: 6 additions & 6 deletions .github/workflows/testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Test

on:
pull_request:
branches: ["dev"]
branches: ["dev", "main"]

jobs:
build:
Expand All @@ -14,12 +14,12 @@ jobs:

- uses: actions/setup-java@v3
with:
distribution: 'adopt'
java-version: '11'
distribution: "adopt"
java-version: "11"

- name: Build
run: chmod +x ./gradlew && ./gradlew build -x test

test:
runs-on: ubuntu-latest
steps:
Expand All @@ -29,8 +29,8 @@ jobs:

- uses: actions/setup-java@v3
with:
distribution: 'adopt'
java-version: '11'
distribution: "adopt"
java-version: "11"

- name: Set up docker environment
run: docker compose up -d
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ node_modules

# python envs
pyenv
venv

# python cache
**/__pycache__

# JDTLS
bin
Expand Down
7 changes: 5 additions & 2 deletions app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ dependencies {
// JWT
implementation 'com.auth0:java-jwt:4.4.0'

// RMI remote stream
// https://mvnrepository.com/artifact/com.healthmarketscience.rmiio/rmiio
implementation 'com.healthmarketscience.rmiio:rmiio:2.1.2'

// --- Validator lib ---

// https://mvnrepository.com/artifact/org.hibernate.validator/hibernate-validator
Expand All @@ -42,7 +46,6 @@ dependencies {
// https://mvnrepository.com/artifact/org.glassfish.expressly/expressly
implementation 'org.glassfish.expressly:expressly:5.0.0'


// --- SOAP service ---

// https://mvnrepository.com/artifact/javax.xml.ws/jaxws-api
Expand Down Expand Up @@ -81,7 +84,7 @@ tasks.named('test') {
events TestLogEvent.FAILED,
TestLogEvent.PASSED,
TestLogEvent.SKIPPED,
TestLogEvent.STANDARD_ERROR
TestLogEvent.STANDARD_ERROR,
TestLogEvent.STANDARD_OUT
exceptionFormat TestExceptionFormat.FULL
showExceptions true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public static ResSession account_register (Credentials credentials)
}

} catch (IOException | InterruptedException e) {
e.printStackTrace ();
System.err.println (e);
}

return res;
Expand Down
2 changes: 1 addition & 1 deletion app/src/main/java/gateway/controller/CtrlAuthLogin.java
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ public static ResSession auth_login (Credentials credentials)
res.error = true;
res.msg = jsonObject.getString ("msg");
}

} catch (Exception e) {
// Handle exceptions such as IOException and InterruptedException, if they occur.
System.err.println (e);
resFileNew.code = 500;
resFileNew.error = true;
resFileNew.msg = "Internal error, try again later";
Expand Down
80 changes: 80 additions & 0 deletions app/src/main/java/gateway/controller/CtrlFileDownload.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package gateway.controller;

import capyfile.rmi.IWorkerService;
import gateway.services.ServiceAuth;
import gateway.services.ServiceMetadata;
import gateway.services.ServiceWorker;
import gateway.services.UtilValidator;
import gateway.soap.request.ReqFile;
import gateway.soap.response.ResFileDownload;
import gateway.soap.response.ResStatus;
import java.io.FileNotFoundException;
import java.util.UUID;

public class CtrlFileDownload
{
public static ResFileDownload file_download (ReqFile args)
{
ResFileDownload s = new ResFileDownload ();
UUID userUUID;

ResStatus resValidate = UtilValidator.validate (args);
if (resValidate.error) {
return ResStatus.downCast (ResFileDownload.class, resValidate);
}

// auth

ResStatus resAuth = ServiceAuth.authenticate (args.token);
if (resAuth.error) {
return ResStatus.downCast (ResFileDownload.class, resAuth);
}
userUUID = UUID.fromString (ServiceAuth.tokenGetClaim (args.token, "uuid"));

// file exists & user has access

ResStatus resRead = ServiceMetadata.canRead (userUUID, args.fileUUID);
if (resRead.error) {
return ResStatus.downCast (ResFileDownload.class, resRead);
}

// which volume

ServiceMetadata.ResFileMetadata resMeta = ServiceMetadata.getFileMetadata (args.fileUUID);
if (resMeta.error) {
return ResStatus.downCast (ResFileDownload.class, resMeta);
}

// try download it from worker

try {
IWorkerService server = ServiceWorker.getServer ();
s.fileContent = ServiceWorker.downloadFile (server, args.fileUUID, resMeta.volume);
s.fileName = resMeta.name;
s.fileUUID = args.fileUUID;

s.code = 200;
s.error = false;
s.msg = "";
} catch (Exception e) {

System.err.println (e);
if (e instanceof FileNotFoundException) {
System.err.println ("File not found");
s.code = 404;
s.msg = "File not found";

// NOTE: At this point the metadata service claims a file
// exists in this volume. But it wasn't found.
} else {
System.err.println ("Can't connect to RMI");
s.code = 500;
s.msg = "Internal error, try again later";
}

s.error = true;
}

return s;
}
}
2 changes: 1 addition & 1 deletion app/src/main/java/gateway/controller/CtrlFileUpload.java
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public static ResFileNew file_upload (ReqFileUpload args)
s.msg = "Your file is being uploaded";
} catch (Exception e) {
System.err.println ("Can't connect to RMI");
e.printStackTrace ();
System.err.println (e);

s.code = 500;
s.error = true;
Expand Down
4 changes: 2 additions & 2 deletions app/src/main/java/gateway/services/ServiceAuth.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public static ResStatus authenticate (String token)
s.msg = jsonObject.getString ("msg");
}
} catch (Exception e) {
e.printStackTrace ();
System.err.println (e);
s.code = 500;
s.error = true;
s.msg = "Internal server error. Try again later";
Expand Down Expand Up @@ -84,7 +84,7 @@ public static ResUUID getUserUUID (String token, String username)
s.msg = jsonObject.getString ("msg");
}
} catch (Exception e) {
e.printStackTrace ();
System.err.println (e);
s.code = 500;
s.error = true;
s.msg = "Internal server error. Try again later";
Expand Down
84 changes: 84 additions & 0 deletions app/src/main/java/gateway/services/ServiceMetadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,90 @@ public static class ResSaveFile extends ResStatus
s.msg = resBody.getString ("message");
}
} catch (Exception e) {
System.err.println (e);
s.code = 500;
s.error = true;
s.msg = "Internal server error. Try again later";
}

return s;
}

public static ResStatus canRead (UUID userUUID, UUID fileUUID)
{
ResStatus s = new ResStatus ();

try {
HttpResponse<String> res = HttpClient.newHttpClient ().send (
HttpRequest.newBuilder ()
.uri (URI.create (String.format (
"%s/files/can_read/%s/%s", Config.getMetadataBaseUrl (),
userUUID.toString (), fileUUID.toString ())))
.GET ()
.build (),
HttpResponse.BodyHandlers.ofString ());

// response
s.code = res.statusCode ();
s.error = true;

if (s.code == 204) {
s.error = false;
} else {
JSONObject resBody = new JSONObject (res.body ());
s.msg = resBody.getString ("message");
}
} catch (Exception e) {
System.err.println (e);
s.code = 500;
s.error = true;
s.msg = "Internal server error. Try again later";
}

return s;
}

public static class ResFileMetadata extends ResStatus
{
public String name;
public String extension;
public int volume;
public long size;
public boolean isShared;
}

public static ResFileMetadata getFileMetadata (UUID fileUUID)
{
ResFileMetadata s = new ResFileMetadata ();

try {
HttpResponse<String> res = HttpClient.newHttpClient ().send (
HttpRequest.newBuilder ()
.uri (URI.create (String.format (
"%s/files/metadata/%s", Config.getMetadataBaseUrl (),
fileUUID.toString ())))
.GET ()
.build (),
HttpResponse.BodyHandlers.ofString ());

// response
JSONObject resBody = new JSONObject (res.body ());
s.code = res.statusCode ();
s.error = true;

if (s.code == 200) {
s.error = false;
s.name = resBody.getString ("name");
s.extension = resBody.isNull ("extension") ? null : resBody.getString ("extension");
s.volume = resBody.getInt ("volume");
s.size = resBody.getLong ("size");
s.isShared =
resBody.getBoolean ("is_shared"); // TODO make a test for true and false
} else {
s.msg = resBody.getString ("message");
}
} catch (Exception e) {
System.err.println (e);
s.code = 500;
s.error = true;
s.msg = "Internal server error. Try again later";
Expand Down
46 changes: 39 additions & 7 deletions app/src/main/java/gateway/services/ServiceWorker.java
Original file line number Diff line number Diff line change
@@ -1,23 +1,55 @@
package gateway.services;

import capyfile.rmi.DownloadFileArgs;
import capyfile.rmi.DownloadFileRes;
import capyfile.rmi.IWorkerService;
import com.healthmarketscience.rmiio.RemoteInputStreamClient;
import gateway.config.Config;
import java.io.IOException;
import java.io.InputStream;
import java.rmi.registry.LocateRegistry;
import java.rmi.registry.Registry;
import java.util.UUID;

public class ServiceWorker
{
// TODO: Use worker pool
public static IWorkerService getServer () throws Exception
{
Registry registry =
LocateRegistry.getRegistry (Config.getWorkerHost (), Config.getWorkerPort ());
IWorkerService server = (IWorkerService)registry.lookup ("WorkerService");

return server;
}

public static byte[] downloadFile (IWorkerService worker, UUID fileUUID, int volume)
throws Exception
{
InputStream istream = null;

try {
Registry registry =
LocateRegistry.getRegistry (Config.getWorkerHost (), Config.getWorkerPort ());
IWorkerService server = (IWorkerService)registry.lookup ("WorkerService");
// get data stream

DownloadFileRes res = worker.downloadFile (new DownloadFileArgs (fileUUID, volume));
istream = RemoteInputStreamClient.wrap (res.stream);
byte[] bytes = new byte[(int)res.size];

// copy data from stream by chunks

byte[] buf = new byte[102400]; // 100KB
int bytesPos = 0;
int bytesRead = 0;

while ((bytesRead = istream.read (buf)) >= 0) {
System.arraycopy (buf, 0, bytes, bytesPos, bytesRead);
bytesPos += bytesRead;
}

return server;
} catch (Exception e) {
throw e;
return bytes;
} finally {
if (istream != null) {
istream.close ();
}
}
}
}
Loading

0 comments on commit 0d2eaa1

Please sign in to comment.