Merge pull request #147 from PacoVK/feature/localstorage

implement localstorage

README.md

@@ -51,11 +51,12 @@ You can easily run an instance on your own with the full flexibility and power a
   * See module dependencies, inputs, outputs and resources that will be generated
   * Tapir integrates [terraform-docs](https://terraform-docs.io/) for that purpose
 * It provides several storage adapters
-  * currently S3 and AzureBlob
+  * currently S3, AzureBlob and Local
 * It provides several database adapters for the data
   * currently Dynamodb (default), Elasticsearch, CosmosDb
 * It provides a REST-API for custom integrations and further automation
   Tapir is build on [Quarkus](https://quarkus.io/) and [ReactJS](https://reactjs.org/). You can run Tapir wherever you can run Docker images.
+* If you run Tapir with local storage, it can even be operated in an **air-gaped** environment, with no internet access 
 
 Apart from the above, [this is what Wikipedia knows about Tapirs](https://en.wikipedia.org/wiki/Tapir).
 
@@ -68,6 +69,14 @@ There are samples with Terraform in `examples/`.
 
 ### Configure
 
+#### Storage
+
+Available storage backends are:
+* AWS S3
+* Azure Blob
+* Local filestorage (local)
+  * You can mount a volume into the container under ``/tapir`` to persist your data. This is highly recommended. Otherwise, you loose the data if the container gets removed. 
+
 You can configure Tapir passing the following environment variables:
 
 | Variable                         | Description                                                                                                                               | Required                                | Default      |
@@ -82,6 +91,8 @@ You can configure Tapir passing the following environment variables:
 | AZURE_BLOB_CONTAINER_NAME        | Blob container name to be used to store module archives                                                                                   | Yes, if STORAGE_CONFIG is azureBlob     | tf-registry  |
 | S3_STORAGE_BUCKET_NAME           | S3 bucket name to be used to store module archives                                                                                        | Yes, if STORAGE_CONFIG is s3            | tf-registry  |
 | S3_STORAGE_BUCKET_REGION         | AWS region of the target S3 bucket                                                                                                        | Yes, if STORAGE_CONFIG is s3            | eu-central-1 |
+| REGISTRY_HOSTNAME                | The hostname of the registry, must be set to the DNS record of Tapir                                                                      | Yes, if STORAGE_CONFIG is local         | localhost    |
+| REGISTRY_PORT                    | The port of the registry                                                                                                                  | Yes, if STORAGE_CONFIG is local         | 443          |
 | API_MAX_BODY_SIZE                | The maximum payload size for module/providers to be uploaded                                                                              | X                                       | 100M         |
 | REGISTRY_GPG_KEYS_0__ID          | GPG key ID of the key to be used (eg. D17C807B4156558133A1FB843C7461473EB779BD)                                                           | X                                       |              |
 | REGISTRY_GPG_KEYS_0__ASCII_ARMOR | Ascii armored and bas64 encoded GPG public key (only RSA/DSA supported)                                                                   | X                                       |              |

pom.xml

@@ -4,15 +4,15 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>de.euhus</groupId>
   <artifactId>tapir</artifactId>
-  <version>0.2.2</version>
+  <version>0.3.0</version>
   <properties>
     <compiler-plugin.version>3.11.0</compiler-plugin.version>
     <maven.compiler.release>17</maven.compiler.release>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
     <quarkus.platform.group-id>io.quarkus.platform</quarkus.platform.group-id>
-    <quarkus.platform.version>2.16.7.Final</quarkus.platform.version>
+    <quarkus.platform.version>3.0.3.Final</quarkus.platform.version>
     <quarkus.quinoa.version>2.0.3</quarkus.quinoa.version>
     <microsoft.azure.bom.version>1.2.12</microsoft.azure.bom.version>
     <skipITs>true</skipITs>

src/main/java/api/Discovery.java

@@ -1,11 +1,11 @@
 package api;
 
 import core.terraform.ServiceDiscovery;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 
 @Produces(MediaType.APPLICATION_JSON)
 @Path("/.well-known")

src/main/java/api/Modules.java

@@ -9,16 +9,16 @@
 import core.upload.service.UploadService;
 import io.vertx.core.json.JsonObject;
 import io.vertx.mutiny.core.eventbus.EventBus;
+import jakarta.enterprise.inject.Instance;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 import java.time.Instant;
 import java.util.List;
-import javax.enterprise.inject.Instance;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
 
 @Produces(MediaType.APPLICATION_JSON)
 @Path("/terraform/modules/v1")

src/main/java/api/Providers.java

@@ -14,17 +14,17 @@
 import core.upload.service.UploadService;
 import io.vertx.core.json.JsonObject;
 import io.vertx.mutiny.core.eventbus.EventBus;
+import jakarta.enterprise.inject.Instance;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 import java.time.Instant;
 import java.util.List;
 import java.util.NoSuchElementException;
-import javax.enterprise.inject.Instance;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
 
 @Produces(MediaType.APPLICATION_JSON)
 @Path("/terraform/providers/v1/")

src/main/java/api/Reports.java

@@ -4,13 +4,13 @@
 import core.exceptions.ReportNotFoundException;
 import core.terraform.Module;
 import extensions.core.Report;
+import jakarta.enterprise.inject.Instance;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 import java.io.IOException;
-import javax.enterprise.inject.Instance;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
 
 @Produces(MediaType.APPLICATION_JSON)
 @Path("/reports")

src/main/java/api/Search.java

@@ -2,14 +2,14 @@
 
 import api.dto.PaginationDto;
 import core.backend.SearchService;
-import javax.enterprise.inject.Instance;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
+import jakarta.enterprise.inject.Instance;
+import jakarta.ws.rs.DefaultValue;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 
 @Produces(MediaType.APPLICATION_JSON)
 @Path("/search")

src/main/java/api/Tapir.java

@@ -0,0 +1,42 @@
+package api;
+
+import core.exceptions.ModuleNotFoundException;
+import core.storage.local.LocalStorageService;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Response;
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.logging.Logger;
+
+@Path("/tapir")
+public class Tapir {
+
+  static final Logger LOGGER = Logger.getLogger(Tapir.class.getName());
+
+  @GET
+  @Path("/storage/{namespace}/{name}/{identifier}/{filename}")
+  @Produces("application/zip")
+  public Response download(String namespace, String name, String identifier, String filename)
+          throws ModuleNotFoundException {
+    String path = Paths.get(namespace, name, identifier, filename).toString();
+    if (identifier.matches(".*\\d.*")) {
+      LOGGER.fine("Identifier is a version string, assume user requested a provider " + identifier);
+      LOGGER.info("Requested the download of provider " + path);
+      path = LocalStorageService.PROVIDER_RESOURCE_DIR + path;
+    } else {
+      LOGGER.info("Requested the download of module " + path);
+      path = LocalStorageService.MODULE_RESOURCE_DIR + path;
+    }
+    File moduleArchive = new File(path);
+
+    if (!moduleArchive.exists()) {
+      throw new ModuleNotFoundException(path);
+    }
+
+    return Response
+            .ok(moduleArchive)
+            .build();
+  }
+}

src/main/java/api/dto/mapper/ProviderMapper.java

@@ -8,10 +8,10 @@
 import core.storage.util.StorageUtil;
 import core.terraform.Provider;
 import core.terraform.ProviderPlatform;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Instance;
 import java.util.List;
 import java.util.Map;
-import javax.enterprise.context.ApplicationScoped;
-import javax.enterprise.inject.Instance;
 
 @ApplicationScoped
 public class ProviderMapper {

src/main/java/api/dto/mapper/ProviderVersionMapper.java

@@ -2,9 +2,9 @@
 
 import api.dto.ProviderVersionsDto;
 import core.terraform.Provider;
+import jakarta.enterprise.context.ApplicationScoped;
 import java.util.LinkedList;
 import java.util.List;
-import javax.enterprise.context.ApplicationScoped;
 
 @ApplicationScoped
 public class ProviderVersionMapper {

src/main/java/api/mapper/exceptions/ConstraintViolationMapper.java

@@ -1,15 +1,15 @@
 package api.mapper.exceptions;
 
 import api.mapper.exceptions.response.ErrorResponse;
+import jakarta.validation.ConstraintViolationException;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.ext.ExceptionMapper;
+import jakarta.ws.rs.ext.Provider;
 import java.util.List;
 import java.util.UUID;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
-import javax.validation.ConstraintViolationException;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.ExceptionMapper;
-import javax.ws.rs.ext.Provider;
 
 @Provider
 public class ConstraintViolationMapper implements ExceptionMapper<ConstraintViolationException> {

src/main/java/api/mapper/exceptions/TapirExceptionMapper.java

@@ -3,12 +3,12 @@
 import api.mapper.exceptions.response.ErrorResponse;
 import core.exceptions.NotFoundException;
 import core.exceptions.TapirException;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.ext.ExceptionMapper;
+import jakarta.ws.rs.ext.Provider;
 import java.util.UUID;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.ExceptionMapper;
-import javax.ws.rs.ext.Provider;
 
 @Provider
 public class TapirExceptionMapper implements ExceptionMapper<TapirException> {

src/main/java/api/mapper/exceptions/ThrowableMapper.java

@@ -1,13 +1,13 @@
 package api.mapper.exceptions;
 
 import api.mapper.exceptions.response.ErrorResponse;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.ext.ExceptionMapper;
+import jakarta.ws.rs.ext.Provider;
 import java.util.ResourceBundle;
 import java.util.UUID;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.ExceptionMapper;
-import javax.ws.rs.ext.Provider;
 
 @Provider
 public class ThrowableMapper implements ExceptionMapper<Throwable> {

src/main/java/core/Bootstrap.java

@@ -3,10 +3,10 @@
 import core.backend.SearchService;
 import core.config.SigningKeys;
 import core.exceptions.InvalidConfigurationException;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Singleton;
 import java.util.Base64;
 import java.util.logging.Logger;
-import javax.enterprise.inject.Instance;
-import javax.inject.Singleton;
 import org.eclipse.microprofile.config.ConfigProvider;
 
 @Singleton

src/main/java/core/backend/aws/dynamodb/repository/DynamodbRepository.java

@@ -9,10 +9,10 @@
 import core.terraform.Provider;
 import extensions.core.Report;
 import io.quarkus.arc.lookup.LookupIfProperty;
+import jakarta.enterprise.context.ApplicationScoped;
 import java.util.Collections;
 import java.util.Map;
 import java.util.logging.Logger;
-import javax.enterprise.context.ApplicationScoped;
 import software.amazon.awssdk.core.internal.waiters.ResponseOrException;
 import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient;
 import software.amazon.awssdk.enhanced.dynamodb.DynamoDbTable;

src/main/java/core/backend/azure/cosmosdb/CosmosDbRepository.java

@@ -22,10 +22,10 @@
 import core.terraform.Provider;
 import extensions.core.Report;
 import io.quarkus.arc.lookup.LookupIfProperty;
+import jakarta.annotation.PostConstruct;
+import jakarta.enterprise.context.ApplicationScoped;
 import java.util.Collections;
 import java.util.List;
-import javax.annotation.PostConstruct;
-import javax.enterprise.context.ApplicationScoped;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 
 @LookupIfProperty(name = "registry.search.backend", stringValue = "cosmosdb")

src/main/java/core/backend/elasticsearch/ElasticSearchRepository.java

@@ -12,12 +12,12 @@
 import io.quarkus.arc.lookup.LookupIfProperty;
 import io.vertx.core.json.JsonArray;
 import io.vertx.core.json.JsonObject;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.ws.rs.HttpMethod;
 import java.io.IOException;
 import java.util.List;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
-import javax.enterprise.context.ApplicationScoped;
-import javax.ws.rs.HttpMethod;
 import org.apache.http.HttpEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;

src/main/java/core/storage/aws/S3StorageService.java

@@ -7,12 +7,12 @@
 import core.terraform.Provider;
 import core.upload.FormData;
 import io.quarkus.arc.lookup.LookupIfProperty;
+import jakarta.enterprise.context.ApplicationScoped;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.logging.Logger;
 import java.util.stream.Stream;
-import javax.enterprise.context.ApplicationScoped;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.services.s3.S3Client;

src/main/java/core/storage/azure/BlobStorageService.java

@@ -13,12 +13,12 @@
 import core.terraform.Provider;
 import core.upload.FormData;
 import io.quarkus.arc.lookup.LookupIfProperty;
+import jakarta.annotation.PostConstruct;
+import jakarta.enterprise.context.ApplicationScoped;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.util.stream.Stream;
-import javax.annotation.PostConstruct;
-import javax.enterprise.context.ApplicationScoped;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 
 @LookupIfProperty(name = "registry.storage.backend", stringValue = "azureBlob")