Merge pull request #3832 from gchq/gh-3026-docker-managed-vols

PR for #3026 - Proxy does not have a docker managed vol for the sqlite DB

stroom-app/docker/Dockerfile

@@ -65,6 +65,8 @@ VOLUME /stroom/config/
 # Create Docker volume for the content_pack_import dir as this will be mutated on 
 VOLUME /stroom/content_pack_import/
 
+# lmdb_library mouted as tmpfs not a managed volume
+
 # Create Docker volume for SLF4J output
 VOLUME /stroom/logs/
 
@@ -80,9 +82,6 @@ VOLUME /stroom/reference_data/
 # Create Docker volume for Stroom's off-heap search result data
 VOLUME /stroom/search_results/
 
-# Create Docker volume for Stroom's LMDB system library binary
-VOLUME /stroom/lmdb_library/
-
 # Create Docker volume for Stroom's volumes dir
 VOLUME /stroom/volumes/
 
@@ -99,6 +98,9 @@ CMD ["./start.sh", "server"]
 # see https://github.com/docker-library/openjdk/issues/73#issuecomment-207816707
 
 # Create a user with no home and no shell
+# IMPORTANT!!!
+# If you add any new dirs here, make sure you also chown them in docker-entrypoint.sh
+# IMPORTANT!!!
 RUN \
     apk add --no-cache \
         ttf-dejavu && \
@@ -120,7 +122,9 @@ RUN \
     mkdir -p /stroom/volumes && \
     chown -R stroom:stroom /stroom
 
-# IMPORTANT - if you add/change a dir above you need to do the same in entrypoint.sh
+# IMPORTANT!!!
+# If you add any new dirs above, make sure you also chown them in docker-entrypoint.sh
+# IMPORTANT!!!
 
 # Those that are most likely to have changed get copied last to speed up build
 

stroom-app/src/main/resources/ui/noauth/swagger/stroom.json

@@ -8957,6 +8957,36 @@
         "tags" : [ "Authorisation" ]
       }
     },
+    "/userNames/v1/findAssociates" : {
+      "post" : {
+        "operationId" : "findAssociateUserNames",
+        "requestBody" : {
+          "content" : {
+            "application/json" : {
+              "schema" : {
+                "$ref" : "#/components/schemas/FindUserNameCriteria"
+              }
+            }
+          },
+          "description" : "criteria",
+          "required" : true
+        },
+        "responses" : {
+          "default" : {
+            "content" : {
+              "application/json" : {
+                "schema" : {
+                  "$ref" : "#/components/schemas/ResultPageUserName"
+                }
+              }
+            },
+            "description" : "default response"
+          }
+        },
+        "summary" : "Find the user names matching the supplied criteria of users who belong to at least one of the same groups as the current user. If the current user is admin or has Manage Users permission then they can see all users.",
+        "tags" : [ "Authorisation" ]
+      }
+    },
     "/userNames/v1/getByDisplayName/{displayName}" : {
       "get" : {
         "operationId" : "getByDisplayName",

stroom-app/src/main/resources/ui/noauth/swagger/stroom.yaml

@@ -6150,6 +6150,28 @@ paths:
       summary: Find the user names matching the supplied criteria
       tags:
       - Authorisation
+  /userNames/v1/findAssociates:
+    post:
+      operationId: findAssociateUserNames
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/FindUserNameCriteria'
+        description: criteria
+        required: true
+      responses:
+        default:
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ResultPageUserName'
+          description: default response
+      summary: Find the user names matching the supplied criteria of users who belong
+        to at least one of the same groups as the current user. If the current user
+        is admin or has Manage Users permission then they can see all users.
+      tags:
+      - Authorisation
   /userNames/v1/getByDisplayName/{displayName}:
     get:
       operationId: getByDisplayName

stroom-proxy/stroom-proxy-app/docker/Dockerfile

@@ -58,6 +58,9 @@ VOLUME /stroom-proxy/certs/
 VOLUME /stroom-proxy/content/
 VOLUME /stroom-proxy/repo/
 VOLUME /stroom-proxy/logs/
+VOLUME /stroom-proxy/db/
+VOLUME /stroom-proxy/failures/
+# sqlite_library mounted as tmpfs not a managed volumne
 
 # run entrypoint script inside tini for better unix process handling, 
 # see https://github.com/krallin/tini/issues/8
@@ -72,7 +75,10 @@ CMD ["./start.sh"]
 #RUN echo "151.101.112.249 dl-cdn.alpinelinux.org" >> /etc/hosts && \
 
 # Create a system user/group with no home and no shell and a uid:gid of 1000:1000
+
+# IMPORTANT!!!
 # If you add any new dirs here, make sure you also chown them in docker-entrypoint.sh
+# IMPORTANT!!!
 RUN \
     addgroup -g 1000 -S proxy && \
     adduser -u 1000 -S -s /bin/false -D -G proxy proxy && \
@@ -86,6 +92,9 @@ RUN \
     mkdir -p /stroom-proxy/logs/send && \
     mkdir -p /stroom-proxy/logs/app && \
     mkdir -p /stroom-proxy/repo && \
+    mkdir -p /stroom-proxy/db && \
+    mkdir -p /stroom-proxy/failures && \
+    mkdir -p /stroom-proxy/sqlite_library && \
     chown -R proxy:proxy /stroom-proxy
 
 # Copy all the fat jars for the application and connectors

stroom-proxy/stroom-proxy-app/docker/docker-entrypoint.sh

@@ -15,6 +15,9 @@ if [ "$(id -u)" = '0' ]; then
     chown proxy:proxy /stroom-proxy/logs
     chown proxy:proxy /stroom-proxy/logs/extra_headers.txt
     chown proxy:proxy /stroom-proxy/repo
+    chown proxy:proxy /stroom-proxy/db
+    chown proxy:proxy /stroom-proxy/failures
+    chown proxy:proxy /stroom-proxy/sqlite_library
 
     # This is a bit of a cludge to get round "Text file in use" errors
     # See: https://github.com/moby/moby/issues/9547

stroom-proxy/stroom-proxy-app/src/test/resources/stroom/dist/proxy-expected.yaml

@@ -13,7 +13,7 @@ proxyConfig:
     syncFrequency: "PT1M"
     upstreamUrl: null
   db:
-    batchSize: 10000000
+    batchSize: 1000000
     cleanupFrequency: "PT1S"
     connectionPragma:
     - "pragma synchronous = normal;"
@@ -23,6 +23,7 @@ proxyConfig:
     flushFrequency: "PT1S"
     globalPragma:
     - "pragma journal_mode = WAL;"
+    libraryDir: "sqlite_library"
     maintenancePragma:
     - "pragma wal_checkpoint(truncate);"
     - "pragma vacuum;"

stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/ProxyDbConfig.java

@@ -35,12 +35,12 @@ public class ProxyDbConfig extends AbstractConfig implements IsProxyConfig {
 //    "pragma incremental_vacuum;""
 
     private static final StroomDuration DEFAULT_MAINTENANCE_PRAGMA_FREQUENCY = StroomDuration.ofMinutes(1);
-    // TODO: 08/11/2022 Is this meant to be 1mil or 10mil
-    private static final int DEFAULT_BATCH_SIZE = 1_0000_000;
+    private static final int DEFAULT_BATCH_SIZE = 1_000_000;
     protected static final StroomDuration DEFAULT_FLUSH_FREQUENCY = StroomDuration.ofSeconds(1);
     protected static final StroomDuration DEFAULT_CLEANUP_FREQUENCY = StroomDuration.ofSeconds(1);
 
     private final String dbDir;
+    private final String libraryDir;
     private final List<String> globalPragma;
     private final List<String> connectionPragma;
     private final List<String> maintenancePragma;
@@ -52,6 +52,7 @@ public class ProxyDbConfig extends AbstractConfig implements IsProxyConfig {
 
     public ProxyDbConfig() {
         dbDir = "db";
+        libraryDir = "sqlite_library";
         globalPragma = DEFAULT_GLOBAL_PRAGMA;
         connectionPragma = DEFAULT_CONNECTION_PRAGMA;
         maintenancePragma = DEFAULT_MAINTENANCE_PRAGMA;
@@ -64,6 +65,7 @@ public ProxyDbConfig() {
     @JsonCreator
     public ProxyDbConfig(
             @JsonProperty("dbDir") final String dbDir,
+            @JsonProperty("libraryDir") final String libraryDir,
             @JsonProperty("globalPragma") final List<String> globalPragma,
             @JsonProperty("connectionPragma") final List<String> connectionPragma,
             @JsonProperty("maintenancePragma") final List<String> maintenancePragma,
@@ -72,6 +74,7 @@ public ProxyDbConfig(
             @JsonProperty("flushFrequency") final StroomDuration flushFrequency,
             @JsonProperty("cleanupFrequency") final StroomDuration cleanupFrequency) {
         this.dbDir = dbDir;
+        this.libraryDir = libraryDir;
         this.globalPragma = List.copyOf(globalPragma);
         this.connectionPragma = List.copyOf(connectionPragma);
         this.maintenancePragma = maintenancePragma;
@@ -88,6 +91,13 @@ public String getDbDir() {
         return dbDir;
     }
 
+    @RequiresProxyRestart
+    @JsonProperty
+    @JsonPropertyDescription("The directory to use for the proxy repository DB Sqlite library files")
+    public String getLibraryDir() {
+        return libraryDir;
+    }
+
     @RequiresProxyRestart
     @JsonProperty
     @JsonPropertyDescription("A list of statements to run on the database on startup")
@@ -146,18 +156,22 @@ public boolean equals(final Object o) {
             return false;
         }
         final ProxyDbConfig that = (ProxyDbConfig) o;
-        return batchSize == that.batchSize && Objects.equals(dbDir, that.dbDir) && Objects.equals(
-                globalPragma,
-                that.globalPragma) && Objects.equals(connectionPragma,
-                that.connectionPragma) && Objects.equals(maintenancePragma,
-                that.maintenancePragma) && Objects.equals(maintenancePragmaFrequency,
-                that.maintenancePragmaFrequency) && Objects.equals(flushFrequency,
-                that.flushFrequency) && Objects.equals(cleanupFrequency, that.cleanupFrequency);
+        return batchSize == that.batchSize
+                && Objects.equals(dbDir, that.dbDir)
+                && Objects.equals(libraryDir, that.libraryDir)
+                && Objects.equals(globalPragma, that.globalPragma)
+                && Objects.equals(connectionPragma, that.connectionPragma)
+                && Objects.equals(maintenancePragma, that.maintenancePragma)
+                && Objects.equals(maintenancePragmaFrequency, that.maintenancePragmaFrequency)
+                && Objects.equals(flushFrequency, that.flushFrequency)
+                && Objects.equals(cleanupFrequency, that.cleanupFrequency);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(dbDir,
+        return Objects.hash(
+                dbDir,
+                libraryDir,
                 globalPragma,
                 connectionPragma,
                 maintenancePragma,
@@ -171,6 +185,7 @@ public int hashCode() {
     public String toString() {
         return "ProxyDbConfig{" +
                 "dbDir='" + dbDir + '\'' +
+                ", libraryDir=" + libraryDir +
                 ", globalPragma=" + globalPragma +
                 ", connectionPragma=" + connectionPragma +
                 ", maintenancePragma=" + maintenancePragma +

stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/ProxyDbModule.java

@@ -11,8 +11,10 @@
 import stroom.proxy.repo.dao.ForwardSourceDao;
 import stroom.proxy.repo.dao.SourceDao;
 import stroom.proxy.repo.dao.SourceItemDao;
+import stroom.util.NullSafe;
 import stroom.util.guice.GuiceUtil;
 import stroom.util.io.FileUtil;
+import stroom.util.io.PathCreator;
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
 import stroom.util.shared.Flushable;
@@ -35,6 +37,7 @@ public class ProxyDbModule extends AbstractModule {
     private static final String FLYWAY_TABLE = "proxy_repo_schema_history";
 
     private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(AbstractDataSourceProviderModule.class);
+    public static final String SQLITE_TEMPDIR_PROP = "org.sqlite.tmpdir";
 
     @Override
     protected void configure() {
@@ -52,23 +55,37 @@ protected void configure() {
     public ProxyRepoDbConnProvider getConnectionProvider(
             final RepoDbDirProvider repoDbDirProvider,
             final DataSourceFactory dataSourceFactory,
-            final ProxyDbConfig proxyDbConfig) {
+            final ProxyDbConfig proxyDbConfig,
+            final PathCreator pathCreator) {
         LOGGER.debug(() -> "Getting connection provider for " + MODULE);
 
-        final AbstractDbConfig config = getDbConfig(repoDbDirProvider);
+        final AbstractDbConfig config = getDbConfig(repoDbDirProvider, proxyDbConfig, pathCreator);
         final DataSource dataSource = dataSourceFactory.create(config, MODULE, true);
         FlywayUtil.migrate(dataSource, FLYWAY_LOCATIONS, FLYWAY_TABLE, MODULE);
         return new DataSourceImpl(dataSource, proxyDbConfig);
     }
 
-    private AbstractDbConfig getDbConfig(final RepoDbDirProvider repoDbDirProvider) {
+    private AbstractDbConfig getDbConfig(final RepoDbDirProvider repoDbDirProvider,
+                                         final ProxyDbConfig proxyDbConfig,
+                                         final PathCreator pathCreator) {
         final Path dbDir = repoDbDirProvider.get();
 
         FileUtil.mkdirs(dbDir);
         if (!Files.isDirectory(dbDir)) {
             throw new RuntimeException("Unable to find DB dir: " + FileUtil.getCanonicalPath(dbDir));
         }
 
+        final String libraryDirStr = proxyDbConfig.getLibraryDir();
+        if (!NullSafe.isBlankString(libraryDirStr)) {
+            final Path libraryDir = pathCreator.toAppPath(libraryDirStr);
+            LOGGER.info("Setting {} to '{}'", SQLITE_TEMPDIR_PROP, libraryDir.toString());
+            System.setProperty(SQLITE_TEMPDIR_PROP, libraryDir.toString());
+            if (!Files.exists(libraryDir)) {
+                LOGGER.info("Ensuring '{}' exists", libraryDir);
+                FileUtil.ensureDirExists(libraryDir);
+            }
+        }
+
         final Path path = dbDir.resolve("proxy-repo.db");
         final String fullPath = FileUtil.getCanonicalPath(path);
 
@@ -83,13 +100,21 @@ private AbstractDbConfig getDbConfig(final RepoDbDirProvider repoDbDirProvider)
         return new MyProxyRepoDbConfig(connectionConfig);
     }
 
+
+    // --------------------------------------------------------------------------------
+
+
     private static class MyProxyRepoDbConfig extends AbstractDbConfig {
 
         public MyProxyRepoDbConfig(final ConnectionConfig connectionConfig) {
             super(connectionConfig, new ConnectionPoolConfig());
         }
     }
 
+
+    // --------------------------------------------------------------------------------
+
+
     public static class DataSourceImpl extends DataSourceProxy implements ProxyRepoDbConnProvider {
 
         private final ProxyDbConfig proxyDbConfig;

stroom-proxy/stroom-proxy-repo/src/test/java/stroom/proxy/repo/ProxyRepoTestModule.java

@@ -1,7 +1,7 @@
 package stroom.proxy.repo;
 
 import stroom.db.util.DbModule;
-import stroom.util.io.FileUtil;
+import stroom.test.common.util.test.TestingHomeAndTempProvidersModule;
 
 import com.google.inject.AbstractModule;
 
@@ -14,8 +14,13 @@ protected void configure() {
         install(new ProxyDbModule());
         install(new DbModule());
 
-        final Path repoDir = FileUtil.createTempDirectory("stroom-proxy-repo");
-        final Path dbDir = FileUtil.createTempDirectory("stroom-proxy-db");
+        final TestingHomeAndTempProvidersModule homeAndTempProvidersModule = new TestingHomeAndTempProvidersModule();
+        install(homeAndTempProvidersModule);
+
+        final Path homeDir = homeAndTempProvidersModule.getHomeDir();
+        final Path repoDir = homeDir.resolve("repo");
+        final Path dbDir = homeDir.resolve("db");
+
         bind(RepoDirProvider.class).toInstance(() -> repoDir);
         bind(RepoDbDirProvider.class).toInstance(() -> dbDir);
         bind(ErrorReceiver.class).to(ErrorReceiverImpl.class);

stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/AuthenticationService.java

@@ -129,9 +129,7 @@ public Optional<User> getUser(final String subjectId) {
 
         try {
             optUser = userDao.getBySubjectId(subjectId, false);
-            if (optUser.isEmpty()
-                    && User.ADMIN_SUBJECT_ID.equals(subjectId)
-                    && IdpType.INTERNAL_IDP.equals(openIdConfigProvider.get().getIdentityProviderType())) {
+            if (optUser.isEmpty() && shouldCreateAdminUser(subjectId)) {
 
                 // TODO @AT Probably should be an explicit command to create this to avoid the accidental
                 //   running of stroom in UseInternal mode which then leaves admin/admin open
@@ -147,6 +145,13 @@ public Optional<User> getUser(final String subjectId) {
         return optUser;
     }
 
+    private boolean shouldCreateAdminUser(final String subjectId) {
+        return User.ADMIN_SUBJECT_ID.equals(subjectId)
+                && (
+                IdpType.INTERNAL_IDP.equals(openIdConfigProvider.get().getIdentityProviderType())
+                        || IdpType.TEST_CREDENTIALS.equals(openIdConfigProvider.get().getIdentityProviderType()));
+    }
+
     public User createOrRefreshUser(final String subjectId) {
         return createOrRefreshUserOrGroup(subjectId, false);
     }
@@ -167,8 +172,7 @@ private User createOrRefreshUserOrGroup(final String subjectId, final boolean is
                     final User userRef = create(subjectId, isGroup);
 
                     // Creating the admin user so create its group too
-                    if (User.ADMIN_SUBJECT_ID.equals(subjectId)
-                            && IdpType.INTERNAL_IDP.equals(openIdConfigProvider.get().getIdentityProviderType())) {
+                    if (shouldCreateAdminUser(subjectId)) {
                         try {
                             User userGroup = createOrRefreshAdminUserGroup(ADMINISTRATORS_GROUP_SUBJECT_ID);
                             userDao.addUserToGroup(userRef.getUuid(), userGroup.getUuid());