From 3add291e567dcb7eb08817a2e38374a3a8f7608d Mon Sep 17 00:00:00 2001 From: Julien Ruaux Date: Wed, 20 Nov 2019 20:10:30 -0800 Subject: [PATCH] refactored redis writers --- README.adoc | 1 - build.gradle | 24 +- pom.xml | 298 ------------------ .../riot/ManifestVersionProvider.java | 42 --- .../com/redislabs/riot/OneLineLogFormat.java | 47 --- src/main/java/com/redislabs/riot/Riot.java | 105 +----- .../file/AbstractResourceItemWriter.java | 2 - .../batch/file}/CountingOutputStream.java | 2 +- .../redis/AbstractLettuceItemWriter.java | 43 +++ .../batch/redis/AbstractRedisItemWriter.java | 46 +-- .../riot/batch/redis/AbstractRedisWriter.java | 14 + .../batch/redis/JedisClusterItemWriter.java | 41 --- .../riot/batch/redis/JedisClusterWriter.java | 35 ++ ...disItemReader.java => JedisMapReader.java} | 6 +- ...emWriter.java => JedisPipelineWriter.java} | 23 +- .../batch/redis/LettuceAsyncItemWriter.java | 51 +++ .../riot/batch/redis/LettuceConnector.java | 22 ++ .../riot/batch/redis/LettuceItemWriter.java | 84 ----- .../redis/LettuceReactiveItemWriter.java | 59 ++++ .../batch/redis/LettuceSyncItemWriter.java | 35 ++ .../riot/batch/redis/RedisWriter.java | 7 + .../batch/redis/map/AbstractMapWriter.java | 74 +++++ .../batch/redis/map/CollectionMapWriter.java | 20 ++ .../riot/batch/redis/map/DebugMapWriter.java | 16 + .../batch/redis/map/EvalshaMapWriter.java | 37 +++ .../riot/batch/redis/map/ExpireMapWriter.java | 22 ++ .../riot/batch/redis/map/GeoaddMapWriter.java | 39 +++ .../riot/batch/redis/map/HmsetMapWriter.java | 13 + .../batch/redis/map/JedisClusterCommands.java | 98 ++++++ .../riot/batch/redis/map/JedisCommands.java | 97 ++++++ .../redis/map/JedisPipelineCommands.java | 97 ++++++ .../batch/redis/map/LettuceAsyncCommands.java | 104 ++++++ .../redis/map/LettuceReactiveCommands.java | 105 ++++++ .../batch/redis/map/LettuceSyncCommands.java | 108 +++++++ .../riot/batch/redis/map/LpushMapWriter.java | 12 + .../riot/batch/redis/map/NoopMapWriter.java | 12 + .../riot/batch/redis/map/RedisCommands.java | 44 +++ .../riot/batch/redis/map/RpushMapWriter.java | 12 + .../riot/batch/redis/map/SaddMapWriter.java | 12 + .../batch/redis/map/SetFieldMapWriter.java | 19 ++ .../riot/batch/redis/map/SetMapWriter.java | 18 ++ .../{writer => map}/SetObjectMapWriter.java | 12 +- .../riot/batch/redis/map/XaddIdMapWriter.java | 21 ++ .../redis/map/XaddIdMaxlenMapWriter.java | 25 ++ .../riot/batch/redis/map/XaddMapWriter.java | 13 + .../batch/redis/map/XaddMaxlenMapWriter.java | 22 ++ .../riot/batch/redis/map/ZaddMapWriter.java | 22 ++ .../redis/writer/AbstractFlatMapWriter.java | 41 --- .../writer/AbstractRedisFlatMapWriter.java | 38 --- .../redis/writer/CollectionMapWriter.java | 43 --- .../batch/redis/writer/DebugMapWriter.java | 35 -- .../batch/redis/writer/EvalshaMapWriter.java | 59 ---- .../batch/redis/writer/ExpireMapWriter.java | 44 --- .../batch/redis/writer/GeoaddMapWriter.java | 73 ----- .../batch/redis/writer/HmsetMapWriter.java | 31 -- .../batch/redis/writer/LpushMapWriter.java | 29 -- .../batch/redis/writer/NoopMapWriter.java | 26 -- .../batch/redis/writer/RedisMapWriter.java | 18 -- .../batch/redis/writer/RpushMapWriter.java | 29 -- .../batch/redis/writer/SaddMapWriter.java | 29 -- .../batch/redis/writer/SetFieldMapWriter.java | 17 - .../riot/batch/redis/writer/SetMapWriter.java | 43 --- .../batch/redis/writer/XaddIdMapWriter.java | 47 --- .../redis/writer/XaddIdMaxlenMapWriter.java | 56 ---- .../batch/redis/writer/XaddMapWriter.java | 30 -- .../redis/writer/XaddMaxlenMapWriter.java | 42 --- .../batch/redis/writer/ZaddMapWriter.java | 44 --- .../writer/AbstractLettuSearchMapWriter.java | 52 +-- .../writer/AbstractSearchMapWriter.java | 27 +- .../redisearch/writer/FtaddMapWriter.java | 12 +- .../writer/FtaddPayloadMapWriter.java | 24 +- .../writer/JedisSuggestMapWriter.java | 29 +- .../redisearch/writer/SugaddMapWriter.java | 34 +- .../writer/SugaddPayloadMapWriter.java | 20 +- .../redislabs/riot/cli/AbstractCommand.java | 4 +- .../riot/cli/ConsoleExportCommand.java | 4 +- .../com/redislabs/riot/cli/ExportCommand.java | 7 +- .../redislabs/riot/cli/GeneratorCommand.java | 4 +- .../com/redislabs/riot/cli/ImportCommand.java | 7 +- .../redislabs/riot/cli/ProcessorOptions.java | 7 +- .../redislabs/riot/cli/RediSearchCommand.java | 7 - .../com/redislabs/riot/cli/RedisCommand.java | 2 +- .../redislabs/riot/cli/TransferCommand.java | 10 +- .../riot/cli/db/DatabaseExportCommand.java | 4 +- .../riot/cli/db/DatabaseImportCommand.java | 4 +- .../riot/cli/db/DatabaseWriterOptions.java | 6 +- .../riot/cli/file/FileExportCommand.java | 5 +- .../riot/cli/file/FileImportCommand.java | 4 +- .../redislabs/riot/cli/redis/Endpoint.java | 36 --- .../riot/cli/redis/ExpireCommandOptions.java | 9 +- .../riot/cli/redis/GeoCommandOptions.java | 10 +- .../cli/redis/JedisConnectionOptions.java | 14 - .../cli/redis/LettuceConnectionOptions.java | 116 ------- .../riot/cli/redis/LuaCommandOptions.java | 11 +- .../cli/redis/RediSearchCommandOptions.java | 47 ++- .../riot/cli/redis/RedisCommandOptions.java | 40 ++- .../cli/redis/RedisConnectionOptions.java | 174 ---------- .../redislabs/riot/cli/redis/RedisDriver.java | 5 - .../riot/cli/redis/RedisExportCommand.java | 5 +- .../riot/cli/redis/RedisReaderOptions.java | 6 +- .../riot/cli/redis/RedisWriterOptions.java | 98 ++++-- .../redislabs/riot/cli/redis/SslProvider.java | 5 - .../riot/cli/redis/StreamCommandOptions.java | 29 +- .../riot/cli/redis/StringCommandOptions.java | 20 +- .../riot/cli/redis/ZsetCommandOptions.java | 9 +- .../redislabs/riot/cli/test/TestCommand.java | 4 +- .../file-import-csv-processor-search-geo.txt | 2 +- .../file-import-csv-processor-search.txt | 2 +- .../commands/file-import-csv-search.txt | 2 +- 109 files changed, 1609 insertions(+), 2143 deletions(-) delete mode 100644 pom.xml delete mode 100644 src/main/java/com/redislabs/riot/ManifestVersionProvider.java delete mode 100644 src/main/java/com/redislabs/riot/OneLineLogFormat.java rename src/main/java/com/{google/common/io => redislabs/riot/batch/file}/CountingOutputStream.java (97%) create mode 100644 src/main/java/com/redislabs/riot/batch/redis/AbstractLettuceItemWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/AbstractRedisWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/JedisClusterItemWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/JedisClusterWriter.java rename src/main/java/com/redislabs/riot/batch/redis/{JedisItemReader.java => JedisMapReader.java} (89%) rename src/main/java/com/redislabs/riot/batch/redis/{JedisItemWriter.java => JedisPipelineWriter.java} (57%) create mode 100644 src/main/java/com/redislabs/riot/batch/redis/LettuceAsyncItemWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/LettuceConnector.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/LettuceItemWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/LettuceReactiveItemWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/LettuceSyncItemWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/RedisWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/AbstractMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/CollectionMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/DebugMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/EvalshaMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/ExpireMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/GeoaddMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/HmsetMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/JedisClusterCommands.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/JedisCommands.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/JedisPipelineCommands.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/LettuceAsyncCommands.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/LettuceReactiveCommands.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/LettuceSyncCommands.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/LpushMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/NoopMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/RedisCommands.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/RpushMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/SaddMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/SetFieldMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/SetMapWriter.java rename src/main/java/com/redislabs/riot/batch/redis/{writer => map}/SetObjectMapWriter.java (69%) create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMaxlenMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/XaddMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/XaddMaxlenMapWriter.java create mode 100644 src/main/java/com/redislabs/riot/batch/redis/map/ZaddMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/AbstractFlatMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/AbstractRedisFlatMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/CollectionMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/DebugMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/EvalshaMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/ExpireMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/GeoaddMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/HmsetMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/LpushMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/NoopMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/RedisMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/RpushMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/SaddMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/SetFieldMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/SetMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMaxlenMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/XaddMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/XaddMaxlenMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/batch/redis/writer/ZaddMapWriter.java delete mode 100644 src/main/java/com/redislabs/riot/cli/RediSearchCommand.java delete mode 100644 src/main/java/com/redislabs/riot/cli/redis/Endpoint.java delete mode 100644 src/main/java/com/redislabs/riot/cli/redis/JedisConnectionOptions.java delete mode 100644 src/main/java/com/redislabs/riot/cli/redis/LettuceConnectionOptions.java delete mode 100644 src/main/java/com/redislabs/riot/cli/redis/RedisConnectionOptions.java delete mode 100644 src/main/java/com/redislabs/riot/cli/redis/RedisDriver.java delete mode 100644 src/main/java/com/redislabs/riot/cli/redis/SslProvider.java diff --git a/README.adoc b/README.adoc index d3b0d086d..a537a5e95 100644 --- a/README.adoc +++ b/README.adoc @@ -3,7 +3,6 @@ :idprefix: :idseparator: - ifdef::env-github,env-browser[:outfilesuffix: .adoc] -endif::[] ifndef::env-github[:icons: font] // URIs :project-repo: Redislabs-Solution-Architects/riot diff --git a/build.gradle b/build.gradle index bfa9c232e..2552e3a69 100644 --- a/build.gradle +++ b/build.gradle @@ -23,27 +23,25 @@ repositories { } dependencies { + compile 'com.redislabs:picocli-redis:1.0.6' + compile 'org.springframework.batch:spring-batch-core:4.1.2.RELEASE' + compile 'org.springframework.boot:spring-boot-autoconfigure:2.1.8.RELEASE' + compile 'org.springframework:spring-jdbc:5.1.9.RELEASE' + compile 'org.springframework.cloud:spring-cloud-aws-context:2.1.3.RELEASE' + compile 'org.springframework.cloud:spring-cloud-aws-autoconfigure:2.1.3.RELEASE' compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.9.9' compile 'com.github.javafaker:javafaker:1.0.1' + compile 'com.zaxxer:HikariCP:3.4.1' compile 'com.ibm.db2:jcc:11.5.0.0' compile 'com.microsoft.sqlserver:mssql-jdbc:7.4.1.jre8' compile 'com.oracle.ojdbc:ojdbc8:19.3.0.0' + compile 'org.postgresql:postgresql:42.2.8' compile 'com.redislabs:jredisearch:1.0.0' compile 'com.redislabs:lettusearch:1.7.1' - compile 'com.zaxxer:HikariCP:3.4.1' - compile 'info.picocli:picocli:4.0.4' - compile 'org.apache.commons:commons-pool2:2.7.0' - compile 'org.latencyutils:LatencyUtils:2.0.3' - compile 'org.postgresql:postgresql:42.2.8' - compile "org.slf4j:slf4j-api:1.7.28" - compile "org.slf4j:jcl-over-slf4j:1.7.28" - compile "org.slf4j:slf4j-jdk14:1.7.28" - compile 'org.springframework.batch:spring-batch-core:4.1.2.RELEASE' - compile 'org.springframework.boot:spring-boot-autoconfigure:2.1.8.RELEASE' - compile 'org.springframework:spring-jdbc:5.1.9.RELEASE' - compile 'org.springframework.cloud:spring-cloud-aws-context:2.1.3.RELEASE' - compile 'org.springframework.cloud:spring-cloud-aws-autoconfigure:2.1.3.RELEASE' compile 'org.xerial:sqlite-jdbc:3.28.0' + compile 'org.slf4j:slf4j-api:1.7.28' + compile 'org.slf4j:jcl-over-slf4j:1.7.28' + compile 'org.slf4j:slf4j-jdk14:1.7.28' compile 'redis.clients:jedis:3.1.0' compileOnly 'org.projectlombok:lombok:1.18.10' annotationProcessor 'org.projectlombok:lombok:1.18.10' diff --git a/pom.xml b/pom.xml deleted file mode 100644 index f6c8725bd..000000000 --- a/pom.xml +++ /dev/null @@ -1,298 +0,0 @@ - - - 4.0.0 - com.redislabs - riot - 0.45.1-SNAPSHOT - RIOT - Redis Input/Output Tool - https://github.com/Redislabs-Solution-Architects/riot - - https://github.com/Redislabs-Solution-Architects/riot - scm:git:git://github.com/Redislabs-Solution-Architects/riot.git - scm:git:git@github.com:Redislabs-Solution-Architects/riot.git - HEAD - - - - jruaux - Julien Ruaux - - - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 - - - - - sonatype-snapshots - Sonatype Snapshot Repository - https://oss.sonatype.org/content/repositories/snapshots/ - - true - - - - - UTF-8 - UTF-8 - 1.8 - ${java.version} - ${java.version} - 3.8.1 - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - 2.9.9 - - - com.github.javafaker - javafaker - 1.0.1 - - - com.ibm.db2 - jcc - 11.5.0.0 - - - com.microsoft.sqlserver - mssql-jdbc - 7.4.1.jre8 - - - com.oracle.ojdbc - ojdbc8 - 19.3.0.0 - - - com.redislabs - jredisearch - 1.0.0 - - - com.redislabs - lettusearch - 1.7.3 - - - com.zaxxer - HikariCP - 3.4.1 - - - info.picocli - picocli - 4.0.4 - - - org.apache.commons - commons-pool2 - 2.7.0 - - - org.latencyutils - LatencyUtils - 2.0.3 - - - org.postgresql - postgresql - 42.2.8 - - - org.slf4j - slf4j-api - 1.7.28 - - - org.springframework.batch - spring-batch-core - 4.2.0.RELEASE - - - org.springframework.boot - spring-boot-autoconfigure - 2.1.8.RELEASE - - - org.springframework - spring-jdbc - 5.1.9.RELEASE - - - org.springframework.cloud - spring-cloud-aws-context - 2.1.3.RELEASE - - - org.springframework.cloud - spring-cloud-aws-autoconfigure - 2.1.3.RELEASE - - - org.xerial - sqlite-jdbc - 3.28.0 - - - redis.clients - jedis - 3.1.0 - - - org.projectlombok - lombok - 1.18.10 - provided - - - org.codehaus.plexus - plexus-utils - 3.2.1 - test - - - org.hsqldb - hsqldb - 2.5.0 - test - - - org.junit.jupiter - junit-jupiter-engine - 5.5.2 - test - - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - org.codehaus.mojo - cobertura-maven-plugin - 2.7 - - - html - xml - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - true - - ossrh - https://oss.sonatype.org/ - true - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.22.2 - - false - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - true - - - - org.apache.maven.plugins - maven-release-plugin - - true - false - release - deploy - - - - - - - - - release - - - - - org.apache.maven.plugins - maven-source-plugin - 3.1.0 - - true - - - - attach-sources - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.1.0 - - true - -Xdoclint:none - -Xdoclint:none - - - - attach-javadocs - - jar - - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - - - - - - - diff --git a/src/main/java/com/redislabs/riot/ManifestVersionProvider.java b/src/main/java/com/redislabs/riot/ManifestVersionProvider.java deleted file mode 100644 index 53455d401..000000000 --- a/src/main/java/com/redislabs/riot/ManifestVersionProvider.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.redislabs.riot; - -import java.io.IOException; -import java.net.URL; -import java.util.Enumeration; -import java.util.jar.Attributes; -import java.util.jar.Manifest; - -import picocli.CommandLine.IVersionProvider; - -/** - * {@link IVersionProvider} implementation that returns version information from - * the riot-x.x.x.jar file's {@code /META-INF/MANIFEST.MF} file. - */ -public class ManifestVersionProvider implements IVersionProvider { - public String[] getVersion() throws Exception { - Enumeration resources = Riot.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); - while (resources.hasMoreElements()) { - URL url = resources.nextElement(); - try { - Manifest manifest = new Manifest(url.openStream()); - if (isApplicableManifest(manifest)) { - Attributes attr = manifest.getMainAttributes(); - return new String[] { get(attr, "Implementation-Title") + " version \"" - + get(attr, "Implementation-Version") + "\"" }; - } - } catch (IOException ex) { - return new String[] { "Unable to read from " + url + ": " + ex }; - } - } - return new String[0]; - } - - private boolean isApplicableManifest(Manifest manifest) { - Attributes attributes = manifest.getMainAttributes(); - return "RIOT".equals(get(attributes, "Implementation-Title")); - } - - private static Object get(Attributes attributes, String key) { - return attributes.get(new Attributes.Name(key)); - } -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/OneLineLogFormat.java b/src/main/java/com/redislabs/riot/OneLineLogFormat.java deleted file mode 100644 index 2e3bfcae0..000000000 --- a/src/main/java/com/redislabs/riot/OneLineLogFormat.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.redislabs.riot; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.logging.Formatter; -import java.util.logging.LogRecord; - -public class OneLineLogFormat extends Formatter { - - private DateTimeFormatter d = new DateTimeFormatterBuilder().appendValue(ChronoField.HOUR_OF_DAY, 2) - .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2).optionalStart().appendLiteral(':') - .appendValue(ChronoField.SECOND_OF_MINUTE, 2).optionalStart() - .appendFraction(ChronoField.NANO_OF_SECOND, 3, 3, true).toFormatter(); - private ZoneId offset = ZoneOffset.systemDefault(); - private boolean verbose; - - public OneLineLogFormat(boolean verbose) { - this.verbose = verbose; - } - - @Override - public String format(LogRecord record) { - String message = formatMessage(record); - ZonedDateTime time = Instant.ofEpochMilli(record.getMillis()).atZone(offset); - if (record.getThrown() == null) { - return String.format("%s\t%s%n", time.format(d), message); - } - if (verbose) { - return String.format("%s\t%s%n%s%n", time.format(d), message, stackTrace(record)); - } - return String.format("%s\t%s: %s%n", time.format(d), message, record.getThrown().getMessage()); - } - - private String stackTrace(LogRecord record) { - StringWriter sw = new StringWriter(4096); - PrintWriter pw = new PrintWriter(sw); - record.getThrown().printStackTrace(pw); - return sw.toString(); - } -} diff --git a/src/main/java/com/redislabs/riot/Riot.java b/src/main/java/com/redislabs/riot/Riot.java index 3add651f6..34d334257 100644 --- a/src/main/java/com/redislabs/riot/Riot.java +++ b/src/main/java/com/redislabs/riot/Riot.java @@ -1,14 +1,8 @@ package com.redislabs.riot; -import java.security.Security; -import java.util.logging.ConsoleHandler; -import java.util.logging.Level; -import java.util.logging.LogManager; -import java.util.logging.Logger; - -import org.slf4j.LoggerFactory; import org.springframework.batch.item.file.transform.Range; +import com.redislabs.picocliredis.Main; import com.redislabs.riot.cli.ConsoleExportCommand; import com.redislabs.riot.cli.GeneratorCommand; import com.redislabs.riot.cli.db.DatabaseExportCommand; @@ -16,108 +10,25 @@ import com.redislabs.riot.cli.file.FileExportCommand; import com.redislabs.riot.cli.file.FileImportCommand; import com.redislabs.riot.cli.file.RangeConverter; -import com.redislabs.riot.cli.redis.Endpoint; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import com.redislabs.riot.cli.redis.RedisExportCommand; import com.redislabs.riot.cli.test.TestCommand; -import io.netty.util.internal.logging.InternalLoggerFactory; -import io.netty.util.internal.logging.JdkLoggerFactory; -import picocli.AutoComplete; import picocli.CommandLine; -import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; -import picocli.CommandLine.Model.CommandSpec; -import picocli.CommandLine.Option; -import picocli.CommandLine.ParseResult; -import picocli.CommandLine.PicocliException; -import picocli.CommandLine.Spec; - -@Command(name = "riot", abbreviateSynopsis = true, mixinStandardHelpOptions = true, subcommands = { - FileImportCommand.class, FileExportCommand.class, DatabaseImportCommand.class, DatabaseExportCommand.class, - RedisExportCommand.class, ConsoleExportCommand.class, GeneratorCommand.class, - TestCommand.class }, versionProvider = ManifestVersionProvider.class, usageHelpAutoWidth = true) -public class Riot implements Runnable { - - @Spec - private CommandSpec spec; - - private final static String DNS_CACHE_TTL = "networkaddress.cache.ttl"; - private final static String DNS_CACHE_NEGATIVE_TTL = "networkaddress.cache.negative.ttl"; -// private final static String SSL_PREFIX = "javax.net.ssl."; -// private final static String SSL_TRUST_STORE = SSL_PREFIX + "trustStore"; -// private final static String SSL_TRUST_STORE_TYPE = SSL_PREFIX + "trustStoreType"; -// private final static String SSL_TRUST_STORE_PASSWORD = SSL_PREFIX + "trustStorePassword"; -// private final static String SSL_KEY_STORE = SSL_PREFIX + "keyStore"; -// private final static String SSL_KEY_STORE_TYPE = SSL_PREFIX + "keyStoreType"; -// private final static String SSL_KEY_STORE_PASSWORD = SSL_PREFIX + "keyStorePassword"; - private static final String ROOT_LOGGER = ""; - - @Option(names = "--completion-script", hidden = true) - private boolean completionScript; - @Option(names = { "-d", "--debug" }, description = "Enable verbose logging") - private boolean verbose; - @Option(names = { "-q", "--quiet" }, description = "Disable all logging") - private boolean quiet; - @Option(names = "--dns-ttl", description = "DNS cache TTL", paramLabel = "") - private int dnsTtl = 0; - @Option(names = "--dns-neg-ttl", description = "DNS cache negative TTL", paramLabel = "") - private int dnsNegativeTtl = 0; - - @ArgGroup(exclusive = false, heading = "Redis connection options%n", order = 1) - private RedisConnectionOptions redisOptions = new RedisConnectionOptions(); +@Command(name = "riot", subcommands = { FileImportCommand.class, FileExportCommand.class, DatabaseImportCommand.class, + DatabaseExportCommand.class, RedisExportCommand.class, ConsoleExportCommand.class, GeneratorCommand.class, + TestCommand.class }) +public class Riot extends Main { public static void main(String[] args) { System.exit(new Riot().execute(args)); } - public RedisConnectionOptions getRedisOptions() { - return redisOptions; - } - - public int execute(String[] args) { - CommandLine commandLine = new CommandLine(this).registerConverter(Range.class, new RangeConverter()) - .registerConverter(Endpoint.class, s -> new Endpoint(s)).setCaseInsensitiveEnumValuesAllowed(true); - try { - ParseResult parseResult = commandLine.parseArgs(args); - configureLogging(); - configureDns(); - return commandLine.getExecutionStrategy().execute(parseResult); - } catch (PicocliException e) { - System.err.println(e.getMessage()); - return 1; - } - } - - private void configureDns() { - org.slf4j.Logger log = LoggerFactory.getLogger(Riot.class); - log.debug("Setting {}={}", DNS_CACHE_TTL, dnsTtl); - Security.setProperty(DNS_CACHE_TTL, String.valueOf(dnsTtl)); - log.debug("Setting {}={}", DNS_CACHE_NEGATIVE_TTL, dnsNegativeTtl); - Security.setProperty(DNS_CACHE_NEGATIVE_TTL, String.valueOf(dnsNegativeTtl)); - } - - private void configureLogging() { - InternalLoggerFactory.setDefaultFactory(JdkLoggerFactory.INSTANCE); - LogManager.getLogManager().reset(); - Logger activeLogger = Logger.getLogger(ROOT_LOGGER); - ConsoleHandler handler = new ConsoleHandler(); - handler.setLevel(Level.ALL); - handler.setFormatter(new OneLineLogFormat(verbose)); - activeLogger.addHandler(handler); - Logger.getLogger(ROOT_LOGGER).setLevel(quiet ? Level.OFF : (verbose ? Level.INFO : Level.SEVERE)); - Logger.getLogger(Riot.class.getPackage().getName()) - .setLevel(quiet ? Level.OFF : (verbose ? Level.FINEST : Level.INFO)); - } - @Override - public void run() { - if (completionScript) { - System.out.println(AutoComplete.bash(spec.name(), new CommandLine(new Riot()))); - } else { - CommandLine.usage(this, System.out); - } + protected void registerConverters(CommandLine commandLine) { + commandLine.registerConverter(Range.class, new RangeConverter()); + super.registerConverters(commandLine); } } diff --git a/src/main/java/com/redislabs/riot/batch/file/AbstractResourceItemWriter.java b/src/main/java/com/redislabs/riot/batch/file/AbstractResourceItemWriter.java index d96616d1b..ea1ee5db3 100644 --- a/src/main/java/com/redislabs/riot/batch/file/AbstractResourceItemWriter.java +++ b/src/main/java/com/redislabs/riot/batch/file/AbstractResourceItemWriter.java @@ -40,8 +40,6 @@ import org.springframework.core.io.WritableResource; import org.springframework.util.Assert; -import com.google.common.io.CountingOutputStream; - /** * Base class for item writers that write data to a file or stream. * This class provides common features like restart, force sync, append etc. diff --git a/src/main/java/com/google/common/io/CountingOutputStream.java b/src/main/java/com/redislabs/riot/batch/file/CountingOutputStream.java similarity index 97% rename from src/main/java/com/google/common/io/CountingOutputStream.java rename to src/main/java/com/redislabs/riot/batch/file/CountingOutputStream.java index ae251c575..b9fe3ed69 100644 --- a/src/main/java/com/google/common/io/CountingOutputStream.java +++ b/src/main/java/com/redislabs/riot/batch/file/CountingOutputStream.java @@ -12,7 +12,7 @@ * the License. */ -package com.google.common.io; +package com.redislabs.riot.batch.file; import java.io.FilterOutputStream; import java.io.IOException; diff --git a/src/main/java/com/redislabs/riot/batch/redis/AbstractLettuceItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/AbstractLettuceItemWriter.java new file mode 100644 index 000000000..178a150d1 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/AbstractLettuceItemWriter.java @@ -0,0 +1,43 @@ +package com.redislabs.riot.batch.redis; + +import java.util.List; + +import io.lettuce.core.api.StatefulConnection; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractLettuceItemWriter, R, O> + extends AbstractRedisItemWriter { + + private LettuceConnector connector; + + public AbstractLettuceItemWriter(LettuceConnector connector) { + this.connector = connector; + } + + @Override + public void write(List items) throws Exception { + C connection = connector.getPool().borrowObject(); + R commands = connector.getApi().apply(connection); + try { + write(items, commands); + } finally { + connector.getPool().returnObject(connection); + } + } + + protected abstract void write(List items, R commands); + + @Override + public synchronized void close() { + // Take care of multi-threaded writer by only closing on the last call + if (connector.getPool() != null && !hasActiveThreads()) { + log.debug("Closing pool"); + connector.getPool().close(); + connector.getClient().shutdown(); + connector.getResources().get().shutdown(); + } + super.close(); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/AbstractRedisItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/AbstractRedisItemWriter.java index 4573ae97f..910196346 100644 --- a/src/main/java/com/redislabs/riot/batch/redis/AbstractRedisItemWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redis/AbstractRedisItemWriter.java @@ -1,10 +1,5 @@ package com.redislabs.riot.batch.redis; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.springframework.batch.item.ExecutionContext; @@ -14,7 +9,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public abstract class AbstractRedisItemWriter extends AbstractItemStreamItemWriter> { +public abstract class AbstractRedisItemWriter extends AbstractItemStreamItemWriter { private AtomicInteger activeThreads = new AtomicInteger(0); @@ -22,45 +17,6 @@ public AbstractRedisItemWriter() { setName(ClassUtils.getShortName(this.getClass())); } - @Override - public void write(List> items) throws Exception { - for (Map item : items) { - Map flatMap = new HashMap<>(); - item.forEach((k, v) -> flatMap.putAll(flatten(k, v))); - item.clear(); - item.putAll(flatMap); - } - items.forEach(item -> item.forEach((k, v) -> { - item.putAll(flatten(k, v)); - })); - doWrite(items); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private Map flatten(String key, Object value) { - Map flatMap = new HashMap(); - if (value instanceof Map) { - ((Map) value).forEach((k, v) -> { - flatMap.putAll(flatten(key + "." + k, v)); - }); - } else { - if (value instanceof Collection) { - Collection collection = (Collection) value; - Iterator iterator = collection.iterator(); - int index = 0; - while (iterator.hasNext()) { - flatMap.putAll(flatten(key + "[" + index + "]", iterator.next())); - index++; - } - } else { - flatMap.put(key, value); - } - } - return flatMap; - } - - protected abstract void doWrite(List> flatMaps) throws Exception; - @Override public synchronized void open(ExecutionContext executionContext) { int threads = activeThreads.incrementAndGet(); diff --git a/src/main/java/com/redislabs/riot/batch/redis/AbstractRedisWriter.java b/src/main/java/com/redislabs/riot/batch/redis/AbstractRedisWriter.java new file mode 100644 index 000000000..0d6a9d8d1 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/AbstractRedisWriter.java @@ -0,0 +1,14 @@ +package com.redislabs.riot.batch.redis; + +import com.redislabs.riot.batch.redis.map.RedisCommands; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public abstract class AbstractRedisWriter implements RedisWriter { + + @Setter + protected RedisCommands commands; + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/JedisClusterItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/JedisClusterItemWriter.java deleted file mode 100644 index 27d3095af..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/JedisClusterItemWriter.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.redislabs.riot.batch.redis; - -import java.util.List; -import java.util.Map; - -import org.springframework.batch.item.support.AbstractItemStreamItemWriter; -import org.springframework.util.ClassUtils; - -import com.redislabs.riot.batch.redis.writer.RedisMapWriter; - -import lombok.extern.slf4j.Slf4j; -import redis.clients.jedis.JedisCluster; - -@Slf4j -public class JedisClusterItemWriter extends AbstractItemStreamItemWriter> { - - private JedisCluster cluster; - private RedisMapWriter writer; - - public JedisClusterItemWriter(JedisCluster cluster, RedisMapWriter writer) { - setName(ClassUtils.getShortName(JedisClusterItemWriter.class)); - this.cluster = cluster; - this.writer = writer; - } - - public void write(List> items) throws Exception { - for (Map item : items) { - try { - writer.write(cluster, item); - } catch (Exception e) { - log.error("Could not write item {}", item, e); - } - } - } - - @Override - public void close() { - cluster.close(); - super.close(); - } -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/JedisClusterWriter.java b/src/main/java/com/redislabs/riot/batch/redis/JedisClusterWriter.java new file mode 100644 index 000000000..824852837 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/JedisClusterWriter.java @@ -0,0 +1,35 @@ +package com.redislabs.riot.batch.redis; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; +import redis.clients.jedis.JedisCluster; + +@Slf4j +public class JedisClusterWriter extends AbstractRedisItemWriter { + + private JedisCluster cluster; + private RedisWriter writer; + + public JedisClusterWriter(JedisCluster cluster, RedisWriter writer) { + this.cluster = cluster; + this.writer = writer; + } + + @Override + public void write(List items) { + for (O item : items) { + try { + writer.write(cluster, item); + } catch (Exception e) { + log.error("Could not write item {}", item, e); + } + } + } + + @Override + public void close() { + cluster.close(); + super.close(); + } +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/JedisItemReader.java b/src/main/java/com/redislabs/riot/batch/redis/JedisMapReader.java similarity index 89% rename from src/main/java/com/redislabs/riot/batch/redis/JedisItemReader.java rename to src/main/java/com/redislabs/riot/batch/redis/JedisMapReader.java index b67fa883e..ab0934b2c 100644 --- a/src/main/java/com/redislabs/riot/batch/redis/JedisItemReader.java +++ b/src/main/java/com/redislabs/riot/batch/redis/JedisMapReader.java @@ -11,7 +11,7 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.util.Pool; -public class JedisItemReader extends AbstractItemCountingItemStreamItemReader> { +public class JedisMapReader extends AbstractItemCountingItemStreamItemReader> { @Setter private Integer count; @@ -29,8 +29,8 @@ public class JedisItemReader extends AbstractItemCountingItemStreamItemReader jedisPool) { - setName(ClassUtils.getShortName(JedisItemReader.class)); + public JedisMapReader(Pool jedisPool) { + setName(ClassUtils.getShortName(JedisMapReader.class)); this.jedisPool = jedisPool; } diff --git a/src/main/java/com/redislabs/riot/batch/redis/JedisItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/JedisPipelineWriter.java similarity index 57% rename from src/main/java/com/redislabs/riot/batch/redis/JedisItemWriter.java rename to src/main/java/com/redislabs/riot/batch/redis/JedisPipelineWriter.java index 7819bc076..615b09280 100644 --- a/src/main/java/com/redislabs/riot/batch/redis/JedisItemWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redis/JedisPipelineWriter.java @@ -2,11 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; - -import org.springframework.util.ClassUtils; - -import com.redislabs.riot.batch.redis.writer.RedisMapWriter; import lombok.extern.slf4j.Slf4j; import redis.clients.jedis.Jedis; @@ -15,25 +10,27 @@ import redis.clients.jedis.util.Pool; @Slf4j -public class JedisItemWriter extends AbstractRedisItemWriter { +public class JedisPipelineWriter extends AbstractRedisItemWriter { private Pool pool; - private RedisMapWriter writer; + private RedisWriter writer; - public JedisItemWriter(Pool pool, RedisMapWriter writer) { - setName(ClassUtils.getShortName(JedisItemWriter.class)); + public JedisPipelineWriter(Pool pool, RedisWriter writer) { this.pool = pool; this.writer = writer; } + @SuppressWarnings("rawtypes") @Override - protected void doWrite(List> items) { + public void write(List items) { try (Jedis jedis = pool.getResource()) { Pipeline p = jedis.pipelined(); - List> responses = new ArrayList<>(); - items.forEach(item -> responses.add(writer.write(p, item))); + List responses = new ArrayList<>(); + for (O item : items) { + responses.add((Response) writer.write(p, item)); + } p.sync(); - for (Response response : responses) { + for (Response response : responses) { if (response == null) { continue; } diff --git a/src/main/java/com/redislabs/riot/batch/redis/LettuceAsyncItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/LettuceAsyncItemWriter.java new file mode 100644 index 000000000..b313fe175 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/LettuceAsyncItemWriter.java @@ -0,0 +1,51 @@ +package com.redislabs.riot.batch.redis; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import io.lettuce.core.RedisFuture; +import io.lettuce.core.api.StatefulConnection; +import io.lettuce.core.api.async.BaseRedisAsyncCommands; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LettuceAsyncItemWriter, R extends BaseRedisAsyncCommands, O> + extends AbstractLettuceItemWriter { + + private RedisWriter writer; + private long timeout; + + public LettuceAsyncItemWriter(LettuceConnector connector, RedisWriter writer, long timeout) { + super(connector); + this.writer = writer; + this.timeout = timeout; + } + + @SuppressWarnings("rawtypes") + @Override + protected void write(List items, R commands) { + List futures = new ArrayList<>(); + commands.setAutoFlushCommands(false); + for (O item : items) { + futures.add((RedisFuture) writer.write(commands, item)); + } + commands.flushCommands(); + for (int index = 0; index < futures.size(); index++) { + RedisFuture future = futures.get(index); + if (future == null) { + continue; + } + try { + future.get(timeout, TimeUnit.SECONDS); + } catch (Exception e) { + if (log.isDebugEnabled()) { + log.debug("Could not write record {}", items.get(index), e); + } else { + log.error("Could not write record: {}", e.getMessage()); + } + } + } + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/LettuceConnector.java b/src/main/java/com/redislabs/riot/batch/redis/LettuceConnector.java new file mode 100644 index 000000000..ae925a835 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/LettuceConnector.java @@ -0,0 +1,22 @@ +package com.redislabs.riot.batch.redis; + +import java.util.function.Function; +import java.util.function.Supplier; + +import org.apache.commons.pool2.impl.GenericObjectPool; + +import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.api.StatefulConnection; +import io.lettuce.core.resource.ClientResources; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public @Getter class LettuceConnector, R> { + + private AbstractRedisClient client; + private Supplier resources; + private GenericObjectPool pool; + private Function api; + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/LettuceItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/LettuceItemWriter.java deleted file mode 100644 index 5b72cc509..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/LettuceItemWriter.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.redislabs.riot.batch.redis; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.apache.commons.pool2.impl.GenericObjectPool; - -import com.redislabs.riot.batch.redis.writer.RedisMapWriter; - -import io.lettuce.core.AbstractRedisClient; -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.StatefulConnection; -import io.lettuce.core.api.async.BaseRedisAsyncCommands; -import io.lettuce.core.resource.ClientResources; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class LettuceItemWriter, C extends BaseRedisAsyncCommands> - extends AbstractRedisItemWriter { - - private AbstractRedisClient client; - private Supplier resources; - private GenericObjectPool pool; - private RedisMapWriter writer; - private Function async; - private long timeout; - - public LettuceItemWriter(AbstractRedisClient client, Supplier resources, GenericObjectPool pool, - RedisMapWriter writer, Function async, long timeout) { - this.client = client; - this.resources = resources; - this.pool = pool; - this.writer = writer; - this.async = async; - this.timeout = timeout; - } - - @Override - protected void doWrite(List> items) throws Exception { - S connection = pool.borrowObject(); - List> futures = new ArrayList<>(); - try { - C commands = async.apply(connection); - commands.setAutoFlushCommands(false); - items.forEach(item -> futures.add(writer.write(commands, item))); - commands.flushCommands(); - for (int index = 0; index < futures.size(); index++) { - RedisFuture future = futures.get(index); - if (future == null) { - continue; - } - try { - future.get(timeout, TimeUnit.SECONDS); - } catch (Exception e) { - if (log.isDebugEnabled()) { - log.debug("Could not write record {}", items.get(index), e); - } else { - log.error("Could not write record: {}", e.getMessage()); - } - } - } - } finally { - pool.returnObject(connection); - } - } - - @Override - public synchronized void close() { - // Take care of multi-threaded writer by only closing on the last call - if (pool != null && !hasActiveThreads()) { - log.debug("Closing pool"); - pool.close(); - pool = null; - client.shutdown(); - resources.get().shutdown(); - } - super.close(); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/LettuceReactiveItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/LettuceReactiveItemWriter.java new file mode 100644 index 000000000..75476baca --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/LettuceReactiveItemWriter.java @@ -0,0 +1,59 @@ +package com.redislabs.riot.batch.redis; + +import java.util.List; + +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import io.lettuce.core.api.StatefulConnection; +import io.lettuce.core.api.reactive.BaseRedisReactiveCommands; +import lombok.extern.slf4j.Slf4j; +import reactor.core.CorePublisher; + +@Slf4j +public class LettuceReactiveItemWriter, R extends BaseRedisReactiveCommands, O> + extends AbstractLettuceItemWriter { + + private RedisWriter writer; + + public LettuceReactiveItemWriter(LettuceConnector connector, RedisWriter writer) { + super(connector); + this.writer = writer; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Override + protected void write(List items, R commands) { + for (O item : items) { + CorePublisher publisher = (CorePublisher) writer.write(commands, item); + if (publisher == null) { + continue; + } + publisher.subscribe(new Subscriber() { + + @Override + public void onComplete() { + // do nothing + } + + @Override + public void onError(Throwable t) { + if (log.isDebugEnabled()) { + log.debug("Could not write record {}", item, t); + } else { + log.error("Could not write record: {}", t.getMessage()); + } + } + + public void onNext(Object t) { + } + + @Override + public void onSubscribe(Subscription s) { + } + + }); + } + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/LettuceSyncItemWriter.java b/src/main/java/com/redislabs/riot/batch/redis/LettuceSyncItemWriter.java new file mode 100644 index 000000000..6d8e1cdb5 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/LettuceSyncItemWriter.java @@ -0,0 +1,35 @@ +package com.redislabs.riot.batch.redis; + +import java.util.List; + +import io.lettuce.core.api.StatefulConnection; +import io.lettuce.core.api.sync.BaseRedisCommands; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LettuceSyncItemWriter, R extends BaseRedisCommands, O> + extends AbstractLettuceItemWriter { + + private RedisWriter writer; + + public LettuceSyncItemWriter(LettuceConnector connector, RedisWriter writer) { + super(connector); + this.writer = writer; + } + + @Override + protected void write(List items, R commands) { + for (O item : items) { + try { + writer.write(commands, item); + } catch (Exception e) { + if (log.isDebugEnabled()) { + log.debug("Could not write record {}", item, e); + } else { + log.error("Could not write record: {}", e.getMessage()); + } + } + } + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/RedisWriter.java b/src/main/java/com/redislabs/riot/batch/redis/RedisWriter.java new file mode 100644 index 000000000..339455c6c --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/RedisWriter.java @@ -0,0 +1,7 @@ +package com.redislabs.riot.batch.redis; + +public interface RedisWriter { + + Object write(R redis, O item); + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/AbstractMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/AbstractMapWriter.java new file mode 100644 index 000000000..b8d4aebc6 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/AbstractMapWriter.java @@ -0,0 +1,74 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.support.DefaultConversionService; + +import com.redislabs.riot.batch.redis.AbstractRedisWriter; +import com.redislabs.riot.batch.redis.RedisConverter; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public abstract class AbstractMapWriter extends AbstractRedisWriter> { + + private final ConversionService conversionService = new DefaultConversionService(); + @Setter + private RedisConverter converter; + + protected T convert(Object source, Class targetType) { + return conversionService.convert(source, targetType); + } + + protected String join(Map item, String[] fields) { + return converter.join(item, fields); + } + + protected String key(Map item) { + return converter.key(item); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + protected Map stringMap(Map map) { + map.forEach((k, v) -> map.put(k, conversionService.convert(v, String.class))); + return map; + } + + @Override + public Object write(R redis, Map item) { + Map flatMap = new HashMap<>(); + item.forEach((k, v) -> flatMap.putAll(flatten(k, v))); + return write(redis, key(item), flatMap); + } + + protected abstract Object write(R redis, String key, Map item); + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private Map flatten(String key, Object value) { + Map flatMap = new HashMap(); + if (value instanceof Map) { + ((Map) value).forEach((k, v) -> { + flatMap.putAll(flatten(key + "." + k, v)); + }); + } else { + if (value instanceof Collection) { + Collection collection = (Collection) value; + Iterator iterator = collection.iterator(); + int index = 0; + while (iterator.hasNext()) { + flatMap.putAll(flatten(key + "[" + index + "]", iterator.next())); + index++; + } + } else { + flatMap.put(key, value); + } + } + return flatMap; + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/CollectionMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/CollectionMapWriter.java new file mode 100644 index 000000000..5686135a5 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/CollectionMapWriter.java @@ -0,0 +1,20 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; + +public abstract class CollectionMapWriter extends AbstractMapWriter { + + @Setter + private String[] fields = new String[0]; + + @Override + protected Object write(R redis, String key, Map item) { + String member = join(item, fields); + return write(redis, key, member, item); + } + + protected abstract Object write(R redis, String key, String member, Map item); + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/DebugMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/DebugMapWriter.java new file mode 100644 index 000000000..7fbd7b103 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/DebugMapWriter.java @@ -0,0 +1,16 @@ +package com.redislabs.riot.batch.redis.map; + +import com.redislabs.riot.batch.redis.AbstractRedisWriter; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DebugMapWriter extends AbstractRedisWriter { + + @Override + public Object write(R redis, O item) { + log.info("{}", item); + return null; + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/EvalshaMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/EvalshaMapWriter.java new file mode 100644 index 000000000..9bdf3d890 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/EvalshaMapWriter.java @@ -0,0 +1,37 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.List; +import java.util.Map; + +import io.lettuce.core.ScriptOutputType; +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class EvalshaMapWriter extends AbstractMapWriter { + + @Setter + private String sha; + @Setter + private List keys; + @Setter + private List args; + @Setter + private ScriptOutputType outputType; + + @Override + protected Object write(R redis, String key, Map item) { + String[] keys = array(this.keys, item); + String[] args = array(this.args, item); + return commands.evalsha(redis, sha, outputType, keys, args); + } + + private String[] array(List fields, Map item) { + String[] array = new String[fields.size()]; + for (int index = 0; index < fields.size(); index++) { + array[index] = convert(item.get(fields.get(index)), String.class); + } + return array; + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/ExpireMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/ExpireMapWriter.java new file mode 100644 index 000000000..390750664 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/ExpireMapWriter.java @@ -0,0 +1,22 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class ExpireMapWriter extends AbstractMapWriter { + + @Setter + private String timeoutField; + @Setter + private Long defaultTimeout; + + @Override + protected Object write(R redis, String key, Map item) { + long timeout = convert(item.getOrDefault(timeoutField, defaultTimeout), Long.class); + return commands.expire(redis, key, timeout); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/GeoaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/GeoaddMapWriter.java new file mode 100644 index 000000000..98da0e66b --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/GeoaddMapWriter.java @@ -0,0 +1,39 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class GeoaddMapWriter extends CollectionMapWriter { + + @Setter + private String longitudeField; + @Setter + private String latitudeField; + + @Override + protected Object write(R redis, String key, String member, Map item) { + Double longitude = coordinate(item, longitudeField); + if (longitude == null) { + return null; + } + Double latitude = coordinate(item, latitudeField); + if (latitude == null) { + return null; + } + return commands.geoadd(redis, key, longitude, latitude, member); + } + + private Double coordinate(Map item, String field) { + if (item.containsKey(field)) { + Object value = item.get(field); + if (value != null && !"".equals(value)) { + return convert(value, Double.class); + } + } + return null; + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/HmsetMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/HmsetMapWriter.java new file mode 100644 index 000000000..b7e3f30f2 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/HmsetMapWriter.java @@ -0,0 +1,13 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +public class HmsetMapWriter extends AbstractMapWriter { + + @SuppressWarnings("unchecked") + @Override + protected Object write(R redis, String key, Map item) { + return commands.hmset(redis, key, stringMap(item)); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/JedisClusterCommands.java b/src/main/java/com/redislabs/riot/batch/redis/map/JedisClusterCommands.java new file mode 100644 index 000000000..b8038f33a --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/JedisClusterCommands.java @@ -0,0 +1,98 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Arrays; +import java.util.Map; + +import com.redislabs.lettusearch.search.AddOptions; + +import io.lettuce.core.ScriptOutputType; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.StreamEntryID; + +public class JedisClusterCommands implements RedisCommands { + + @Override + public Object geoadd(JedisCluster redis, String key, double longitude, double latitude, String member) { + return redis.geoadd(key, longitude, latitude, member); + } + + @Override + public Object hmset(JedisCluster redis, String key, Map map) { + return redis.hmset(key, map); + } + + @Override + public Object zadd(JedisCluster redis, String key, double score, String member) { + return redis.zadd(key, score, member); + } + + @Override + public Object xadd(JedisCluster redis, String key, Map map) { + return redis.xadd(key, null, map); + } + + @Override + public Object xadd(JedisCluster redis, String key, String id, Map map) { + return redis.xadd(key, new StreamEntryID(id), map); + } + + @Override + public Object xadd(JedisCluster redis, String key, String id, Map map, long maxlen, + boolean approximateTrimming) { + return redis.xadd(key, new StreamEntryID(id), map, maxlen, approximateTrimming); + } + + @Override + public Object set(JedisCluster redis, String key, String value) { + return redis.set(key, value); + } + + @Override + public Object sadd(JedisCluster redis, String key, String member) { + return redis.sadd(key, member); + } + + @Override + public Object rpush(JedisCluster redis, String key, String member) { + return redis.rpush(key, member); + } + + @Override + public Object lpush(JedisCluster redis, String key, String member) { + return redis.lpush(key, member); + } + + @Override + public Object expire(JedisCluster redis, String key, long timeout) { + return redis.expire(key, Math.toIntExact(timeout)); + } + + @Override + public Object evalsha(JedisCluster redis, String sha, ScriptOutputType type, String[] keys, String[] args) { + return redis.evalsha(sha, Arrays.asList(keys), Arrays.asList(args)); + } + + @Override + public Object ftadd(JedisCluster redis, String index, String docId, double score, Map map, + AddOptions options) { + throw new UnsupportedOperationException("Jedis Cluster not supported with RediSearch"); + } + + @Override + public Object ftadd(JedisCluster redis, String index, String docId, double score, Map map, + AddOptions options, String payload) { + throw new UnsupportedOperationException("Jedis Cluster not supported with RediSearch"); + } + + @Override + public Object sugadd(JedisCluster redis, String index, String string, double score, boolean increment) { + throw new UnsupportedOperationException("Jedis Cluster not supported with RediSearch"); + } + + @Override + public Object sugadd(JedisCluster redis, String index, String string, double score, boolean increment, + String payload) { + throw new UnsupportedOperationException("Jedis Cluster not supported with RediSearch"); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/JedisCommands.java b/src/main/java/com/redislabs/riot/batch/redis/map/JedisCommands.java new file mode 100644 index 000000000..c6bdd77c3 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/JedisCommands.java @@ -0,0 +1,97 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Arrays; +import java.util.Map; + +import com.redislabs.lettusearch.search.AddOptions; + +import io.lettuce.core.ScriptOutputType; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.StreamEntryID; + +public class JedisCommands implements RedisCommands { + + @Override + public Object geoadd(Jedis redis, String key, double longitude, double latitude, String member) { + return redis.geoadd(key, longitude, latitude, member); + } + + @Override + public Object hmset(Jedis redis, String key, Map map) { + return redis.hmset(key, map); + } + + @Override + public Object zadd(Jedis redis, String key, double score, String member) { + return redis.zadd(key, score, member); + } + + @Override + public Object xadd(Jedis redis, String key, Map map) { + return redis.xadd(key, null, map); + } + + @Override + public Object xadd(Jedis redis, String key, String id, Map map) { + return redis.xadd(key, new StreamEntryID(id), map); + } + + @Override + public Object xadd(Jedis redis, String key, String id, Map map, long maxlen, + boolean approximateTrimming) { + return redis.xadd(key, new StreamEntryID(id), map, maxlen, approximateTrimming); + } + + @Override + public Object set(Jedis redis, String key, String value) { + return redis.set(key, value); + } + + @Override + public Object sadd(Jedis redis, String key, String member) { + return redis.sadd(key, member); + } + + @Override + public Object rpush(Jedis redis, String key, String member) { + return redis.rpush(key, member); + } + + @Override + public Object lpush(Jedis redis, String key, String member) { + return redis.lpush(key, member); + } + + @Override + public Object expire(Jedis redis, String key, long timeout) { + return redis.expire(key, Math.toIntExact(timeout)); + } + + @Override + public Object evalsha(Jedis redis, String sha, ScriptOutputType type, String[] keys, String[] args) { + return redis.evalsha(sha, Arrays.asList(keys), Arrays.asList(args)); + } + + @Override + public Object ftadd(Jedis redis, String index, String docId, double score, Map map, + AddOptions options) { + throw new UnsupportedOperationException("Jedis not supported with RediSearch"); + } + + @Override + public Object ftadd(Jedis redis, String index, String docId, double score, Map map, + AddOptions options, String payload) { + throw new UnsupportedOperationException("Jedis not supported with RediSearch"); + } + + @Override + public Object sugadd(Jedis redis, String index, String string, double score, boolean increment) { + throw new UnsupportedOperationException("Jedis not supported with RediSearch"); + } + + @Override + public Object sugadd(Jedis redis, String index, String string, double score, boolean increment, String payload) { + throw new UnsupportedOperationException("Jedis not supported with RediSearch"); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/JedisPipelineCommands.java b/src/main/java/com/redislabs/riot/batch/redis/map/JedisPipelineCommands.java new file mode 100644 index 000000000..59e28dc2d --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/JedisPipelineCommands.java @@ -0,0 +1,97 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Arrays; +import java.util.Map; + +import com.redislabs.lettusearch.search.AddOptions; + +import io.lettuce.core.ScriptOutputType; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.StreamEntryID; + +public class JedisPipelineCommands implements RedisCommands { + + @Override + public Object geoadd(Pipeline redis, String key, double longitude, double latitude, String member) { + return redis.geoadd(key, longitude, latitude, member); + } + + @Override + public Object hmset(Pipeline redis, String key, Map map) { + return redis.hmset(key, map); + } + + @Override + public Object zadd(Pipeline redis, String key, double score, String member) { + return redis.zadd(key, score, member); + } + + @Override + public Object xadd(Pipeline redis, String key, Map map) { + return redis.xadd(key, null, map); + } + + @Override + public Object xadd(Pipeline redis, String key, String id, Map map) { + return redis.xadd(key, new StreamEntryID(id), map); + } + + @Override + public Object xadd(Pipeline redis, String key, String id, Map map, long maxlen, + boolean approximateTrimming) { + return redis.xadd(key, new StreamEntryID(id), map, maxlen, approximateTrimming); + } + + @Override + public Object set(Pipeline redis, String key, String value) { + return redis.set(key, value); + } + + @Override + public Object sadd(Pipeline redis, String key, String member) { + return redis.sadd(key, member); + } + + @Override + public Object rpush(Pipeline redis, String key, String member) { + return redis.rpush(key, member); + } + + @Override + public Object lpush(Pipeline redis, String key, String member) { + return redis.lpush(key, member); + } + + @Override + public Object expire(Pipeline redis, String key, long timeout) { + return redis.expire(key, Math.toIntExact(timeout)); + } + + @Override + public Object evalsha(Pipeline redis, String sha, ScriptOutputType type, String[] keys, String[] args) { + return redis.evalsha(sha, Arrays.asList(keys), Arrays.asList(args)); + } + + @Override + public Object ftadd(Pipeline redis, String index, String docId, double score, Map map, + AddOptions options) { + throw new UnsupportedOperationException("Pipeline not supported with JRediSearch client"); + } + + @Override + public Object ftadd(Pipeline redis, String index, String docId, double score, Map map, + AddOptions options, String payload) { + throw new UnsupportedOperationException("Pipeline not supported with JRediSearch client"); + } + + @Override + public Object sugadd(Pipeline redis, String index, String string, double score, boolean increment) { + throw new UnsupportedOperationException("Pipeline not supported with JRediSearch client"); + } + + @Override + public Object sugadd(Pipeline redis, String index, String string, double score, boolean increment, String payload) { + throw new UnsupportedOperationException("Pipeline not supported with JRediSearch client"); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/LettuceAsyncCommands.java b/src/main/java/com/redislabs/riot/batch/redis/map/LettuceAsyncCommands.java new file mode 100644 index 000000000..5e5b32364 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/LettuceAsyncCommands.java @@ -0,0 +1,104 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import com.redislabs.lettusearch.RediSearchAsyncCommands; +import com.redislabs.lettusearch.search.AddOptions; + +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.XAddArgs; +import io.lettuce.core.api.async.RedisAsyncCommands; + +public class LettuceAsyncCommands implements RedisCommands> { + + @Override + public Object geoadd(RedisAsyncCommands redis, String key, double longitude, double latitude, + String member) { + return redis.geoadd(key, longitude, latitude, member); + } + + @Override + public Object hmset(RedisAsyncCommands redis, String key, Map map) { + return redis.hmset(key, map); + } + + @Override + public Object xadd(RedisAsyncCommands redis, String key, Map map) { + return redis.xadd(key, map); + } + + @Override + public Object xadd(RedisAsyncCommands redis, String key, String id, Map map, + long maxlen, boolean approximateTrimming) { + return redis.xadd(key, xaddArgs(id).maxlen(maxlen).approximateTrimming(approximateTrimming), map); + } + + private XAddArgs xaddArgs(String id) { + return new XAddArgs().id(id); + } + + @Override + public Object zadd(RedisAsyncCommands redis, String key, double score, String member) { + return redis.zadd(key, score, member); + } + + @Override + public Object xadd(RedisAsyncCommands redis, String key, String id, Map map) { + return redis.xadd(key, xaddArgs(id), map); + } + + @Override + public Object set(RedisAsyncCommands redis, String key, String value) { + return redis.set(key, value); + } + + @Override + public Object sadd(RedisAsyncCommands redis, String key, String member) { + return redis.sadd(key, member); + } + + @Override + public Object rpush(RedisAsyncCommands redis, String key, String member) { + return redis.rpush(key, member); + } + + @Override + public Object lpush(RedisAsyncCommands redis, String key, String member) { + return redis.lpush(key, member); + } + + @Override + public Object expire(RedisAsyncCommands redis, String key, long timeout) { + return redis.expire(key, timeout); + } + + @Override + public Object evalsha(RedisAsyncCommands redis, String sha, ScriptOutputType type, String[] keys, + String[] args) { + return redis.evalsha(sha, type, keys, args); + } + + @Override + public Object ftadd(RedisAsyncCommands redis, String index, String docId, double score, + Map map, AddOptions options) { + return ((RediSearchAsyncCommands) redis).add(index, docId, score, map, options); + } + + @Override + public Object ftadd(RedisAsyncCommands redis, String index, String docId, double score, + Map map, AddOptions options, String payload) { + return ((RediSearchAsyncCommands) redis).add(index, docId, score, map, options, payload); + } + + @Override + public Object sugadd(RedisAsyncCommands redis, String index, String string, double score, + boolean increment) { + return ((RediSearchAsyncCommands) redis).sugadd(index, string, score, increment); + } + + @Override + public Object sugadd(RedisAsyncCommands redis, String index, String string, double score, + boolean increment, String payload) { + return ((RediSearchAsyncCommands) redis).sugadd(index, string, score, increment, payload); + } +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/LettuceReactiveCommands.java b/src/main/java/com/redislabs/riot/batch/redis/map/LettuceReactiveCommands.java new file mode 100644 index 000000000..e42ed960e --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/LettuceReactiveCommands.java @@ -0,0 +1,105 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import com.redislabs.lettusearch.RediSearchReactiveCommands; +import com.redislabs.lettusearch.search.AddOptions; + +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.XAddArgs; +import io.lettuce.core.api.reactive.RedisReactiveCommands; + +public class LettuceReactiveCommands implements RedisCommands> { + + @Override + public Object geoadd(RedisReactiveCommands redis, String key, double longitude, double latitude, + String member) { + return redis.geoadd(key, longitude, latitude, member); + } + + @Override + public Object hmset(RedisReactiveCommands redis, String key, Map map) { + return redis.hmset(key, map); + } + + @Override + public Object xadd(RedisReactiveCommands redis, String key, Map map) { + return redis.xadd(key, map); + } + + @Override + public Object xadd(RedisReactiveCommands redis, String key, String id, Map map, + long maxlen, boolean approximateTrimming) { + return redis.xadd(key, xaddArgs(id).maxlen(maxlen).approximateTrimming(approximateTrimming), map); + } + + private XAddArgs xaddArgs(String id) { + return new XAddArgs().id(id); + } + + @Override + public Object zadd(RedisReactiveCommands redis, String key, double score, String member) { + return redis.zadd(key, score, member); + } + + @Override + public Object xadd(RedisReactiveCommands redis, String key, String id, Map map) { + return redis.xadd(key, xaddArgs(id), map); + } + + @Override + public Object set(RedisReactiveCommands redis, String key, String value) { + return redis.set(key, value); + } + + @Override + public Object sadd(RedisReactiveCommands redis, String key, String member) { + return redis.sadd(key, member); + } + + @Override + public Object rpush(RedisReactiveCommands redis, String key, String member) { + return redis.rpush(key, member); + } + + @Override + public Object lpush(RedisReactiveCommands redis, String key, String member) { + return redis.lpush(key, member); + } + + @Override + public Object expire(RedisReactiveCommands redis, String key, long timeout) { + return redis.expire(key, timeout); + } + + @Override + public Object evalsha(RedisReactiveCommands redis, String sha, ScriptOutputType type, String[] keys, + String[] args) { + return redis.evalsha(sha, type, keys, args); + } + + @Override + public Object ftadd(RedisReactiveCommands redis, String index, String docId, double score, + Map map, AddOptions options) { + return ((RediSearchReactiveCommands) redis).add(index, docId, score, map, options); + } + + @Override + public Object ftadd(RedisReactiveCommands redis, String index, String docId, double score, + Map map, AddOptions options, String payload) { + return ((RediSearchReactiveCommands) redis).add(index, docId, score, map, options, payload); + } + + @Override + public Object sugadd(RedisReactiveCommands redis, String index, String string, double score, + boolean increment) { + return ((RediSearchReactiveCommands) redis).sugadd(index, string, score, increment); + } + + @Override + public Object sugadd(RedisReactiveCommands redis, String index, String string, double score, + boolean increment, String payload) { + return ((RediSearchReactiveCommands) redis).sugadd(index, string, score, increment, payload); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/LettuceSyncCommands.java b/src/main/java/com/redislabs/riot/batch/redis/map/LettuceSyncCommands.java new file mode 100644 index 000000000..fa79262d1 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/LettuceSyncCommands.java @@ -0,0 +1,108 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import com.redislabs.lettusearch.RediSearchCommands; +import com.redislabs.lettusearch.search.AddOptions; + +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.XAddArgs; + +public class LettuceSyncCommands implements RedisCommands> { + + @Override + public Object geoadd(io.lettuce.core.api.sync.RedisCommands redis, String key, double longitude, + double latitude, String member) { + return redis.geoadd(key, longitude, latitude, member); + } + + @Override + public Object hmset(io.lettuce.core.api.sync.RedisCommands redis, String key, + Map map) { + return redis.hmset(key, map); + } + + @Override + public Object xadd(io.lettuce.core.api.sync.RedisCommands redis, String key, + Map map) { + return redis.xadd(key, map); + } + + @Override + public Object xadd(io.lettuce.core.api.sync.RedisCommands redis, String key, String id, + Map map, long maxlen, boolean approximateTrimming) { + return redis.xadd(key, xaddArgs(id).maxlen(maxlen).approximateTrimming(approximateTrimming), map); + } + + private XAddArgs xaddArgs(String id) { + return new XAddArgs().id(id); + } + + @Override + public Object zadd(io.lettuce.core.api.sync.RedisCommands redis, String key, double score, + String member) { + return redis.zadd(key, score, member); + } + + @Override + public Object xadd(io.lettuce.core.api.sync.RedisCommands redis, String key, String id, + Map map) { + return redis.xadd(key, xaddArgs(id), map); + } + + @Override + public Object set(io.lettuce.core.api.sync.RedisCommands redis, String key, String value) { + return redis.set(key, value); + } + + @Override + public Object sadd(io.lettuce.core.api.sync.RedisCommands redis, String key, String member) { + return redis.sadd(key, member); + } + + @Override + public Object rpush(io.lettuce.core.api.sync.RedisCommands redis, String key, String member) { + return redis.rpush(key, member); + } + + @Override + public Object lpush(io.lettuce.core.api.sync.RedisCommands redis, String key, String member) { + return redis.lpush(key, member); + } + + @Override + public Object expire(io.lettuce.core.api.sync.RedisCommands redis, String key, long timeout) { + return redis.expire(key, timeout); + } + + @Override + public Object evalsha(io.lettuce.core.api.sync.RedisCommands redis, String sha, + ScriptOutputType type, String[] keys, String[] args) { + return redis.evalsha(sha, type, keys, args); + } + + @Override + public Object ftadd(io.lettuce.core.api.sync.RedisCommands redis, String index, String docId, + double score, Map map, AddOptions options) { + return ((RediSearchCommands) redis).add(index, docId, score, map, options); + } + + @Override + public Object ftadd(io.lettuce.core.api.sync.RedisCommands redis, String index, String docId, + double score, Map map, AddOptions options, String payload) { + return ((RediSearchCommands) redis).add(index, docId, score, map, options, payload); + } + + @Override + public Object sugadd(io.lettuce.core.api.sync.RedisCommands redis, String index, String string, + double score, boolean increment) { + return ((RediSearchCommands) redis).sugadd(index, string, score, increment); + } + + @Override + public Object sugadd(io.lettuce.core.api.sync.RedisCommands redis, String index, String string, + double score, boolean increment, String payload) { + return ((RediSearchCommands) redis).sugadd(index, string, score, increment, payload); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/LpushMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/LpushMapWriter.java new file mode 100644 index 000000000..02baae77a --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/LpushMapWriter.java @@ -0,0 +1,12 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +public class LpushMapWriter extends CollectionMapWriter { + + @Override + protected Object write(R redis, String key, String member, Map item) { + return commands.lpush(redis, key, member); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/NoopMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/NoopMapWriter.java new file mode 100644 index 000000000..363dd9204 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/NoopMapWriter.java @@ -0,0 +1,12 @@ +package com.redislabs.riot.batch.redis.map; + +import com.redislabs.riot.batch.redis.AbstractRedisWriter; + +public class NoopMapWriter extends AbstractRedisWriter { + + @Override + public Object write(R redis, O item) { + return null; + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/RedisCommands.java b/src/main/java/com/redislabs/riot/batch/redis/map/RedisCommands.java new file mode 100644 index 000000000..d88951fc3 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/RedisCommands.java @@ -0,0 +1,44 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import com.redislabs.lettusearch.search.AddOptions; + +import io.lettuce.core.ScriptOutputType; + +public interface RedisCommands { + + Object geoadd(T redis, String key, double longitude, double latitude, String member); + + Object hmset(T redis, String key, Map map); + + Object zadd(T redis, String key, double score, String member); + + Object xadd(T redis, String key, Map map); + + Object xadd(T redis, String key, String id, Map map); + + Object xadd(T redis, String key, String id, Map map, long maxlen, boolean approximateTrimming); + + Object set(T redis, String key, String value); + + Object sadd(T redis, String key, String member); + + Object rpush(T redis, String key, String member); + + Object lpush(T redis, String key, String member); + + Object expire(T redis, String key, long timeout); + + Object evalsha(T redis, String sha, ScriptOutputType type, String[] keys, String[] args); + + Object ftadd(T redis, String index, String docId, double score, Map map, AddOptions options); + + Object ftadd(T redis, String index, String docId, double score, Map map, AddOptions options, + String payload); + + Object sugadd(T redis, String index, String string, double score, boolean increment); + + Object sugadd(T redis, String index, String string, double score, boolean increment, String payload); + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/RpushMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/RpushMapWriter.java new file mode 100644 index 000000000..4dbddaaa8 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/RpushMapWriter.java @@ -0,0 +1,12 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +public class RpushMapWriter extends CollectionMapWriter { + + @Override + protected Object write(R redis, String key, String member, Map item) { + return commands.rpush(redis, key, member); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/SaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/SaddMapWriter.java new file mode 100644 index 000000000..8e423d228 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/SaddMapWriter.java @@ -0,0 +1,12 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +public class SaddMapWriter extends CollectionMapWriter { + + @Override + protected Object write(R redis, String key, String member, Map item) { + return commands.sadd(redis, key, member); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/SetFieldMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/SetFieldMapWriter.java new file mode 100644 index 000000000..71241fa5a --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/SetFieldMapWriter.java @@ -0,0 +1,19 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class SetFieldMapWriter extends SetMapWriter { + + @Setter + private String field; + + @Override + protected String value(Map item) { + return convert(item.get(field), String.class); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/SetMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/SetMapWriter.java new file mode 100644 index 000000000..64eee925c --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/SetMapWriter.java @@ -0,0 +1,18 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +public abstract class SetMapWriter extends AbstractMapWriter { + + @Override + protected Object write(R redis, String key, Map item) { + String value = value(item); + if (value == null) { + return null; + } + return commands.set(redis, key, value); + } + + protected abstract String value(Map item); + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/SetObjectMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/SetObjectMapWriter.java similarity index 69% rename from src/main/java/com/redislabs/riot/batch/redis/writer/SetObjectMapWriter.java rename to src/main/java/com/redislabs/riot/batch/redis/map/SetObjectMapWriter.java index 9ac1f4ab2..448804f1d 100644 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/SetObjectMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redis/map/SetObjectMapWriter.java @@ -1,21 +1,21 @@ -package com.redislabs.riot.batch.redis.writer; +package com.redislabs.riot.batch.redis.map; import java.util.Map; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectWriter; +import lombok.Setter; +import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; @Slf4j -public class SetObjectMapWriter extends SetMapWriter { +@Accessors(fluent = true) +public class SetObjectMapWriter extends SetMapWriter { + @Setter private ObjectWriter objectWriter; - public void setObjectWriter(ObjectWriter objectWriter) { - this.objectWriter = objectWriter; - } - @Override protected String value(Map item) { try { diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMapWriter.java new file mode 100644 index 000000000..8f504c63d --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMapWriter.java @@ -0,0 +1,21 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class XaddIdMapWriter extends AbstractMapWriter { + + @Setter + private String idField; + + @SuppressWarnings("unchecked") + @Override + protected Object write(R redis, String key, Map item) { + String id = convert(item.remove(idField), String.class); + return commands.xadd(redis, key, id, stringMap(item)); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMaxlenMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMaxlenMapWriter.java new file mode 100644 index 000000000..de59dc717 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/XaddIdMaxlenMapWriter.java @@ -0,0 +1,25 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class XaddIdMaxlenMapWriter extends AbstractMapWriter { + + @Setter + private String idField; + @Setter + private Long maxlen; + @Setter + private boolean approximateTrimming; + + @SuppressWarnings("unchecked") + @Override + protected Object write(R redis, String key, Map item) { + String id = convert(item.remove(idField), String.class); + return commands.xadd(redis, key, id, stringMap(item), maxlen, approximateTrimming); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/XaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/XaddMapWriter.java new file mode 100644 index 000000000..27cf5cc8d --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/XaddMapWriter.java @@ -0,0 +1,13 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +@SuppressWarnings("unchecked") +public class XaddMapWriter extends AbstractMapWriter { + + @Override + protected Object write(R redis, String key, Map item) { + return commands.xadd(redis, key, stringMap(item)); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/XaddMaxlenMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/XaddMaxlenMapWriter.java new file mode 100644 index 000000000..ce60cfd6c --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/XaddMaxlenMapWriter.java @@ -0,0 +1,22 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class XaddMaxlenMapWriter extends AbstractMapWriter { + + @Setter + private Long maxlen; + @Setter + private boolean approximateTrimming; + + @SuppressWarnings("unchecked") + @Override + protected Object write(R redis, String key, Map item) { + return commands.xadd(redis, key, null, stringMap(item), maxlen, approximateTrimming); + } + +} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/map/ZaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/map/ZaddMapWriter.java new file mode 100644 index 000000000..659aef089 --- /dev/null +++ b/src/main/java/com/redislabs/riot/batch/redis/map/ZaddMapWriter.java @@ -0,0 +1,22 @@ +package com.redislabs.riot.batch.redis.map; + +import java.util.Map; + +import lombok.Setter; +import lombok.experimental.Accessors; + +@Accessors(fluent = true) +public class ZaddMapWriter extends CollectionMapWriter { + + @Setter + private String scoreField; + @Setter + private double defaultScore; + + @Override + protected Object write(R redis, String key, String member, Map item) { + Double score = convert(item.getOrDefault(scoreField, defaultScore), Double.class); + return commands.zadd(redis, key, score, member); + } + +} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/AbstractFlatMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/AbstractFlatMapWriter.java deleted file mode 100644 index 1c9e48eb7..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/AbstractFlatMapWriter.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import org.springframework.core.convert.ConversionService; -import org.springframework.core.convert.support.DefaultConversionService; - -import com.redislabs.riot.batch.redis.RedisConverter; - -public abstract class AbstractFlatMapWriter implements RedisMapWriter { - - private ConversionService conversionService = new DefaultConversionService(); - private RedisConverter converter; - - public RedisConverter getConverter() { - return converter; - } - - public void setConverter(RedisConverter converter) { - this.converter = converter; - } - - protected T convert(Object source, Class targetType) { - return conversionService.convert(source, targetType); - } - - protected String join(Map item, String[] fields) { - return converter.join(item, fields); - } - - protected String key(Map item) { - return converter.key(item); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected Map stringMap(Map map) { - map.forEach((k, v) -> map.put(k, conversionService.convert(v, String.class))); - return map; - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/AbstractRedisFlatMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/AbstractRedisFlatMapWriter.java deleted file mode 100644 index 8ff87e09f..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/AbstractRedisFlatMapWriter.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public abstract class AbstractRedisFlatMapWriter extends AbstractFlatMapWriter { - - @Override - public Response write(Pipeline pipeline, Map item) { - return write(pipeline, key(item), item); - } - - @Override - public void write(JedisCluster cluster, Map item) { - write(cluster, key(item), item); - } - - protected abstract Response write(Pipeline pipeline, String key, Map item); - - protected abstract void write(JedisCluster cluster, String key, Map item); - - @Override - public RedisFuture write(Object commands, Map item) { - return write(commands, key(item), item); - } - - protected abstract RedisFuture write(Object commands, String key, Map item); - - @Override - public String toString() { - return String.format("Redis %s", getConverter()); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/CollectionMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/CollectionMapWriter.java deleted file mode 100644 index 83af46ad8..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/CollectionMapWriter.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public abstract class CollectionMapWriter extends AbstractRedisFlatMapWriter { - - private String[] fields = new String[0]; - - public void setFields(String[] fields) { - this.fields = fields; - } - - private String member(Map item) { - return join(item, fields); - } - - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return write(pipeline, key, member(item), item); - } - - @Override - protected void write(JedisCluster cluster, String key, Map item) { - write(cluster, key, member(item), item); - } - - protected abstract void write(JedisCluster cluster, String key, String member, Map item); - - protected abstract Response write(Pipeline pipeline, String key, String member, Map item); - - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return write(commands, key, member(item), item); - } - - protected abstract RedisFuture write(Object commands, String key, String member, Map item); - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/DebugMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/DebugMapWriter.java deleted file mode 100644 index 991548a9d..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/DebugMapWriter.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import lombok.extern.slf4j.Slf4j; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -@Slf4j -public class DebugMapWriter implements RedisMapWriter { - - @Override - public RedisFuture write(Object commands, Map item) { - debug(item); - return null; - } - - private void debug(Map item) { - log.info("{}", item); - } - - @Override - public Response write(Pipeline pipeline, Map item) { - debug(item); - return null; - } - - @Override - public void write(JedisCluster cluster, Map item) { - debug(item); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/EvalshaMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/EvalshaMapWriter.java deleted file mode 100644 index 67f0f6f36..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/EvalshaMapWriter.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.ScriptOutputType; -import io.lettuce.core.api.async.RedisScriptingAsyncCommands; -import lombok.Setter; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class EvalshaMapWriter extends AbstractRedisFlatMapWriter { - - @Setter - private String sha; - @Setter - private List keys; - @Setter - private List args; - @Setter - private ScriptOutputType outputType; - - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return pipeline.evalsha(sha, Arrays.asList(keys(item)), Arrays.asList(args(item))); - } - - @Override - protected void write(JedisCluster cluster, String key, Map item) { - cluster.evalsha(sha, Arrays.asList(keys(item)), Arrays.asList(args(item))); - } - - private String[] keys(Map item) { - return array(keys, item); - } - - private String[] args(Map item) { - return array(args, item); - } - - private String[] array(List fields, Map item) { - String[] array = new String[fields.size()]; - for (int index = 0; index < fields.size(); index++) { - array[index] = convert(item.get(fields.get(index)), String.class); - } - return array; - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return ((RedisScriptingAsyncCommands) commands).evalsha(sha, outputType, keys(item), - args(item)); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/ExpireMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/ExpireMapWriter.java deleted file mode 100644 index ec418deb7..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/ExpireMapWriter.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisKeyAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class ExpireMapWriter extends AbstractRedisFlatMapWriter { - - private String timeoutField; - private Long defaultTimeout; - - public void setTimeoutField(String timeoutField) { - this.timeoutField = timeoutField; - } - - public void setDefaultTimeout(Long defaultTimeout) { - this.defaultTimeout = defaultTimeout; - } - - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return pipeline.expire(key, Math.toIntExact(timeout(item))); - } - - @Override - protected void write(JedisCluster cluster, String key, Map item) { - cluster.expire(key, Math.toIntExact(timeout(item))); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return ((RedisKeyAsyncCommands) commands).expire(key, timeout(item)); - } - - private long timeout(Map item) { - return convert(item.getOrDefault(timeoutField, defaultTimeout), Long.class); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/GeoaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/GeoaddMapWriter.java deleted file mode 100644 index abaff5e0d..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/GeoaddMapWriter.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisGeoAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class GeoaddMapWriter extends CollectionMapWriter { - - private String longitudeField; - private String latitudeField; - - public void setLatitudeField(String latitudeField) { - this.latitudeField = latitudeField; - } - - public void setLongitudeField(String longitudeField) { - this.longitudeField = longitudeField; - } - - @Override - protected Response write(Pipeline pipeline, String key, String member, Map item) { - Double longitude = longitude(item); - Double latitude = latitude(item); - if (longitude == null || latitude == null) { - return null; - } - return pipeline.geoadd(key, longitude, latitude, member); - } - - @Override - protected void write(JedisCluster cluster, String key, String member, Map item) { - Double longitude = longitude(item); - Double latitude = latitude(item); - if (longitude == null || latitude == null) { - return; - } - cluster.geoadd(key, longitude, latitude, member); - } - - private Double coordinate(Map item, String field) { - if (item.containsKey(field)) { - Object value = item.get(field); - if (value != null && !"".equals(value)) { - return convert(value, Double.class); - } - } - return null; - } - - private Double longitude(Map item) { - return coordinate(item, longitudeField); - } - - private Double latitude(Map item) { - return coordinate(item, latitudeField); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, String member, Map item) { - Double longitude = longitude(item); - Double latitude = latitude(item); - if (longitude == null || latitude == null) { - return null; - } - return ((RedisGeoAsyncCommands) commands).geoadd(key, longitude, latitude, member); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/HmsetMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/HmsetMapWriter.java deleted file mode 100644 index c848320f6..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/HmsetMapWriter.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisHashAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class HmsetMapWriter extends AbstractRedisFlatMapWriter { - - @SuppressWarnings("unchecked") - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return pipeline.hmset(key, stringMap(item)); - } - - @SuppressWarnings("unchecked") - @Override - protected void write(JedisCluster cluster, String key, Map item) { - cluster.hmset(key, stringMap(item)); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return ((RedisHashAsyncCommands) commands).hmset(key, stringMap(item)); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/LpushMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/LpushMapWriter.java deleted file mode 100644 index b7f153453..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/LpushMapWriter.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisListAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class LpushMapWriter extends CollectionMapWriter { - - @Override - protected Response write(Pipeline pipeline, String key, String member, Map item) { - return pipeline.lpush(key, member); - } - - @Override - protected void write(JedisCluster cluster, String key, String member, Map item) { - cluster.lpush(key, member); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, String member, Map item) { - return ((RedisListAsyncCommands) commands).lpush(key, member); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/NoopMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/NoopMapWriter.java deleted file mode 100644 index 18a7931c4..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/NoopMapWriter.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class NoopMapWriter implements RedisMapWriter { - - @Override - public RedisFuture write(Object commands, Map item) { - return null; - } - - @Override - public Response write(Pipeline pipeline, Map item) { - return null; - } - - @Override - public void write(JedisCluster cluster, Map item) { - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/RedisMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/RedisMapWriter.java deleted file mode 100644 index db815ab45..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/RedisMapWriter.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public interface RedisMapWriter { - - RedisFuture write(Object commands, Map item); - - Response write(Pipeline pipeline, Map item); - - void write(JedisCluster cluster, Map item); - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/RpushMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/RpushMapWriter.java deleted file mode 100644 index 4199bd1c3..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/RpushMapWriter.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisListAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class RpushMapWriter extends CollectionMapWriter { - - @Override - protected Response write(Pipeline pipeline, String key, String member, Map item) { - return pipeline.rpush(key, member); - } - - @Override - protected void write(JedisCluster cluster, String key, String member, Map item) { - cluster.rpush(key, member); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, String member, Map item) { - return ((RedisListAsyncCommands) commands).rpush(key, member); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/SaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/SaddMapWriter.java deleted file mode 100644 index 5e2524911..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/SaddMapWriter.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisSetAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class SaddMapWriter extends CollectionMapWriter { - - @Override - protected Response write(Pipeline pipeline, String key, String member, Map item) { - return pipeline.sadd(key, member); - } - - @Override - protected void write(JedisCluster cluster, String key, String member, Map item) { - cluster.sadd(key, member); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, String member, Map item) { - return ((RedisSetAsyncCommands) commands).sadd(key, member); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/SetFieldMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/SetFieldMapWriter.java deleted file mode 100644 index 5613c13a2..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/SetFieldMapWriter.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -public class SetFieldMapWriter extends SetMapWriter { - private String field; - - public void setField(String field) { - this.field = field; - } - - @Override - protected String value(Map item) { - return convert(item.get(field), String.class); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/SetMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/SetMapWriter.java deleted file mode 100644 index 9d1af6d4a..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/SetMapWriter.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisStringAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public abstract class SetMapWriter extends AbstractRedisFlatMapWriter { - - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - String value = value(item); - if (value == null) { - return null; - } - return pipeline.set(key, value); - } - - @Override - protected void write(JedisCluster cluster, String key, Map item) { - String value = value(item); - if (value == null) { - return; - } - cluster.set(key, value); - } - - protected abstract String value(Map item); - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, Map item) { - String value = value(item); - if (value == null) { - return null; - } - return ((RedisStringAsyncCommands) commands).set(key, value); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMapWriter.java deleted file mode 100644 index db59848e1..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMapWriter.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.XAddArgs; -import io.lettuce.core.api.async.RedisStreamAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -import redis.clients.jedis.StreamEntryID; - -public class XaddIdMapWriter extends AbstractRedisFlatMapWriter { - - private String idField; - - public void setIdField(String idField) { - this.idField = idField; - } - - private String id(Map item) { - return convert(item.remove(idField), String.class); - } - - @SuppressWarnings("unchecked") - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return pipeline.xadd(key, new StreamEntryID(id(item)), stringMap(item)); - } - - @SuppressWarnings("unchecked") - @Override - protected void write(JedisCluster cluster, String key, Map item) { - cluster.xadd(key, new StreamEntryID(id(item)), stringMap(item)); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return ((RedisStreamAsyncCommands) commands).xadd(key, xAddArgs(item), stringMap(item)); - } - - private XAddArgs xAddArgs(Map item) { - return new XAddArgs().id(id(item)); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMaxlenMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMaxlenMapWriter.java deleted file mode 100644 index c65daf94c..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddIdMaxlenMapWriter.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.XAddArgs; -import io.lettuce.core.api.async.RedisStreamAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -import redis.clients.jedis.StreamEntryID; - -@SuppressWarnings("unchecked") -public class XaddIdMaxlenMapWriter extends AbstractRedisFlatMapWriter { - - private String idField; - private Long maxlen; - private boolean approximateTrimming; - - public void setIdField(String idField) { - this.idField = idField; - } - - public void setMaxlen(Long maxlen) { - this.maxlen = maxlen; - } - - public void setApproximateTrimming(boolean approximateTrimming) { - this.approximateTrimming = approximateTrimming; - } - - private String id(Map item) { - return convert(item.remove(idField), String.class); - } - - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return pipeline.xadd(key, new StreamEntryID(id(item)), stringMap(item), maxlen, approximateTrimming); - } - - @Override - protected void write(JedisCluster cluster, String key, Map item) { - cluster.xadd(key, new StreamEntryID(id(item)), stringMap(item), maxlen, approximateTrimming); - } - - private XAddArgs xAddArgs(Map item) { - return new XAddArgs().id(id(item)); - } - - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return ((RedisStreamAsyncCommands) commands).xadd(key, xAddArgs(item), stringMap(item), maxlen, - approximateTrimming); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/XaddMapWriter.java deleted file mode 100644 index 1811b6aa7..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddMapWriter.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisStreamAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -import redis.clients.jedis.StreamEntryID; - -@SuppressWarnings("unchecked") -public class XaddMapWriter extends AbstractRedisFlatMapWriter { - - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return pipeline.xadd(key, null, stringMap(item)); - } - - @Override - protected void write(JedisCluster cluster, String key, Map item) { - cluster.xadd(key, null, stringMap(item)); - } - - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return ((RedisStreamAsyncCommands) commands).xadd(key, stringMap(item)); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddMaxlenMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/XaddMaxlenMapWriter.java deleted file mode 100644 index 023518ea2..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/XaddMaxlenMapWriter.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisStreamAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -import redis.clients.jedis.StreamEntryID; - -@SuppressWarnings("unchecked") -public class XaddMaxlenMapWriter extends AbstractRedisFlatMapWriter { - - private Long maxlen; - private boolean approximateTrimming; - - public void setMaxlen(Long maxlen) { - this.maxlen = maxlen; - } - - public void setApproximateTrimming(boolean approximateTrimming) { - this.approximateTrimming = approximateTrimming; - } - - @Override - protected Response write(Pipeline pipeline, String key, Map item) { - return pipeline.xadd(key, null, stringMap(item), maxlen, approximateTrimming); - } - - @Override - protected void write(JedisCluster cluster, String key, Map item) { - cluster.xadd(key, null, stringMap(item), maxlen, approximateTrimming); - } - - @Override - protected RedisFuture write(Object commands, String key, Map item) { - return ((RedisStreamAsyncCommands) commands).xadd(key, stringMap(item), maxlen, - approximateTrimming); - } - -} \ No newline at end of file diff --git a/src/main/java/com/redislabs/riot/batch/redis/writer/ZaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redis/writer/ZaddMapWriter.java deleted file mode 100644 index db66f17f4..000000000 --- a/src/main/java/com/redislabs/riot/batch/redis/writer/ZaddMapWriter.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.redislabs.riot.batch.redis.writer; - -import java.util.Map; - -import io.lettuce.core.RedisFuture; -import io.lettuce.core.api.async.RedisSortedSetAsyncCommands; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; - -public class ZaddMapWriter extends CollectionMapWriter { - - private String scoreField; - private double defaultScore; - - public void setScoreField(String scoreField) { - this.scoreField = scoreField; - } - - public void setDefaultScore(double defaultScore) { - this.defaultScore = defaultScore; - } - - private double score(Map item) { - return convert(item.getOrDefault(scoreField, defaultScore), Double.class); - } - - @Override - protected Response write(Pipeline pipeline, String key, String member, Map item) { - return pipeline.zadd(key, score(item), member); - } - - @Override - protected void write(JedisCluster cluster, String key, String member, Map item) { - cluster.zadd(key, score(item), member); - } - - @SuppressWarnings("unchecked") - @Override - protected RedisFuture write(Object commands, String key, String member, Map item) { - return ((RedisSortedSetAsyncCommands) commands).zadd(key, score(item), member); - } - -} diff --git a/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractLettuSearchMapWriter.java b/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractLettuSearchMapWriter.java index d5be7b21b..fb0ea73c6 100644 --- a/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractLettuSearchMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractLettuSearchMapWriter.java @@ -2,57 +2,23 @@ import java.util.Map; -import com.redislabs.lettusearch.RediSearchAsyncCommands; -import com.redislabs.riot.batch.redis.writer.AbstractFlatMapWriter; +import com.redislabs.riot.batch.redis.map.AbstractMapWriter; -import io.lettuce.core.RedisFuture; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; +import lombok.Setter; +import lombok.experimental.Accessors; -public abstract class AbstractLettuSearchMapWriter extends AbstractFlatMapWriter { +@Accessors(fluent = true) +public abstract class AbstractLettuSearchMapWriter extends AbstractMapWriter { - private String index; + @Setter + protected String index; + @Setter private String scoreField; + @Setter private double defaultScore = 1d; - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public void setDefaultScore(double defaultScore) { - this.defaultScore = defaultScore; - } - - public void setScoreField(String scoreField) { - this.scoreField = scoreField; - } - protected double score(Map item) { return convert(item.getOrDefault(scoreField, defaultScore), Double.class); } - @SuppressWarnings("unchecked") - @Override - public RedisFuture write(Object commands, Map item) { - return write((RediSearchAsyncCommands) commands, index, item); - } - - protected abstract RedisFuture write(RediSearchAsyncCommands commands, String index, - Map item); - - @Override - public Response write(Pipeline pipeline, Map item) { - throw new UnsupportedOperationException("Jedis not supported for RediSearch module"); - } - - @Override - public void write(JedisCluster cluster, Map item) { - throw new UnsupportedOperationException("Jedis not supported for RediSearch module"); - } - } diff --git a/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractSearchMapWriter.java b/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractSearchMapWriter.java index 14fdf0a27..5848aa9c2 100644 --- a/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractSearchMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redisearch/writer/AbstractSearchMapWriter.java @@ -1,32 +1,19 @@ package com.redislabs.riot.batch.redisearch.writer; -import java.util.Map; - -import com.redislabs.lettusearch.RediSearchAsyncCommands; import com.redislabs.lettusearch.search.AddOptions; -import io.lettuce.core.RedisFuture; - -public abstract class AbstractSearchMapWriter extends AbstractLettuSearchMapWriter { +import lombok.Setter; +import lombok.experimental.Accessors; - private AddOptions options; - - public void setOptions(AddOptions options) { - this.options = options; - } - - @Override - protected RedisFuture write(RediSearchAsyncCommands commands, String index, - Map item) { - return write(commands, index, item, options); - } +@Accessors(fluent = true) +public abstract class AbstractSearchMapWriter extends AbstractLettuSearchMapWriter { - protected abstract RedisFuture write(RediSearchAsyncCommands commands, String index, - Map item, AddOptions options); + @Setter + protected AddOptions options; @Override public String toString() { - return String.format("RediSearch index %s", getIndex()); + return String.format("RediSearch index %s", index); } } diff --git a/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddMapWriter.java index 556f7b301..a68642d07 100644 --- a/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddMapWriter.java @@ -2,18 +2,12 @@ import java.util.Map; -import com.redislabs.lettusearch.RediSearchAsyncCommands; -import com.redislabs.lettusearch.search.AddOptions; - -import io.lettuce.core.RedisFuture; - @SuppressWarnings("unchecked") -public class FtaddMapWriter extends AbstractSearchMapWriter { +public class FtaddMapWriter extends AbstractSearchMapWriter { @Override - protected RedisFuture write(RediSearchAsyncCommands commands, String index, - Map item, AddOptions options) { - return commands.add(index, key(item), score(item), stringMap(item), options); + protected Object write(R redis, String key, Map item) { + return commands.ftadd(redis, index, key, score(item), stringMap(item), options); } } diff --git a/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddPayloadMapWriter.java b/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddPayloadMapWriter.java index 0f727c3b6..cee7562ba 100644 --- a/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddPayloadMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redisearch/writer/FtaddPayloadMapWriter.java @@ -2,28 +2,20 @@ import java.util.Map; -import com.redislabs.lettusearch.RediSearchAsyncCommands; -import com.redislabs.lettusearch.search.AddOptions; - -import io.lettuce.core.RedisFuture; +import lombok.Setter; +import lombok.experimental.Accessors; @SuppressWarnings("unchecked") -public class FtaddPayloadMapWriter extends AbstractSearchMapWriter { +@Accessors(fluent = true) +public class FtaddPayloadMapWriter extends AbstractSearchMapWriter { + @Setter private String payloadField; - public void setPayloadField(String payloadField) { - this.payloadField = payloadField; - } - - private String payload(Map item) { - return convert(item.remove(payloadField), String.class); - } - @Override - protected RedisFuture write(RediSearchAsyncCommands commands, String index, - Map item, AddOptions options) { - return commands.add(index, key(item), score(item), stringMap(item), options, payload(item)); + protected Object write(R redis, String key, Map item) { + String payload = convert(item.remove(payloadField), String.class); + return commands.ftadd(redis, index, key, score(item), stringMap(item), options, payload); } } diff --git a/src/main/java/com/redislabs/riot/batch/redisearch/writer/JedisSuggestMapWriter.java b/src/main/java/com/redislabs/riot/batch/redisearch/writer/JedisSuggestMapWriter.java index c81dde8b9..9f437777f 100644 --- a/src/main/java/com/redislabs/riot/batch/redisearch/writer/JedisSuggestMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redisearch/writer/JedisSuggestMapWriter.java @@ -10,14 +10,23 @@ import io.redisearch.Suggestion; import io.redisearch.client.Client; +import lombok.Setter; +import lombok.experimental.Accessors; +@Accessors(fluent = true) public class JedisSuggestMapWriter extends AbstractItemStreamItemWriter> { + @Setter private Client client; + @Setter private String field; + @Setter private String scoreField; + @Setter private double defaultScore = 1d; + @Setter private boolean increment; + @Setter private String payloadField; private ConversionService conversionService = new DefaultConversionService(); @@ -46,24 +55,4 @@ private String payload(Map item) { return conversionService.convert(item.remove(payloadField), String.class); } - public void setField(String field) { - this.field = field; - } - - public void setScoreField(String scoreField) { - this.scoreField = scoreField; - } - - public void setDefaultScore(double defaultScore) { - this.defaultScore = defaultScore; - } - - public void setIncrement(boolean increment) { - this.increment = increment; - } - - public void setPayloadField(String payloadField) { - this.payloadField = payloadField; - } - } diff --git a/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddMapWriter.java b/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddMapWriter.java index bbd6b7011..4afa4f182 100644 --- a/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddMapWriter.java @@ -2,45 +2,39 @@ import java.util.Map; -import com.redislabs.lettusearch.RediSearchAsyncCommands; +import lombok.Setter; +import lombok.experimental.Accessors; -import io.lettuce.core.RedisFuture; - -public class SugaddMapWriter extends AbstractLettuSearchMapWriter { +@Accessors(fluent = true) +public class SugaddMapWriter extends AbstractLettuSearchMapWriter { + @Setter private String field; - private boolean increment; - - public void setField(String field) { - this.field = field; - } - - public void setIncrement(boolean increment) { - this.increment = increment; - } + @Setter + protected boolean increment; private String string(Map item) { return convert(item.get(field), String.class); } @Override - protected RedisFuture write(RediSearchAsyncCommands commands, String index, - Map item) { + protected Object write(R redis, String key, Map item) { String string = string(item); if (string == null) { return null; } - return sugadd(commands, index, string, score(item), increment, item); + double score = score(item); + return write(redis, key, item, string, score); } - protected RedisFuture sugadd(RediSearchAsyncCommands commands, String index, String string, - double score, boolean increment, @SuppressWarnings("unused") Map item) { - return commands.sugadd(index, string, score, increment); + @SuppressWarnings("unused") + protected Object write(R redis, String key, Map item, String string, double score) { + return commands.sugadd(redis, index, string, score, increment); } @Override public String toString() { - return String.format("RediSearch suggestion index %s", getIndex()); + return String.format("RediSearch suggestion index %s", index); } } diff --git a/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddPayloadMapWriter.java b/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddPayloadMapWriter.java index 5cf9d7c12..307423e33 100644 --- a/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddPayloadMapWriter.java +++ b/src/main/java/com/redislabs/riot/batch/redisearch/writer/SugaddPayloadMapWriter.java @@ -2,16 +2,18 @@ import java.util.Map; -import com.redislabs.lettusearch.RediSearchAsyncCommands; +import lombok.Setter; +import lombok.experimental.Accessors; -import io.lettuce.core.RedisFuture; - -public class SugaddPayloadMapWriter extends SugaddMapWriter { +@Accessors(fluent = true) +public class SugaddPayloadMapWriter extends SugaddMapWriter { + @Setter private String payloadField; - public void setPayloadField(String payloadField) { - this.payloadField = payloadField; + @Override + protected Object write(R redis, String key, Map item, String string, double score) { + return commands.sugadd(redis, index, string, score, increment, payload(item)); } private String payload(Map item) { @@ -21,10 +23,4 @@ private String payload(Map item) { return convert(item.remove(payloadField), String.class); } - @Override - protected RedisFuture sugadd(RediSearchAsyncCommands commands, String index, String string, - double score, boolean increment, Map item) { - return commands.sugadd(index, string, score, increment, payload(item)); - } - } diff --git a/src/main/java/com/redislabs/riot/cli/AbstractCommand.java b/src/main/java/com/redislabs/riot/cli/AbstractCommand.java index 019ec4b40..db47a38cd 100644 --- a/src/main/java/com/redislabs/riot/cli/AbstractCommand.java +++ b/src/main/java/com/redislabs/riot/cli/AbstractCommand.java @@ -1,7 +1,7 @@ package com.redislabs.riot.cli; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.Riot; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import picocli.CommandLine.Command; import picocli.CommandLine.Model.CommandSpec; @@ -25,6 +25,6 @@ public void run() { execute(spec.name(), riot.getRedisOptions()); } - public abstract void execute(String name, RedisConnectionOptions options); + public abstract void execute(String name, RedisOptions options); } diff --git a/src/main/java/com/redislabs/riot/cli/ConsoleExportCommand.java b/src/main/java/com/redislabs/riot/cli/ConsoleExportCommand.java index d279c0cb6..a833c4be1 100644 --- a/src/main/java/com/redislabs/riot/cli/ConsoleExportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/ConsoleExportCommand.java @@ -4,8 +4,8 @@ import org.springframework.batch.item.ItemWriter; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.batch.ConsoleWriter; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import picocli.CommandLine.Command; @@ -13,7 +13,7 @@ public class ConsoleExportCommand extends ExportCommand { @Override - protected ItemWriter> writer(RedisConnectionOptions options) throws Exception { + protected ItemWriter> writer(RedisOptions options) throws Exception { return new ConsoleWriter(); } diff --git a/src/main/java/com/redislabs/riot/cli/ExportCommand.java b/src/main/java/com/redislabs/riot/cli/ExportCommand.java index 0fa2a2bc2..4cc334fb9 100644 --- a/src/main/java/com/redislabs/riot/cli/ExportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/ExportCommand.java @@ -5,8 +5,8 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.cli.redis.RediSearchReaderOptions; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import com.redislabs.riot.cli.redis.RedisReaderOptions; import picocli.CommandLine.ArgGroup; @@ -22,7 +22,7 @@ public abstract class ExportCommand extends TransferCommand { @SuppressWarnings("rawtypes") @Override - protected ItemReader reader(RedisConnectionOptions redisOptions) { + protected ItemReader reader(RedisOptions redisOptions) { if (searchReader.isSet()) { return searchReader.reader(redisOptions.lettuSearchClient()); } @@ -30,8 +30,7 @@ protected ItemReader reader(RedisConnectionOptions redisOptions) { } @Override - protected ItemProcessor, Map> processor(RedisConnectionOptions options) - throws Exception { + protected ItemProcessor, Map> processor(RedisOptions options) throws Exception { return null; } diff --git a/src/main/java/com/redislabs/riot/cli/GeneratorCommand.java b/src/main/java/com/redislabs/riot/cli/GeneratorCommand.java index 31d20263b..2a8f6d67b 100644 --- a/src/main/java/com/redislabs/riot/cli/GeneratorCommand.java +++ b/src/main/java/com/redislabs/riot/cli/GeneratorCommand.java @@ -5,8 +5,8 @@ import java.util.List; import com.github.javafaker.Faker; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.batch.generator.GeneratorReader; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -18,7 +18,7 @@ public class GeneratorCommand extends ImportCommand { private GeneratorOptions options = new GeneratorOptions(); @Override - protected GeneratorReader reader(RedisConnectionOptions redisOptions) { + protected GeneratorReader reader(RedisOptions redisOptions) { return options.reader(); } diff --git a/src/main/java/com/redislabs/riot/cli/ImportCommand.java b/src/main/java/com/redislabs/riot/cli/ImportCommand.java index 518d9eddb..cc1c0fced 100644 --- a/src/main/java/com/redislabs/riot/cli/ImportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/ImportCommand.java @@ -5,7 +5,7 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemWriter; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.cli.redis.RedisWriterOptions; import lombok.Setter; @@ -23,13 +23,12 @@ public abstract class ImportCommand extends TransferCommand { private ProcessorOptions processorOptions = new ProcessorOptions(); @Override - protected ItemProcessor, Map> processor(RedisConnectionOptions options) - throws Exception { + protected ItemProcessor, Map> processor(RedisOptions options) throws Exception { return processorOptions.processor(options); } @Override - protected ItemWriter> writer(RedisConnectionOptions options) { + protected ItemWriter> writer(RedisOptions options) { return redisWriterOptions.writer(options); } diff --git a/src/main/java/com/redislabs/riot/cli/ProcessorOptions.java b/src/main/java/com/redislabs/riot/cli/ProcessorOptions.java index e1981c6c9..d99fb7a13 100644 --- a/src/main/java/com/redislabs/riot/cli/ProcessorOptions.java +++ b/src/main/java/com/redislabs/riot/cli/ProcessorOptions.java @@ -13,9 +13,9 @@ import org.springframework.batch.item.support.builder.ScriptItemProcessorBuilder; import org.springframework.core.io.FileSystemResource; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.batch.RegexProcessor; import com.redislabs.riot.batch.SpelProcessor; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import lombok.Data; import picocli.CommandLine.Option; @@ -43,8 +43,7 @@ public void addField(String name, String expression) { fields.put(name, expression); } - public ItemProcessor, Map> processor(RedisConnectionOptions redis) - throws Exception { + public ItemProcessor, Map> processor(RedisOptions redis) throws Exception { List, Map>> processors = new ArrayList<>(); if (regexes != null) { processors.add(new RegexProcessor(regexes)); @@ -79,7 +78,7 @@ public ItemProcessor, Map> processor(RedisCo return processor; } - public Object redis(RedisConnectionOptions redis) { + public Object redis(RedisOptions redis) { if (redis.isJedis()) { return redis.jedisPool(); } diff --git a/src/main/java/com/redislabs/riot/cli/RediSearchCommand.java b/src/main/java/com/redislabs/riot/cli/RediSearchCommand.java deleted file mode 100644 index 050c50095..000000000 --- a/src/main/java/com/redislabs/riot/cli/RediSearchCommand.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.redislabs.riot.cli; - -public enum RediSearchCommand { - - add, sugadd; - -} diff --git a/src/main/java/com/redislabs/riot/cli/RedisCommand.java b/src/main/java/com/redislabs/riot/cli/RedisCommand.java index 065fbcfa9..14bc404ec 100644 --- a/src/main/java/com/redislabs/riot/cli/RedisCommand.java +++ b/src/main/java/com/redislabs/riot/cli/RedisCommand.java @@ -2,6 +2,6 @@ public enum RedisCommand { - print, evalsha, expire, geoadd, hmset, lpush, noop, rpush, sadd, set, xadd, zadd; + print, evalsha, expire, geoadd, hmset, lpush, noop, rpush, sadd, set, xadd, zadd, ftadd, ftsugadd; } diff --git a/src/main/java/com/redislabs/riot/cli/TransferCommand.java b/src/main/java/com/redislabs/riot/cli/TransferCommand.java index 5d4bbf66e..8a8ce4a1e 100644 --- a/src/main/java/com/redislabs/riot/cli/TransferCommand.java +++ b/src/main/java/com/redislabs/riot/cli/TransferCommand.java @@ -11,8 +11,8 @@ import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.batch.JobExecutor; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -34,7 +34,7 @@ public JobExecution execute(String name, ItemReader reader, ItemProcessor proces } @Override - public void execute(String name, RedisConnectionOptions redisOptions) { + public void execute(String name, RedisOptions redisOptions) { ItemReader reader; try { reader = reader(redisOptions); @@ -79,10 +79,10 @@ public void execute(String name, RedisConnectionOptions redisOptions) { } } - protected abstract ItemReader reader(RedisConnectionOptions options) throws Exception; + protected abstract ItemReader reader(RedisOptions options) throws Exception; - protected abstract ItemProcessor processor(RedisConnectionOptions options) throws Exception; + protected abstract ItemProcessor processor(RedisOptions options) throws Exception; - protected abstract ItemWriter writer(RedisConnectionOptions options) throws Exception; + protected abstract ItemWriter writer(RedisOptions options) throws Exception; } diff --git a/src/main/java/com/redislabs/riot/cli/db/DatabaseExportCommand.java b/src/main/java/com/redislabs/riot/cli/db/DatabaseExportCommand.java index 9f86e3f9c..646b9dad9 100644 --- a/src/main/java/com/redislabs/riot/cli/db/DatabaseExportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/db/DatabaseExportCommand.java @@ -4,8 +4,8 @@ import org.springframework.batch.item.ItemWriter; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.cli.ExportCommand; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -17,7 +17,7 @@ public class DatabaseExportCommand extends ExportCommand { private DatabaseWriterOptions options = new DatabaseWriterOptions(); @Override - protected ItemWriter> writer(RedisConnectionOptions redisConnectionOptions) { + protected ItemWriter> writer(RedisOptions redisConnectionOptions) { return options.writer(); } diff --git a/src/main/java/com/redislabs/riot/cli/db/DatabaseImportCommand.java b/src/main/java/com/redislabs/riot/cli/db/DatabaseImportCommand.java index 3ad369bf0..94194cd75 100644 --- a/src/main/java/com/redislabs/riot/cli/db/DatabaseImportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/db/DatabaseImportCommand.java @@ -4,8 +4,8 @@ import org.springframework.batch.item.ItemReader; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.cli.ImportCommand; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -17,7 +17,7 @@ public class DatabaseImportCommand extends ImportCommand { private DatabaseReaderOptions options = new DatabaseReaderOptions(); @Override - protected ItemReader> reader(RedisConnectionOptions redisConnectionOptions) throws Exception { + protected ItemReader> reader(RedisOptions redisConnectionOptions) throws Exception { return options.reader(); } diff --git a/src/main/java/com/redislabs/riot/cli/db/DatabaseWriterOptions.java b/src/main/java/com/redislabs/riot/cli/db/DatabaseWriterOptions.java index 982d67ad4..30497861b 100644 --- a/src/main/java/com/redislabs/riot/cli/db/DatabaseWriterOptions.java +++ b/src/main/java/com/redislabs/riot/cli/db/DatabaseWriterOptions.java @@ -15,15 +15,15 @@ @Option(required = true, names = "--sql", description = "Insert SQL statement", paramLabel = "") private String sql; - @Option(names = "--assert-updates", description = "Every insert updates at least one row (default: ${DEFAULT-VALUE})", negatable = true) - private boolean assertUpdates = true; + @Option(names = "--no-assert-updates", description = "Disable insert verification") + private boolean noAssertUpdates; public JdbcBatchItemWriter> writer() { JdbcBatchItemWriterBuilder> builder = new JdbcBatchItemWriterBuilder>(); builder.itemSqlParameterSourceProvider(MapSqlParameterSource::new); builder.dataSource(dataSource()); builder.sql(sql); - builder.assertUpdates(assertUpdates); + builder.assertUpdates(!noAssertUpdates); JdbcBatchItemWriter> writer = builder.build(); writer.afterPropertiesSet(); return writer; diff --git a/src/main/java/com/redislabs/riot/cli/file/FileExportCommand.java b/src/main/java/com/redislabs/riot/cli/file/FileExportCommand.java index 592cf1814..73082f7f0 100644 --- a/src/main/java/com/redislabs/riot/cli/file/FileExportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/file/FileExportCommand.java @@ -10,10 +10,10 @@ import org.springframework.batch.item.support.AbstractItemStreamItemWriter; import org.springframework.core.io.Resource; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.batch.file.FlatResourceItemWriterBuilder; import com.redislabs.riot.batch.file.JsonResourceItemWriterBuilder; import com.redislabs.riot.cli.ExportCommand; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -44,8 +44,7 @@ public void writeHeader(Writer writer) throws IOException { } @Override - protected AbstractItemStreamItemWriter> writer(RedisConnectionOptions redisOptions) - throws IOException { + protected AbstractItemStreamItemWriter> writer(RedisOptions redisOptions) throws IOException { Resource resource = fileWriterOptions.outputResource(); switch (fileWriterOptions.type()) { case json: diff --git a/src/main/java/com/redislabs/riot/cli/file/FileImportCommand.java b/src/main/java/com/redislabs/riot/cli/file/FileImportCommand.java index 1e5a3675f..bfe1156dc 100644 --- a/src/main/java/com/redislabs/riot/cli/file/FileImportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/file/FileImportCommand.java @@ -21,8 +21,8 @@ import org.springframework.util.Assert; import com.fasterxml.jackson.databind.ObjectMapper; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.cli.ImportCommand; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -111,7 +111,7 @@ private AbstractItemCountingItemStreamItemReader> jsonReader } @Override - protected ItemReader> reader(RedisConnectionOptions redisOptions) throws Exception { + protected ItemReader> reader(RedisOptions redisOptions) throws Exception { switch (fileReaderOptions.type()) { case json: return jsonReader(); diff --git a/src/main/java/com/redislabs/riot/cli/redis/Endpoint.java b/src/main/java/com/redislabs/riot/cli/redis/Endpoint.java deleted file mode 100644 index 74e2c3599..000000000 --- a/src/main/java/com/redislabs/riot/cli/redis/Endpoint.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.redislabs.riot.cli.redis; - -public class Endpoint { - - private static final String SEPARATOR = ":"; - private String host; - private int port; - - public Endpoint(String endpoint) { - String[] split = endpoint.split(SEPARATOR); - this.host = split[0]; - this.port = Integer.parseInt(split[1]); - } - - public String getHost() { - return host; - } - - public int getPort() { - return port; - } - - public void setHost(String host) { - this.host = host; - } - - public void setPort(int port) { - this.port = port; - } - - @Override - public String toString() { - return host + SEPARATOR + port; - } - -} diff --git a/src/main/java/com/redislabs/riot/cli/redis/ExpireCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/ExpireCommandOptions.java index b450dea81..b2fc00b83 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/ExpireCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/ExpireCommandOptions.java @@ -1,6 +1,6 @@ package com.redislabs.riot.cli.redis; -import com.redislabs.riot.batch.redis.writer.ExpireMapWriter; +import com.redislabs.riot.batch.redis.map.ExpireMapWriter; import lombok.Data; import picocli.CommandLine.Option; @@ -12,10 +12,7 @@ @Option(names = "--expire-timeout", description = "Field to get the timeout value from", paramLabel = "") private String expireTimeout; - public ExpireMapWriter writer() { - ExpireMapWriter expireWriter = new ExpireMapWriter(); - expireWriter.setDefaultTimeout(expireDefaultTimeout); - expireWriter.setTimeoutField(expireTimeout); - return expireWriter; + public ExpireMapWriter writer() { + return new ExpireMapWriter().defaultTimeout(expireDefaultTimeout).timeoutField(expireTimeout); } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/GeoCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/GeoCommandOptions.java index 6e5dd5c31..9a51f9384 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/GeoCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/GeoCommandOptions.java @@ -1,7 +1,6 @@ package com.redislabs.riot.cli.redis; -import com.redislabs.riot.batch.redis.writer.GeoaddMapWriter; -import com.redislabs.riot.batch.redis.writer.RedisMapWriter; +import com.redislabs.riot.batch.redis.map.GeoaddMapWriter; import lombok.Data; import picocli.CommandLine.Option; @@ -13,11 +12,8 @@ @Option(names = "--geo-lat", description = "Latitude field", paramLabel = "") private String geoLat; - public RedisMapWriter writer() { - GeoaddMapWriter writer = new GeoaddMapWriter(); - writer.setLatitudeField(geoLat); - writer.setLongitudeField(geoLon); - return writer; + public GeoaddMapWriter writer() { + return new GeoaddMapWriter().latitudeField(geoLat).longitudeField(geoLon); } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/JedisConnectionOptions.java b/src/main/java/com/redislabs/riot/cli/redis/JedisConnectionOptions.java deleted file mode 100644 index 98860bb6f..000000000 --- a/src/main/java/com/redislabs/riot/cli/redis/JedisConnectionOptions.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.redislabs.riot.cli.redis; - -import lombok.Data; -import picocli.CommandLine.Option; -import redis.clients.jedis.Protocol; - -public @Data class JedisConnectionOptions { - - @Option(names = "--connect-timeout", description = "Connect timeout (default: ${DEFAULT-VALUE})", paramLabel = "") - private int connectTimeout = Protocol.DEFAULT_TIMEOUT; - @Option(names = "--socket-timeout", description = "Socket timeout (default: ${DEFAULT-VALUE})", paramLabel = "") - private int socketTimeout = Protocol.DEFAULT_TIMEOUT; - -} diff --git a/src/main/java/com/redislabs/riot/cli/redis/LettuceConnectionOptions.java b/src/main/java/com/redislabs/riot/cli/redis/LettuceConnectionOptions.java deleted file mode 100644 index 1b432ba1b..000000000 --- a/src/main/java/com/redislabs/riot/cli/redis/LettuceConnectionOptions.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.redislabs.riot.cli.redis; - -import java.io.File; -import java.time.Duration; - -import io.lettuce.core.ClientOptions; -import io.lettuce.core.RedisURI; -import io.lettuce.core.SslOptions; -import io.lettuce.core.event.DefaultEventPublisherOptions; -import io.lettuce.core.event.metrics.CommandLatencyEvent; -import io.lettuce.core.metrics.DefaultCommandLatencyCollectorOptions; -import io.lettuce.core.resource.ClientResources; -import io.lettuce.core.resource.DefaultClientResources; -import io.lettuce.core.resource.DefaultClientResources.Builder; -import lombok.AllArgsConstructor; -import lombok.Builder.Default; -import lombok.Data; -import lombok.NoArgsConstructor; -import picocli.CommandLine.Option; - -@lombok.Builder -@AllArgsConstructor -@NoArgsConstructor -public @Data class LettuceConnectionOptions { - - @Option(names = "--ks", description = "Path to keystore", paramLabel = "") - private File keystore; - @Option(names = "--ks-password", arity = "0..1", interactive = true, description = "Keystore password", paramLabel = "") - private String keystorePassword; - @Option(names = "--ts", description = "Path to truststore", paramLabel = "") - private File truststore; - @Option(names = "--ts-password", arity = "0..1", interactive = true, description = "Truststore password", paramLabel = "") - private String truststorePassword; - @Default - @Option(names = "--comp-threads", description = "Number of computation threads (default: ${DEFAULT-VALUE})", paramLabel = "") - private int computationThreadPoolSize = DefaultClientResources.DEFAULT_COMPUTATION_THREADS; - @Default - @Option(names = "--io-threads", description = "Number of threads for I/O operations (default: ${DEFAULT-VALUE})", paramLabel = "") - private int ioThreadPoolSize = DefaultClientResources.DEFAULT_IO_THREADS; - @Default - @Option(names = "--command-timeout", description = "Timeout for sync command execution (default: ${DEFAULT-VALUE})", paramLabel = "") - private long commandTimeout = RedisURI.DEFAULT_TIMEOUT; - @Option(names = "--metrics", description = "Show metrics") - private boolean showMetrics; - @Default - @Option(names = "--publish-on-sched", description = "Enable publish on scheduler (default: ${DEFAULT-VALUE})") - private boolean publishOnScheduler = ClientOptions.DEFAULT_PUBLISH_ON_SCHEDULER; - @Default - @Option(names = "--auto-reconnect", description = "Auto-reconnect (default: ${DEFAULT-VALUE})", negatable = true) - private boolean autoReconnect = ClientOptions.DEFAULT_AUTO_RECONNECT; - @Default - @Option(names = "--request-queue", description = "Per-connection request queue size (default: max)", paramLabel = "") - private int requestQueueSize = ClientOptions.DEFAULT_REQUEST_QUEUE_SIZE; - @Default - @Option(names = "--ssl-provider", description = "SSL provider: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") - private SslProvider sslProvider = SslProvider.jdk; - - public long getCommandTimeout() { - return commandTimeout; - } - - public ClientOptions clientOptions(boolean ssl, ClientOptions.Builder builder) { - if (ssl) { - builder.sslOptions(sslOptions()); - } - builder.publishOnScheduler(publishOnScheduler); - builder.autoReconnect(autoReconnect); - builder.requestQueueSize(requestQueueSize); - return builder.build(); - } - - private SslOptions sslOptions() { - SslOptions.Builder builder = SslOptions.builder(); - switch (sslProvider) { - case openssl: - builder.openSslProvider(); - break; - default: - builder.jdkSslProvider(); - break; - } - if (keystore != null) { - if (keystorePassword == null) { - builder.keystore(keystore); - } else { - builder.keystore(keystore, keystorePassword.toCharArray()); - } - } - if (truststore != null) { - if (truststorePassword == null) { - builder.truststore(truststore); - } else { - builder.truststore(truststore, truststorePassword); - } - } - return builder.build(); - } - - public ClientResources clientResources() { - Builder builder = DefaultClientResources.builder(); - builder.computationThreadPoolSize(computationThreadPoolSize); - builder.ioThreadPoolSize(ioThreadPoolSize); - if (showMetrics) { - builder.commandLatencyCollectorOptions(DefaultCommandLatencyCollectorOptions.builder().enable().build()); - builder.commandLatencyPublisherOptions( - DefaultEventPublisherOptions.builder().eventEmitInterval(Duration.ofSeconds(1)).build()); - } - ClientResources resources = builder.build(); - if (showMetrics) { - resources.eventBus().get().filter(redisEvent -> redisEvent instanceof CommandLatencyEvent) - .cast(CommandLatencyEvent.class).subscribe(e -> System.out.println(e.getLatencies())); - } - return resources; - } - -} diff --git a/src/main/java/com/redislabs/riot/cli/redis/LuaCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/LuaCommandOptions.java index 45dc79b2c..15bd81e73 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/LuaCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/LuaCommandOptions.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -import com.redislabs.riot.batch.redis.writer.EvalshaMapWriter; +import com.redislabs.riot.batch.redis.map.EvalshaMapWriter; import io.lettuce.core.ScriptOutputType; import lombok.Data; @@ -20,13 +20,8 @@ @Option(names = "--eval-output", description = "Output: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") private ScriptOutputType outputType = ScriptOutputType.STATUS; - public EvalshaMapWriter writer() { - EvalshaMapWriter luaWriter = new EvalshaMapWriter(); - luaWriter.setArgs(evalArgs); - luaWriter.setKeys(evalKeys); - luaWriter.setOutputType(outputType); - luaWriter.setSha(evalSha); - return luaWriter; + public EvalshaMapWriter writer() { + return new EvalshaMapWriter().args(evalArgs).keys(evalKeys).outputType(outputType).sha(evalSha); } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/RediSearchCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/RediSearchCommandOptions.java index 9ce43a586..46550a053 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/RediSearchCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/RediSearchCommandOptions.java @@ -8,16 +8,13 @@ import com.redislabs.riot.batch.redisearch.writer.FtaddPayloadMapWriter; import com.redislabs.riot.batch.redisearch.writer.SugaddMapWriter; import com.redislabs.riot.batch.redisearch.writer.SugaddPayloadMapWriter; -import com.redislabs.riot.cli.RediSearchCommand; +import com.redislabs.riot.cli.RedisCommand; import lombok.Data; import picocli.CommandLine.Option; public @Data class RediSearchCommandOptions { - @Option(names = { "-r", - "--ft-command" }, description = "RediSearch command: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") - private RediSearchCommand command = RediSearchCommand.add; @Option(names = { "-i", "--index" }, description = "Name of the RediSearch index", paramLabel = "") private String index; @Option(names = "--nosave", description = "Do not save docs, only index") @@ -41,41 +38,39 @@ @Option(names = "--suggest-increment", description = "Use increment to set value") private boolean increment; - private AbstractSearchMapWriter searchWriter() { + private AbstractSearchMapWriter searchWriter() { if (payloadField == null) { - return new FtaddMapWriter(); + return new FtaddMapWriter<>(); } - FtaddPayloadMapWriter writer = new FtaddPayloadMapWriter(); - writer.setPayloadField(payloadField); - return writer; + return new FtaddPayloadMapWriter().payloadField(payloadField); } - private SugaddMapWriter suggestWriter() { + private SugaddMapWriter suggestWriter() { if (payloadField == null) { - return new SugaddMapWriter(); + return new SugaddMapWriter<>(); } - SugaddPayloadMapWriter writer = new SugaddPayloadMapWriter(); - writer.setPayloadField(payloadField); + return new SugaddPayloadMapWriter().payloadField(payloadField); + } + + public AbstractLettuSearchMapWriter writer(RedisCommand command) { + AbstractLettuSearchMapWriter writer = createWriter(command); + writer.index(index); + writer.scoreField(scoreField); + writer.defaultScore(defaultScore); return writer; } - public AbstractLettuSearchMapWriter writer() { + private AbstractLettuSearchMapWriter createWriter(RedisCommand command) { switch (command) { - case sugadd: - SugaddMapWriter suggestWriter = suggestWriter(); - suggestWriter.setIndex(index); - suggestWriter.setScoreField(scoreField); - suggestWriter.setDefaultScore(defaultScore); - suggestWriter.setField(suggestField); - suggestWriter.setIncrement(increment); + case ftsugadd: + SugaddMapWriter suggestWriter = suggestWriter(); + suggestWriter.field(suggestField); + suggestWriter.increment(increment); return suggestWriter; default: - AbstractSearchMapWriter addWriter = searchWriter(); - addWriter.setOptions(AddOptions.builder().ifCondition(ifCondition).language(language).noSave(noSave) + AbstractSearchMapWriter addWriter = searchWriter(); + addWriter.options(AddOptions.builder().ifCondition(ifCondition).language(language).noSave(noSave) .replace(replace).replacePartial(partial).build()); - addWriter.setDefaultScore(defaultScore); - addWriter.setScoreField(scoreField); - addWriter.setIndex(index); return addWriter; } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/RedisCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/RedisCommandOptions.java index d3e984ed0..ace7aac0e 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/RedisCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/RedisCommandOptions.java @@ -2,15 +2,16 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; -import com.redislabs.riot.batch.redis.writer.CollectionMapWriter; -import com.redislabs.riot.batch.redis.writer.DebugMapWriter; -import com.redislabs.riot.batch.redis.writer.HmsetMapWriter; -import com.redislabs.riot.batch.redis.writer.LpushMapWriter; -import com.redislabs.riot.batch.redis.writer.NoopMapWriter; -import com.redislabs.riot.batch.redis.writer.RedisMapWriter; -import com.redislabs.riot.batch.redis.writer.RpushMapWriter; -import com.redislabs.riot.batch.redis.writer.SaddMapWriter; +import com.redislabs.riot.batch.redis.AbstractRedisWriter; +import com.redislabs.riot.batch.redis.map.CollectionMapWriter; +import com.redislabs.riot.batch.redis.map.DebugMapWriter; +import com.redislabs.riot.batch.redis.map.HmsetMapWriter; +import com.redislabs.riot.batch.redis.map.LpushMapWriter; +import com.redislabs.riot.batch.redis.map.NoopMapWriter; +import com.redislabs.riot.batch.redis.map.RpushMapWriter; +import com.redislabs.riot.batch.redis.map.SaddMapWriter; import com.redislabs.riot.cli.RedisCommand; import lombok.Data; @@ -19,9 +20,6 @@ public @Data class RedisCommandOptions { - @Option(names = { "-c", - "--command" }, description = "Command: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") - private RedisCommand command = RedisCommand.hmset; @Option(names = "--members", arity = "1..*", description = "Member fields for collections: list geo set zset", paramLabel = "") private List members = new ArrayList<>(); @ArgGroup(exclusive = false, heading = "Lua command options%n", order = 21) @@ -37,15 +35,15 @@ @ArgGroup(exclusive = false, heading = "Expire command options%n", order = 21) private ExpireCommandOptions expire = new ExpireCommandOptions(); - public RedisMapWriter writer() { - RedisMapWriter redisItemWriter = redisItemWriter(command); + public AbstractRedisWriter> writer(RedisCommand command) { + AbstractRedisWriter> redisItemWriter = redisItemWriter(command); if (redisItemWriter instanceof CollectionMapWriter) { - ((CollectionMapWriter) redisItemWriter).setFields(members.toArray(new String[members.size()])); + ((CollectionMapWriter) redisItemWriter).setFields(members.toArray(new String[members.size()])); } return redisItemWriter; } - private RedisMapWriter redisItemWriter(RedisCommand command) { + private AbstractRedisWriter> redisItemWriter(RedisCommand command) { switch (command) { case evalsha: return lua.writer(); @@ -54,11 +52,11 @@ private RedisMapWriter redisItemWriter(RedisCommand command) { case geoadd: return geo.writer(); case lpush: - return new LpushMapWriter(); + return new LpushMapWriter<>(); case rpush: - return new RpushMapWriter(); + return new RpushMapWriter<>(); case sadd: - return new SaddMapWriter(); + return new SaddMapWriter<>(); case set: return string.writer(); case xadd: @@ -66,11 +64,11 @@ private RedisMapWriter redisItemWriter(RedisCommand command) { case zadd: return zset.writer(); case print: - return new DebugMapWriter(); + return new DebugMapWriter<>(); case noop: - return new NoopMapWriter(); + return new NoopMapWriter<>(); default: - return new HmsetMapWriter(); + return new HmsetMapWriter<>(); } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/RedisConnectionOptions.java b/src/main/java/com/redislabs/riot/cli/redis/RedisConnectionOptions.java deleted file mode 100644 index ab4779f53..000000000 --- a/src/main/java/com/redislabs/riot/cli/redis/RedisConnectionOptions.java +++ /dev/null @@ -1,174 +0,0 @@ -package com.redislabs.riot.cli.redis; - -import java.time.Duration; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import org.apache.commons.pool2.impl.GenericObjectPool; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; -import org.springframework.util.ClassUtils; - -import com.redislabs.lettusearch.RediSearchClient; -import com.redislabs.riot.Riot; - -import io.lettuce.core.ClientOptions; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulConnection; -import io.lettuce.core.cluster.ClusterClientOptions; -import io.lettuce.core.cluster.RedisClusterClient; -import io.lettuce.core.support.ConnectionPoolSupport; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import picocli.CommandLine.ArgGroup; -import picocli.CommandLine.Option; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; -import redis.clients.jedis.JedisSentinelPool; -import redis.clients.jedis.util.Pool; - -@Slf4j -public @Data class RedisConnectionOptions { - - @Option(names = "--driver", description = "Redis driver: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") - private RedisDriver driver = RedisDriver.lettuce; - @Option(names = { "-s", - "--server" }, description = "Redis server address (default: ${DEFAULT-VALUE})", paramLabel = "") - private List servers = Arrays.asList(new Endpoint("localhost:6379")); - @Option(names = "--sentinel", description = "Sentinel master", paramLabel = "") - private String sentinelMaster; - @Option(names = "--auth", arity = "0..1", interactive = true, description = "Database login password", paramLabel = "") - private String password; - @Option(names = "--db", description = "Redis database number", paramLabel = "") - private int database = 0; - @Option(names = "--client", description = "Redis client name (default: ${DEFAULT-VALUE})", paramLabel = "") - private String clientName = ClassUtils.getShortName(Riot.class).toLowerCase(); - @Option(names = "--ssl", description = "SSL connection") - private boolean ssl; - @Option(names = "--cluster", description = "Connect to a Redis cluster") - private boolean cluster; - @Option(names = "--max-redirects", description = "Max cluster redirects (default: ${DEFAULT-VALUE})", paramLabel = "") - private int maxRedirects = ClusterClientOptions.DEFAULT_MAX_REDIRECTS; - @Option(names = "--max-total", description = "Max connections; -1 for no limit (default: ${DEFAULT-VALUE})", paramLabel = "") - private int maxTotal = GenericObjectPoolConfig.DEFAULT_MAX_TOTAL; - @Option(names = "--min-idle", description = "Min idle connections (default: ${DEFAULT-VALUE})", paramLabel = "") - private int minIdle = GenericObjectPoolConfig.DEFAULT_MIN_IDLE; - @Option(names = "--max-idle", description = "Max idle connections; -1 for no limit (default: ${DEFAULT-VALUE})", paramLabel = "") - private int maxIdle = GenericObjectPoolConfig.DEFAULT_MAX_IDLE; - @Option(names = "--max-wait", description = "Max duration; -1 for infinite (default: ${DEFAULT-VALUE})", paramLabel = "") - private long maxWait = GenericObjectPoolConfig.DEFAULT_MAX_WAIT_MILLIS; - @ArgGroup(exclusive = false, heading = "Jedis connection options%n", order = 2) - private JedisConnectionOptions jedis = new JedisConnectionOptions(); - @ArgGroup(exclusive = false, heading = "Lettuce connection options%n", order = 3) - private LettuceConnectionOptions lettuce = LettuceConnectionOptions.builder().build(); - - public boolean isCluster() { - return cluster; - } - - public JedisCluster jedisCluster() { - Set hostAndPort = new HashSet<>(); - servers.forEach(node -> hostAndPort.add(new HostAndPort(node.getHost(), node.getPort()))); - if (password == null) { - return new JedisCluster(hostAndPort, jedis.getConnectTimeout(), jedis.getSocketTimeout(), maxRedirects, - configure(new JedisPoolConfig())); - } - return new JedisCluster(hostAndPort, jedis.getConnectTimeout(), jedis.getSocketTimeout(), maxRedirects, - password, configure(new JedisPoolConfig())); - } - - public Pool jedisPool() { - JedisPoolConfig poolConfig = configure(new JedisPoolConfig()); - if (sentinelMaster == null) { - String host = servers.get(0).getHost(); - int port = servers.get(0).getPort(); - log.debug("Creating Jedis connection pool for {}:{} with {}", host, port, poolConfig); - return new JedisPool(poolConfig, host, port, jedis.getConnectTimeout(), jedis.getSocketTimeout(), password, - database, clientName); - } - return new JedisSentinelPool(sentinelMaster, - servers.stream().map(e -> e.toString()).collect(Collectors.toSet()), poolConfig, - jedis.getConnectTimeout(), jedis.getSocketTimeout(), password, database, clientName); - - } - - @SuppressWarnings("rawtypes") - private T configure(T poolConfig) { - poolConfig.setMaxTotal(maxTotal); - poolConfig.setMaxIdle(maxIdle); - poolConfig.setMinIdle(minIdle); - poolConfig.setMaxWaitMillis(maxWait); - poolConfig.setJmxEnabled(false); - return poolConfig; - } - - public > GenericObjectPool pool(Supplier supplier) { - return ConnectionPoolSupport.createGenericObjectPool(supplier, configure(new GenericObjectPoolConfig<>()), - false); - } - - public RedisClient lettuceClient() { - log.debug("Creating Lettuce client"); - RedisClient client = RedisClient.create(lettuce.clientResources(), redisURI()); - client.setOptions(lettuce.clientOptions(ssl, ClientOptions.builder())); - return client; - } - - public RedisClusterClient lettuceClusterClient() { - log.debug("Creating Lettuce cluster client"); - RedisClusterClient client = RedisClusterClient.create(lettuce.clientResources(), - servers.stream().map(e -> redisURI(e)).collect(Collectors.toList())); - ClusterClientOptions.Builder builder = ClusterClientOptions.builder(); - builder.maxRedirects(maxRedirects); - client.setOptions((ClusterClientOptions) lettuce.clientOptions(ssl, builder)); - return client; - } - - public RediSearchClient lettuSearchClient() { - log.debug("Creating LettuSearch client"); - RediSearchClient client = RediSearchClient.create(lettuce.clientResources(), redisURI()); - client.setOptions(lettuce.clientOptions(ssl, ClientOptions.builder())); - return client; - } - - private RedisURI redisURI() { - if (sentinelMaster == null) { - return redisURI(servers.get(0)); - } - RedisURI.Builder builder = RedisURI.Builder.sentinel(servers.get(0).getHost(), servers.get(0).getPort(), - sentinelMaster); - servers.forEach(e -> builder.withSentinel(e.getHost(), e.getPort())); - return redisURI(builder); - } - - private RedisURI redisURI(Endpoint endpoint) { - RedisURI.Builder builder = RedisURI.Builder.redis(endpoint.getHost()).withPort(endpoint.getPort()); - return redisURI(builder); - } - - private RedisURI redisURI(RedisURI.Builder builder) { - builder.withClientName(clientName).withDatabase(database).withTimeout(Duration.ofSeconds(getCommandTimeout())); - if (password != null) { - builder.withPassword(password); - } - if (ssl) { - builder.withSsl(ssl); - } - return builder.build(); - } - - public boolean isJedis() { - return driver == RedisDriver.jedis; - } - - public long getCommandTimeout() { - return lettuce.getCommandTimeout(); - } -} diff --git a/src/main/java/com/redislabs/riot/cli/redis/RedisDriver.java b/src/main/java/com/redislabs/riot/cli/redis/RedisDriver.java deleted file mode 100644 index cf8e75e4a..000000000 --- a/src/main/java/com/redislabs/riot/cli/redis/RedisDriver.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.redislabs.riot.cli.redis; - -public enum RedisDriver { - jedis, lettuce -} diff --git a/src/main/java/com/redislabs/riot/cli/redis/RedisExportCommand.java b/src/main/java/com/redislabs/riot/cli/redis/RedisExportCommand.java index 9a8b9b82a..9d5001c09 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/RedisExportCommand.java +++ b/src/main/java/com/redislabs/riot/cli/redis/RedisExportCommand.java @@ -4,6 +4,7 @@ import org.springframework.batch.item.ItemWriter; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.cli.ExportCommand; import lombok.Setter; @@ -16,13 +17,13 @@ public class RedisExportCommand extends ExportCommand { @Setter @ArgGroup(exclusive = false, heading = "Target Redis connection options%n", order = 1) - private RedisConnectionOptions redis = new RedisConnectionOptions(); + private RedisOptions redis = new RedisOptions(); @Setter @Mixin private RedisWriterOptions writer = new RedisWriterOptions(); @Override - protected ItemWriter> writer(RedisConnectionOptions sourceRedisOptions) { + protected ItemWriter> writer(RedisOptions sourceRedisOptions) { return writer.writer(redis); } diff --git a/src/main/java/com/redislabs/riot/cli/redis/RedisReaderOptions.java b/src/main/java/com/redislabs/riot/cli/redis/RedisReaderOptions.java index 87e121982..b20488123 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/RedisReaderOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/RedisReaderOptions.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -import com.redislabs.riot.batch.redis.JedisItemReader; +import com.redislabs.riot.batch.redis.JedisMapReader; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -23,10 +23,10 @@ @Option(names = { "--scan-keys" }, arity = "1..*", description = "Key fields", paramLabel = "") private List keys = new ArrayList<>(); - public JedisItemReader reader(Pool jedisPool) { + public JedisMapReader reader(Pool jedisPool) { String scanPattern = scanPattern(); log.debug("Creating Redis reader with match={} and count={}", scanPattern, count); - JedisItemReader reader = new JedisItemReader(jedisPool); + JedisMapReader reader = new JedisMapReader(jedisPool); reader.setCount(count); reader.setMatch(scanPattern); reader.setKeys(keys.toArray(new String[keys.size()])); diff --git a/src/main/java/com/redislabs/riot/cli/redis/RedisWriterOptions.java b/src/main/java/com/redislabs/riot/cli/redis/RedisWriterOptions.java index d2c60b932..ef3ee53f7 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/RedisWriterOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/RedisWriterOptions.java @@ -7,12 +7,19 @@ import com.redislabs.lettusearch.RediSearchAsyncCommands; import com.redislabs.lettusearch.RediSearchClient; import com.redislabs.lettusearch.StatefulRediSearchConnection; -import com.redislabs.riot.batch.redis.JedisClusterItemWriter; -import com.redislabs.riot.batch.redis.JedisItemWriter; -import com.redislabs.riot.batch.redis.LettuceItemWriter; -import com.redislabs.riot.batch.redis.writer.AbstractFlatMapWriter; -import com.redislabs.riot.batch.redis.writer.RedisMapWriter; -import com.redislabs.riot.batch.redisearch.writer.AbstractLettuSearchMapWriter; +import com.redislabs.picocliredis.RedisOptions; +import com.redislabs.riot.batch.redis.AbstractRedisWriter; +import com.redislabs.riot.batch.redis.JedisClusterWriter; +import com.redislabs.riot.batch.redis.JedisPipelineWriter; +import com.redislabs.riot.batch.redis.LettuceAsyncItemWriter; +import com.redislabs.riot.batch.redis.LettuceConnector; +import com.redislabs.riot.batch.redis.RedisWriter; +import com.redislabs.riot.batch.redis.map.AbstractMapWriter; +import com.redislabs.riot.batch.redis.map.JedisClusterCommands; +import com.redislabs.riot.batch.redis.map.JedisPipelineCommands; +import com.redislabs.riot.batch.redis.map.LettuceAsyncCommands; +import com.redislabs.riot.batch.redis.map.RedisCommands; +import com.redislabs.riot.cli.RedisCommand; import io.lettuce.core.RedisClient; import io.lettuce.core.api.StatefulRedisConnection; @@ -22,43 +29,80 @@ import io.lettuce.core.cluster.api.async.RedisClusterAsyncCommands; import lombok.Data; import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Option; public @Data class RedisWriterOptions { + @Option(names = { "-c", + "--command" }, description = "Redis command: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") + private RedisCommand command = RedisCommand.hmset; @ArgGroup(exclusive = false, heading = "Redis key options%n", order = 10) private KeyOptions keyOptions = new KeyOptions(); @ArgGroup(exclusive = false, heading = "Redis command options%n", order = 20) - private RedisCommandOptions redisCommandOptions = new RedisCommandOptions(); + private RedisCommandOptions redisWriterOptions = new RedisCommandOptions(); @ArgGroup(exclusive = false, heading = "RediSearch command options%n", order = 30) - private RediSearchCommandOptions rediSearchCommandOptions = new RediSearchCommandOptions(); + private RediSearchCommandOptions rediSearchWriterOptions = new RediSearchCommandOptions(); - public ItemWriter> writer(RedisConnectionOptions redis) { - RedisMapWriter mapWriter = rediSearchCommandOptions.isSet() ? rediSearchCommandOptions.writer() : redisCommandOptions.writer(); - if (mapWriter instanceof AbstractFlatMapWriter) { - ((AbstractFlatMapWriter) mapWriter).setConverter(keyOptions.converter()); - } - if (mapWriter instanceof AbstractLettuSearchMapWriter) { - RediSearchClient client = redis.lettuSearchClient(); - return new LettuceItemWriter, RediSearchAsyncCommands>( - client, client::getResources, redis.pool(client::connect), mapWriter, - StatefulRediSearchConnection::async, redis.getCommandTimeout()); - } + @SuppressWarnings({ "unchecked", "rawtypes" }) + public ItemWriter> writer(RedisOptions redis) { if (redis.isJedis()) { if (redis.isCluster()) { - return new JedisClusterItemWriter(redis.jedisCluster(), mapWriter); + return new JedisClusterWriter<>(redis.jedisCluster(), mapWriter(redis)); } - return new JedisItemWriter(redis.jedisPool(), mapWriter); + return new JedisPipelineWriter<>(redis.jedisPool(), mapWriter(redis)); + } + return new LettuceAsyncItemWriter(lettuceConnector(redis), mapWriter(redis), redis.getCommandTimeout()); + } + + @SuppressWarnings("rawtypes") + private LettuceConnector lettuceConnector(RedisOptions redis) { + if (isRediSearch()) { + RediSearchClient client = redis.lettuSearchClient(); + return new LettuceConnector, RediSearchAsyncCommands>( + client, client::getResources, redis.pool(client::connect), StatefulRediSearchConnection::async); } if (redis.isCluster()) { RedisClusterClient client = redis.lettuceClusterClient(); - return new LettuceItemWriter, RedisClusterAsyncCommands>( - client, client::getResources, redis.pool(client::connect), mapWriter, - StatefulRedisClusterConnection::async, redis.getCommandTimeout()); + return new LettuceConnector, RedisClusterAsyncCommands>( + client, client::getResources, redis.pool(client::connect), StatefulRedisClusterConnection::async); } RedisClient client = redis.lettuceClient(); - return new LettuceItemWriter, RedisAsyncCommands>( - client, client::getResources, redis.pool(client::connect), mapWriter, StatefulRedisConnection::async, - redis.getCommandTimeout()); + return new LettuceConnector, RedisAsyncCommands>( + client, client::getResources, redis.pool(client::connect), StatefulRedisConnection::async); + + } + + @SuppressWarnings("unchecked") + private RedisWriter> mapWriter(RedisOptions redis) { + AbstractRedisWriter> writer = writer(); + writer.commands(redisCommands(redis)); + if (writer instanceof AbstractMapWriter) { + AbstractMapWriter mapWriter = (AbstractMapWriter) writer; + mapWriter.converter(keyOptions.converter()); + } + return writer; + } + + @SuppressWarnings("rawtypes") + private RedisCommands redisCommands(RedisOptions redis) { + if (redis.isJedis()) { + if (redis.isCluster()) { + return new JedisClusterCommands(); + } + return new JedisPipelineCommands(); + } + return new LettuceAsyncCommands(); + } + + private AbstractRedisWriter> writer() { + if (isRediSearch()) { + return rediSearchWriterOptions.writer(command); + } + return redisWriterOptions.writer(command); + } + + private boolean isRediSearch() { + return command == RedisCommand.ftadd || command == RedisCommand.ftsugadd; } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/SslProvider.java b/src/main/java/com/redislabs/riot/cli/redis/SslProvider.java deleted file mode 100644 index 0f091c179..000000000 --- a/src/main/java/com/redislabs/riot/cli/redis/SslProvider.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.redislabs.riot.cli.redis; - -public enum SslProvider { - jdk, openssl -} diff --git a/src/main/java/com/redislabs/riot/cli/redis/StreamCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/StreamCommandOptions.java index 564d07c7d..e76e5c527 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/StreamCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/StreamCommandOptions.java @@ -1,10 +1,10 @@ package com.redislabs.riot.cli.redis; -import com.redislabs.riot.batch.redis.writer.AbstractRedisFlatMapWriter; -import com.redislabs.riot.batch.redis.writer.XaddIdMapWriter; -import com.redislabs.riot.batch.redis.writer.XaddIdMaxlenMapWriter; -import com.redislabs.riot.batch.redis.writer.XaddMapWriter; -import com.redislabs.riot.batch.redis.writer.XaddMaxlenMapWriter; +import com.redislabs.riot.batch.redis.map.AbstractMapWriter; +import com.redislabs.riot.batch.redis.map.XaddIdMapWriter; +import com.redislabs.riot.batch.redis.map.XaddIdMaxlenMapWriter; +import com.redislabs.riot.batch.redis.map.XaddMapWriter; +import com.redislabs.riot.batch.redis.map.XaddMaxlenMapWriter; import lombok.Data; import picocli.CommandLine.Option; @@ -18,26 +18,17 @@ @Option(names = "--xadd-id", description = "Field used for stream entry IDs", paramLabel = "") private String xaddId; - public AbstractRedisFlatMapWriter writer() { + public AbstractMapWriter writer() { if (xaddId == null) { if (xaddMaxlen == null) { - return new XaddMapWriter(); + return new XaddMapWriter(); } - XaddMaxlenMapWriter writer = new XaddMaxlenMapWriter(); - writer.setApproximateTrimming(xaddTrim); - writer.setMaxlen(xaddMaxlen); - return writer; + return new XaddMaxlenMapWriter().approximateTrimming(xaddTrim).maxlen(xaddMaxlen); } if (xaddMaxlen == null) { - XaddIdMapWriter writer = new XaddIdMapWriter(); - writer.setIdField(xaddId); - return writer; + return new XaddIdMapWriter().idField(xaddId); } - XaddIdMaxlenMapWriter writer = new XaddIdMaxlenMapWriter(); - writer.setApproximateTrimming(xaddTrim); - writer.setIdField(xaddId); - writer.setMaxlen(xaddMaxlen); - return writer; + return new XaddIdMaxlenMapWriter().approximateTrimming(xaddTrim).idField(xaddId).maxlen(xaddMaxlen); } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/StringCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/StringCommandOptions.java index 9e5c45b7c..c334bce29 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/StringCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/StringCommandOptions.java @@ -3,9 +3,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.redislabs.riot.batch.redis.writer.SetFieldMapWriter; -import com.redislabs.riot.batch.redis.writer.SetMapWriter; -import com.redislabs.riot.batch.redis.writer.SetObjectMapWriter; +import com.redislabs.riot.batch.redis.map.SetFieldMapWriter; +import com.redislabs.riot.batch.redis.map.SetMapWriter; +import com.redislabs.riot.batch.redis.map.SetObjectMapWriter; import lombok.Data; import picocli.CommandLine.Option; @@ -19,20 +19,14 @@ @Option(names = "--string-value", description = "Value field for raw format", paramLabel = "") private String value; - public SetMapWriter writer() { + public SetMapWriter writer() { switch (format) { case raw: - SetFieldMapWriter fieldWriter = new SetFieldMapWriter(); - fieldWriter.setField(value); - return fieldWriter; + return new SetFieldMapWriter().field(value); case xml: - SetObjectMapWriter xmlWriter = new SetObjectMapWriter(); - xmlWriter.setObjectWriter(objectWriter(new XmlMapper())); - return xmlWriter; + return new SetObjectMapWriter().objectWriter(objectWriter(new XmlMapper())); default: - SetObjectMapWriter jsonWriter = new SetObjectMapWriter(); - jsonWriter.setObjectWriter(objectWriter(new ObjectMapper())); - return jsonWriter; + return new SetObjectMapWriter().objectWriter(objectWriter(new ObjectMapper())); } } diff --git a/src/main/java/com/redislabs/riot/cli/redis/ZsetCommandOptions.java b/src/main/java/com/redislabs/riot/cli/redis/ZsetCommandOptions.java index 41a950c7d..630aef0a4 100644 --- a/src/main/java/com/redislabs/riot/cli/redis/ZsetCommandOptions.java +++ b/src/main/java/com/redislabs/riot/cli/redis/ZsetCommandOptions.java @@ -1,6 +1,6 @@ package com.redislabs.riot.cli.redis; -import com.redislabs.riot.batch.redis.writer.ZaddMapWriter; +import com.redislabs.riot.batch.redis.map.ZaddMapWriter; import lombok.Data; import picocli.CommandLine.Option; @@ -12,10 +12,7 @@ @Option(names = "--zset-default", description = "Score when field not present (default: ${DEFAULT-VALUE})", paramLabel = "") private double zsetDefaultScore = 1d; - public ZaddMapWriter writer() { - ZaddMapWriter writer = new ZaddMapWriter(); - writer.setDefaultScore(zsetDefaultScore); - writer.setScoreField(zsetScore); - return writer; + public ZaddMapWriter writer() { + return new ZaddMapWriter().defaultScore(zsetDefaultScore).scoreField(zsetScore); } } diff --git a/src/main/java/com/redislabs/riot/cli/test/TestCommand.java b/src/main/java/com/redislabs/riot/cli/test/TestCommand.java index 2af0bc986..6c51a346c 100644 --- a/src/main/java/com/redislabs/riot/cli/test/TestCommand.java +++ b/src/main/java/com/redislabs/riot/cli/test/TestCommand.java @@ -2,8 +2,8 @@ import java.util.concurrent.TimeUnit; +import com.redislabs.picocliredis.RedisOptions; import com.redislabs.riot.cli.AbstractCommand; -import com.redislabs.riot.cli.redis.RedisConnectionOptions; import com.redislabs.riot.test.InfoTest; import com.redislabs.riot.test.LatencyTest; import com.redislabs.riot.test.PingTest; @@ -31,7 +31,7 @@ public class TestCommand extends AbstractCommand { private boolean latencyDistribution; @Override - public void execute(String name, RedisConnectionOptions redis) { + public void execute(String name, RedisOptions redis) { RedisTest test = test(); try { if (redis.isJedis()) { diff --git a/src/test/resources/commands/file-import-csv-processor-search-geo.txt b/src/test/resources/commands/file-import-csv-processor-search-geo.txt index 894e66afb..b0f1f2b4f 100644 --- a/src/test/resources/commands/file-import-csv-processor-search-geo.txt +++ b/src/test/resources/commands/file-import-csv-processor-search-geo.txt @@ -1 +1 @@ -$ riot file-import --file https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --index airports -k AirportID --filetype csv --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source --proc "Location=#geo(Longitude,Latitude)" \ No newline at end of file +$ riot file-import --file https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --command ftadd --index airports -k AirportID --filetype csv --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source --proc "Location=#geo(Longitude,Latitude)" \ No newline at end of file diff --git a/src/test/resources/commands/file-import-csv-processor-search.txt b/src/test/resources/commands/file-import-csv-processor-search.txt index 4819a8e13..f7c085cb0 100644 --- a/src/test/resources/commands/file-import-csv-processor-search.txt +++ b/src/test/resources/commands/file-import-csv-processor-search.txt @@ -1 +1 @@ -$ riot file-import --file "https://data.lacity.org/api/views/rx9t-fp7k/rows.csv?accessType=DOWNLOAD" --index laevents --keys Id --header --regex 'Event Location'="\((?.+),\s+(?.+)\)" --proc location=#geo(lon,lat) \ No newline at end of file +$ riot file-import --file "https://data.lacity.org/api/views/rx9t-fp7k/rows.csv?accessType=DOWNLOAD" --command ftadd --index laevents --keys Id --header --regex 'Event Location'="\((?.+),\s+(?.+)\)" --proc location=#geo(lon,lat) \ No newline at end of file diff --git a/src/test/resources/commands/file-import-csv-search.txt b/src/test/resources/commands/file-import-csv-search.txt index a5640c706..77cfafb72 100644 --- a/src/test/resources/commands/file-import-csv-search.txt +++ b/src/test/resources/commands/file-import-csv-search.txt @@ -1 +1 @@ -$ riot file-import --include 1 3 4 5 6 --index beers --keys id --header --file https://git.io/fjxPs \ No newline at end of file +$ riot file-import --include 1 3 4 5 6 --command ftadd --index beers --keys id --header --file https://git.io/fjxPs \ No newline at end of file