From 387cef5389c01adf857ca5e14093391ceb0f0983 Mon Sep 17 00:00:00 2001 From: zhoney Date: Wed, 22 May 2019 15:19:38 +0800 Subject: [PATCH] Support PostgreSQL and CockroachDB backends (#484) implemented: #441 Change-Id: I979affd9f8b2916e6696672fc6e3fda75f9b76da --- .travis.yml | 1 + .../store/cassandra/CassandraTable.java | 18 +- .../backend/serializer/TableSerializer.java | 47 +++- .../hugegraph/backend/store/TableDefine.java | 18 +- hugegraph-dist/pom.xml | 5 + .../assembly/static/conf/hugegraph.properties | 3 +- .../src/assembly/travis/install-backend.sh | 2 + .../src/assembly/travis/install-postgresql.sh | 18 ++ .../src/assembly/travis/start-server.sh | 16 +- .../baidu/hugegraph/dist/RegisterUtil.java | 21 +- .../src/main/resources/backend.properties | 2 +- .../backend/store/mysql/MysqlOptions.java | 10 +- .../backend/store/mysql/MysqlSerializer.java | 5 + .../backend/store/mysql/MysqlSessions.java | 9 +- .../backend/store/mysql/MysqlStore.java | 3 +- .../backend/store/mysql/MysqlTable.java | 65 +++--- .../backend/store/mysql/MysqlTables.java | 128 +++++++---- hugegraph-postgresql/pom.xml | 31 +++ .../store/postgresql/PostgresqlOptions.java | 66 ++++++ .../postgresql/PostgresqlSerializer.java | 75 +++++++ .../store/postgresql/PostgresqlSessions.java | 72 +++++++ .../store/postgresql/PostgresqlStore.java | 37 ++++ .../postgresql/PostgresqlStoreProvider.java | 139 ++++++++++++ .../store/postgresql/PostgresqlTable.java | 132 ++++++++++++ .../store/postgresql/PostgresqlTables.java | 203 ++++++++++++++++++ hugegraph-test/pom.xml | 15 ++ .../baidu/hugegraph/api/MetricsApiTest.java | 1 + .../com/baidu/hugegraph/api/TaskApiTest.java | 2 +- .../baidu/hugegraph/core/EdgeCoreTest.java | 13 +- .../baidu/hugegraph/core/VertexCoreTest.java | 9 +- .../src/main/resources/hugegraph.properties | 1 + pom.xml | 1 + 32 files changed, 1055 insertions(+), 113 deletions(-) create mode 100755 hugegraph-dist/src/assembly/travis/install-postgresql.sh create mode 100644 hugegraph-postgresql/pom.xml create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java create mode 100644 hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java diff --git a/.travis.yml b/.travis.yml index 4f7be19fc7..a2c687549b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,6 +80,7 @@ env: - BACKEND=mysql - BACKEND=hbase - BACKEND=rocksdb + - BACKEND=postgresql global: - RELEASE_BRANCH=^release-.*$ - RELEASE_TAG=^v[0-9]\..*$ diff --git a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java index 646eb6b449..7aadff7bd1 100644 --- a/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java +++ b/hugegraph-cassandra/src/main/java/com/baidu/hugegraph/backend/store/cassandra/CassandraTable.java @@ -266,9 +266,6 @@ protected Clause relation2Cql(Relation relation) { String key = relation.serialKey().toString(); Object value = relation.serialValue(); - // Serialize value (TODO: should move to Serializer) - value = serializeValue(value); - switch (relation.relation()) { case EQ: return QueryBuilder.eq(key, value); @@ -281,12 +278,7 @@ protected Clause relation2Cql(Relation relation) { case LTE: return QueryBuilder.lte(key, value); case IN: - List values = (List) value; - List serializedValues = new ArrayList<>(values.size()); - for (Object v : values) { - serializedValues.add(serializeValue(v)); - } - return QueryBuilder.in(key, serializedValues); + return QueryBuilder.in(key, value); case CONTAINS: return QueryBuilder.contains(key, value); case CONTAINS_KEY: @@ -331,14 +323,6 @@ protected static Select cloneSelect(Select select, String table) { return CopyUtil.copy(select, QueryBuilder.select().from(table)); } - protected static Object serializeValue(Object value) { - // Serialize value (TODO: should move to Serializer) - if (value instanceof Id) { - value = ((Id) value).asObject(); - } - return value; - } - protected Iterator results2Entries(Query q, ResultSet r) { return new CassandraEntryIterator(r, q, (e1, row) -> { CassandraBackendEntry e2 = row2Entry(q.resultType(), row); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java index 63a5047b12..0115a91465 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/serializer/TableSerializer.java @@ -19,7 +19,9 @@ package com.baidu.hugegraph.backend.serializer; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Set; import com.baidu.hugegraph.HugeGraph; @@ -339,7 +341,8 @@ protected Query writeQueryEdgeCondition(Query query) { if (r.key() == HugeKeys.OWNER_VERTEX || r.key() == HugeKeys.OTHER_VERTEX) { // Serialize vertex id - r.serialValue(IdUtil.writeString((Id) value)); + String id = IdUtil.writeString((Id) value); + r.serialValue(this.escapeString(id)); } else { // Serialize label id r.serialValue(((Id) value).asObject()); @@ -353,16 +356,30 @@ protected Query writeQueryEdgeCondition(Query query) { @Override protected Query writeQueryCondition(Query query) { - if (query.resultType().isGraph()) { - ConditionQuery result = (ConditionQuery) query; - // No user-prop when serialize - assert result.allSysprop(); - for (Condition.Relation r : result.relations()) { - if (r.relation() == Condition.RelationType.CONTAINS) { - r.serialValue(JsonUtil.toJson(r.value())); + ConditionQuery result = (ConditionQuery) query; + // No user-prop when serialize + assert result.allSysprop(); + for (Condition.Relation r : result.relations()) { + if (!(r.value().equals(r.serialValue()))) { + continue; + } + if (r.relation() == Condition.RelationType.IN) { + List values = (List) r.value(); + List serializedValues = new ArrayList<>(values.size()); + for (Object v : values) { + serializedValues.add(this.serializeValue(v)); } + r.serialValue(serializedValues); + } else { + r.serialValue(this.serializeValue(r.value())); + } + + if (query.resultType().isGraph() && + r.relation() == Condition.RelationType.CONTAINS) { + r.serialValue(JsonUtil.toJson(r.serialValue())); } } + return query; } @@ -577,6 +594,20 @@ protected abstract void formatProperties(HugeElement element, protected abstract void parseProperties(HugeElement element, TableBackendEntry.Row row); + protected Object serializeValue(Object value) { + if (value instanceof Id) { + value = ((Id) value).asObject(); + } + if (value instanceof String) { + value = this.escapeString((String) value); + } + return value; + } + + protected String escapeString(String value) { + return value; + } + protected void writeEnableLabelIndex(SchemaLabel schema, TableBackendEntry entry) { entry.column(HugeKeys.ENABLE_LABEL_INDEX, schema.enableLabelIndex()); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java index 3dd4358e22..ee952bfa64 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/TableDefine.java @@ -28,21 +28,37 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.InsertionOrderUtil; +import com.google.common.collect.ImmutableMap; public class TableDefine { private final Map columns; private final List keys; + private final Map typesMapping; public TableDefine() { this.columns = InsertionOrderUtil.newMap(); this.keys = InsertionOrderUtil.newList(); + this.typesMapping = ImmutableMap.of(); + } + + public TableDefine(Map typesMapping) { + this.columns = InsertionOrderUtil.newMap(); + this.keys = InsertionOrderUtil.newList(); + this.typesMapping = typesMapping; } public TableDefine column(HugeKeys key, String... desc) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < desc.length; i++) { - sb.append(desc[i]); + String type = desc[i]; + // The first element of 'desc' is column data type, which may be + // mapped to actual data type supported by backend store + if (i == 0 && this.typesMapping.containsKey(type)) { + type = this.typesMapping.get(type); + } + assert type != null; + sb.append(type); if (i != desc.length - 1) { sb.append(" "); } diff --git a/hugegraph-dist/pom.xml b/hugegraph-dist/pom.xml index eb08e872ec..00d583277b 100644 --- a/hugegraph-dist/pom.xml +++ b/hugegraph-dist/pom.xml @@ -59,6 +59,11 @@ hugegraph-hbase ${project.version} + + com.baidu.hugegraph + hugegraph-postgresql + ${project.version} + io.airlift airline diff --git a/hugegraph-dist/src/assembly/static/conf/hugegraph.properties b/hugegraph-dist/src/assembly/static/conf/hugegraph.properties index fe0729034c..5a83ffd30b 100644 --- a/hugegraph-dist/src/assembly/static/conf/hugegraph.properties +++ b/hugegraph-dist/src/assembly/static/conf/hugegraph.properties @@ -33,12 +33,13 @@ cassandra.password= # mysql backend config +#jdbc.driver=com.mysql.jdbc.Driver #jdbc.url=jdbc:mysql://127.0.0.1:3306 #jdbc.username=root #jdbc.password= #jdbc.reconnect_max_times=3 #jdbc.reconnect_interval=3 - +#jdbc.sslmode=disable # palo backend config #palo.host=127.0.0.1 diff --git a/hugegraph-dist/src/assembly/travis/install-backend.sh b/hugegraph-dist/src/assembly/travis/install-backend.sh index 7349b07649..6fdd97ce9f 100755 --- a/hugegraph-dist/src/assembly/travis/install-backend.sh +++ b/hugegraph-dist/src/assembly/travis/install-backend.sh @@ -10,4 +10,6 @@ elif [ "$BACKEND" == "hbase" ]; then $TRAVIS_DIR/install-hbase.sh elif [ "$BACKEND" == "mysql" ]; then $TRAVIS_DIR/install-mysql.sh +elif [ "$BACKEND" == "postgresql" ]; then + $TRAVIS_DIR/install-postgresql.sh fi diff --git a/hugegraph-dist/src/assembly/travis/install-postgresql.sh b/hugegraph-dist/src/assembly/travis/install-postgresql.sh new file mode 100755 index 0000000000..73f318e97a --- /dev/null +++ b/hugegraph-dist/src/assembly/travis/install-postgresql.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -ev + +TRAVIS_DIR=`dirname $0` +CONF=hugegraph-test/src/main/resources/hugegraph.properties + +POSTGRESQL_DRIVER=org.postgresql.Driver +POSTGRESQL_URL=jdbc:postgresql://localhost:5432/ +POSTGRESQL_USERNAME=postgres + +# Set PostgreSQL configurations +sed -i "s/jdbc.driver=.*/jdbc.driver=$POSTGRESQL_DRIVER/" $CONF +sed -i "s?jdbc.url=.*?jdbc.url=$POSTGRESQL_URL?" $CONF +sed -i "s/jdbc.username=.*/jdbc.username=$POSTGRESQL_USERNAME/" $CONF + +sudo service postgresql stop 9.2 +sudo service postgresql start 9.5 diff --git a/hugegraph-dist/src/assembly/travis/start-server.sh b/hugegraph-dist/src/assembly/travis/start-server.sh index 873151d722..6bf0fd8cfe 100755 --- a/hugegraph-dist/src/assembly/travis/start-server.sh +++ b/hugegraph-dist/src/assembly/travis/start-server.sh @@ -7,15 +7,29 @@ BASE_DIR=hugegraph-$VERSION BIN=$BASE_DIR/bin CONF=$BASE_DIR/conf/hugegraph.properties +# PostgreSQL configurations +POSTGRESQL_DRIVER=org.postgresql.Driver +POSTGRESQL_URL=jdbc:postgresql://localhost:5432/ +POSTGRESQL_USERNAME=postgres + declare -A backend_serializer_map=(["memory"]="text" ["cassandra"]="cassandra" \ ["scylladb"]="scylladb" ["mysql"]="mysql" \ - ["hbase"]="hbase" ["rocksdb"]="binary") + ["hbase"]="hbase" ["rocksdb"]="binary" \ + ["postgresql"]="postgresql") SERIALIZER=${backend_serializer_map[$BACKEND]} +# Set backend and serializer sed -i "s/backend=.*/backend=$BACKEND/" $CONF sed -i "s/serializer=.*/serializer=$SERIALIZER/" $CONF +# Set PostgreSQL configurations if needed +if [ "$BACKEND" == "postgresql" ]; then + sed -i "s/#jdbc.driver=.*/jdbc.driver=$POSTGRESQL_DRIVER/" $CONF + sed -i "s?#jdbc.url=.*?jdbc.url=$POSTGRESQL_URL?" $CONF + sed -i "s/#jdbc.username=.*/jdbc.username=$POSTGRESQL_USERNAME/" $CONF +fi + # Append schema.sync_deletion=true to config file echo "schema.sync_deletion=true" >> $CONF diff --git a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java index a93746b829..c789fa7c1c 100644 --- a/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java +++ b/hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java @@ -52,7 +52,8 @@ public static void registerBackends() { String confFile = "/backend.properties"; InputStream input = RegisterUtil.class.getClass() .getResourceAsStream(confFile); - E.checkState(input != null, "Can't read file '%s' as stream", confFile); + E.checkState(input != null, + "Can't read file '%s' as stream", confFile); PropertiesConfiguration props = new PropertiesConfiguration(); props.setDelimiterParsingDisabled(true); @@ -89,8 +90,12 @@ private static void registerBackend(String backend) { case "palo": registerPalo(); break; + case "postgresql": + registerPostgresql(); + break; default: - throw new HugeException("Unsupported backend type '%s'", backend); + throw new HugeException("Unsupported backend type '%s'", + backend); } } @@ -165,6 +170,18 @@ public static void registerPalo() { "com.baidu.hugegraph.backend.store.palo.PaloStoreProvider"); } + public static void registerPostgresql() { + // Register config + OptionSpace.register("postgresql", + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlOptions"); + // Register serializer + SerializerFactory.register("postgresql", + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlSerializer"); + // Register backend + BackendProviderFactory.register("postgresql", + "com.baidu.hugegraph.backend.store.postgresql.PostgresqlStoreProvider"); + } + public static void registerServer() { OptionSpace.register("server", "com.baidu.hugegraph.config.ServerOptions"); } diff --git a/hugegraph-dist/src/main/resources/backend.properties b/hugegraph-dist/src/main/resources/backend.properties index 98c446a84d..45f95d2f1c 100644 --- a/hugegraph-dist/src/main/resources/backend.properties +++ b/hugegraph-dist/src/main/resources/backend.properties @@ -1 +1 @@ -backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase] +backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase, postgresql] diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java index c6bd854864..240b7e7deb 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlOptions.java @@ -27,7 +27,7 @@ public class MysqlOptions extends OptionHolder { - private MysqlOptions() { + protected MysqlOptions() { super(); } @@ -89,4 +89,12 @@ public static synchronized MysqlOptions instance() { rangeInt(1, 10), 3 ); + + public static final ConfigOption SSL_MODE = + new ConfigOption<>( + "jdbc.ssl_mode", + "The SSL mode of connections with database.", + disallowEmpty(), + "disable" + ); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java index 6ad3012d6e..cda08f86e0 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSerializer.java @@ -162,4 +162,9 @@ protected void readUserdata(SchemaElement schema, schema.userdata(e.getKey(), e.getValue()); } } + + @Override + protected String escapeString(String value) { + return MysqlUtil.escapeString(value); + } } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java index 6749596efa..6a87c9eb9d 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java @@ -147,8 +147,11 @@ public void createDatabase() { try (Connection conn = this.openWithoutDB(0)) { conn.createStatement().execute(sql); } catch (SQLException e) { - throw new BackendException("Failed to create database '%s'", - this.database); + if (!e.getMessage().endsWith("already exists")) { + throw new BackendException("Failed to create database '%s'", e, + this.database); + } + // Ignore exception if database already exists } } @@ -194,7 +197,7 @@ public boolean existsDatabase() { /** * Connect DB without specified database */ - private Connection openWithoutDB(int timeout) { + protected Connection openWithoutDB(int timeout) { String jdbcUrl = this.config.get(MysqlOptions.JDBC_URL); String url = new URIBuilder().setPath(jdbcUrl) .setParameter("socketTimeout", diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java index da6e62867c..3c39ddbbc6 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java @@ -114,7 +114,8 @@ public synchronized void open(HugeConfig config) { try { this.sessions.open(config); } catch (Exception e) { - if (!e.getMessage().startsWith("Unknown database")) { + if (!e.getMessage().startsWith("Unknown database") && + !e.getMessage().endsWith("does not exist")) { throw new BackendException("Failed connect with mysql, " + "please ensure it's ok", e); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java index 716e71d3a2..9eb5629d5d 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Set; +import org.apache.logging.log4j.util.Strings; import org.slf4j.Logger; import com.baidu.hugegraph.backend.BackendException; @@ -91,14 +92,16 @@ protected void createTable(Session session, TableDefine tableDefine) { // Specified primary keys sql.append(" PRIMARY KEY ("); int i = 0; + int size = tableDefine.keys().size(); for (HugeKeys key : tableDefine.keys()) { - sql.append(key); - if (++i != tableDefine.keys().size()) { + sql.append(formatKey(key)); + if (++i != size) { sql.append(", "); } } - - sql.append(")) ENGINE=InnoDB;"); + sql.append("))"); + sql.append(this.engine()); + sql.append(";"); LOG.debug("Create table: {}", sql); try { @@ -109,9 +112,13 @@ protected void createTable(Session session, TableDefine tableDefine) { } } + protected String engine() { + return " ENGINE=InnoDB"; + } + protected void dropTable(Session session) { LOG.debug("Drop table: {}", this.table()); - String sql = String.format("DROP TABLE IF EXISTS %s;", this.table()); + String sql = this.buildDropTemplate(); try { session.execute(sql); } catch (SQLException e) { @@ -122,7 +129,7 @@ protected void dropTable(Session session) { protected void truncateTable(Session session) { LOG.debug("Truncate table: {}", this.table()); - String sql = String.format("TRUNCATE TABLE %s;", this.table()); + String sql = this.buildTruncateTemplate(); try { session.execute(sql); } catch (SQLException e) { @@ -190,6 +197,14 @@ protected String buildDeleteTemplate(List idNames) { return this.deleteTemplate; } + protected String buildDropTemplate() { + return String.format("DROP TABLE IF EXISTS %s;", this.table()); + } + + protected String buildTruncateTemplate() { + return String.format("TRUNCATE TABLE %s;", this.table()); + } + /** * Insert an entire row */ @@ -202,7 +217,7 @@ public void insert(Session session, MysqlBackendEntry.Row entry) { // Create or get insert prepare statement insertStmt = session.prepareStatement(template); int i = 1; - for (Object object : entry.columns().values()) { + for (Object object : this.buildInsertObjects(entry)) { insertStmt.setObject(i++, object); } } catch (SQLException e) { @@ -412,9 +427,6 @@ protected StringBuilder relation2Sql(Condition.Relation relation) { String key = relation.serialKey().toString(); Object value = relation.serialValue(); - // Serialize value (TODO: should move to Serializer) - value = serializeValue(value); - StringBuilder sql = new StringBuilder(32); sql.append(key); switch (relation.relation()) { @@ -438,13 +450,8 @@ protected StringBuilder relation2Sql(Condition.Relation relation) { break; case IN: sql.append(" IN ("); - List values = (List) value; - for (int i = 0, n = values.size(); i < n; i++) { - sql.append(serializeValue(values.get(i))); - if (i != n - 1) { - sql.append(", "); - } - } + String values = Strings.join((List) value, ','); + sql.append(values); sql.append(")"); break; case CONTAINS: @@ -503,6 +510,8 @@ protected void wrapPage(StringBuilder select, Query query) { select.append(where.build()); } + select.append(this.orderByKeys()); + assert query.limit() != Query.NO_LIMIT; // Fetch `limit + 1` records for judging whether reached the last page select.append(" limit "); @@ -510,6 +519,10 @@ protected void wrapPage(StringBuilder select, Query query) { select.append(";"); } + protected String orderByKeys() { + return Strings.EMPTY; + } + protected void wrapOffset(StringBuilder select, Query query) { assert query.limit() >= 0; assert query.offset() >= 0; @@ -521,16 +534,6 @@ protected void wrapOffset(StringBuilder select, Query query) { select.append(";"); } - private static Object serializeValue(Object value) { - if (value instanceof Id) { - value = ((Id) value).asObject(); - } - if (value instanceof String) { - value = MysqlUtil.escapeString((String) value); - } - return value; - } - protected Iterator results2Entries(Query query, ResultSet results) { return new MysqlEntryIterator(results, query, this::mergeEntries); @@ -545,6 +548,14 @@ protected void appendPartition(StringBuilder delete) { // pass } + protected List buildInsertObjects(MysqlBackendEntry.Row entry) { + List objects = new ArrayList<>(); + for (Object key : entry.columns().keySet()) { + objects.add(entry.columns().get(key)); + } + return objects; + } + public static String formatKey(HugeKeys key) { return key.name(); } diff --git a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java index c929bd4c4a..faf083a0c4 100644 --- a/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java +++ b/hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.id.EdgeId; @@ -40,20 +41,30 @@ import com.baidu.hugegraph.type.define.HugeKeys; import com.baidu.hugegraph.util.E; +import jersey.repackaged.com.google.common.collect.ImmutableMap; + public class MysqlTables { - private static final String INT = "INT"; + public static final String BOOLEAN = "BOOLEAN"; + public static final String TINYINT = "TINYINT"; + public static final String INT = "INT"; + public static final String DOUBLE = "DOUBLE"; + public static final String SMALL_TEXT = "SMALL_TEXT"; + public static final String MID_TEXT = "MID_TEXT"; + public static final String LARGE_TEXT = "LARGE_TEXT"; + + private static final String DATATYPE_PK = INT; + private static final String DATATYPE_SL = INT; // VL/EL + private static final String DATATYPE_IL = INT; - private static final String DATATYPE_PK = "INT"; - private static final String DATATYPE_SL = "INT"; // VL/EL - private static final String DATATYPE_IL = "INT"; + private static final String SMALL_JSON = MID_TEXT; + private static final String LARGE_JSON = LARGE_TEXT; - private static final String BOOLEAN = "BOOLEAN"; - private static final String TINYINT = "TINYINT"; - private static final String DOUBLE = "DOUBLE"; - private static final String VARCHAR = "VARCHAR(255)"; - private static final String SMALL_JSON = "VARCHAR(1024)"; - private static final String LARGE_JSON = "TEXT"; + private static final Map TYPES_MAPPING = ImmutableMap.of( + SMALL_TEXT, "VARCHAR(255)", + MID_TEXT, "VARCHAR(1024)", + LARGE_TEXT, "TEXT" + ); public static class MysqlTableTemplate extends MysqlTable { @@ -74,10 +85,14 @@ public static class Counters extends MysqlTableTemplate { public static final String TABLE = "counters"; public Counters() { + this(TYPES_MAPPING); + } + + public Counters(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); - this.define.column(HugeKeys.SCHEMA_TYPE, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.SCHEMA_TYPE, SMALL_TEXT); this.define.column(HugeKeys.ID, INT); this.define.keys(HugeKeys.SCHEMA_TYPE); } @@ -106,8 +121,8 @@ public void increaseCounter(Session session, HugeType type, long increment) { String update = String.format( "INSERT INTO %s VALUES ('%s', %s) " + - "ON DUPLICATE KEY UPDATE " + - "ID = ID + 1;", TABLE, type.name(), increment); + "ON DUPLICATE KEY UPDATE ID = ID + %s;", + TABLE, type.name(), increment, increment); try { session.execute(update); } catch (SQLException e) { @@ -122,11 +137,15 @@ public static class VertexLabel extends MysqlTableTemplate { public static final String TABLE = "vertex_labels"; public VertexLabel() { + this(TYPES_MAPPING); + } + + public VertexLabel(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_SL); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.ID_STRATEGY, TINYINT); this.define.column(HugeKeys.PRIMARY_KEYS, SMALL_JSON); this.define.column(HugeKeys.PROPERTIES, SMALL_JSON); @@ -144,11 +163,15 @@ public static class EdgeLabel extends MysqlTableTemplate { public static final String TABLE = "edge_labels"; public EdgeLabel() { + this(TYPES_MAPPING); + } + + public EdgeLabel(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_SL); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.FREQUENCY, TINYINT); this.define.column(HugeKeys.SOURCE_LABEL, DATATYPE_SL); this.define.column(HugeKeys.TARGET_LABEL, DATATYPE_SL); @@ -168,11 +191,15 @@ public static class PropertyKey extends MysqlTableTemplate { public static final String TABLE = "property_keys"; public PropertyKey() { + this(TYPES_MAPPING); + } + + public PropertyKey(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_PK); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.DATA_TYPE, TINYINT); this.define.column(HugeKeys.CARDINALITY, TINYINT); this.define.column(HugeKeys.PROPERTIES, SMALL_JSON); @@ -187,11 +214,15 @@ public static class IndexLabel extends MysqlTableTemplate { public static final String TABLE = "index_labels"; public IndexLabel() { + this(TYPES_MAPPING); + } + + public IndexLabel(Map typesMapping) { super(TABLE); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.ID, DATATYPE_IL); - this.define.column(HugeKeys.NAME, VARCHAR); + this.define.column(HugeKeys.NAME, SMALL_TEXT); this.define.column(HugeKeys.BASE_TYPE, TINYINT); this.define.column(HugeKeys.BASE_VALUE, DATATYPE_SL); this.define.column(HugeKeys.INDEX_TYPE, TINYINT); @@ -206,10 +237,14 @@ public static class Vertex extends MysqlTableTemplate { public static final String TABLE = "vertices"; public Vertex(String store) { + this(store, TYPES_MAPPING); + } + + public Vertex(String store, Map typesMapping) { super(joinTableName(store, TABLE)); - this.define = new TableDefine(); - this.define.column(HugeKeys.ID, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.ID, SMALL_TEXT); this.define.column(HugeKeys.LABEL, DATATYPE_SL); this.define.column(HugeKeys.PROPERTIES, LARGE_JSON); this.define.keys(HugeKeys.ID); @@ -223,7 +258,12 @@ public static class Edge extends MysqlTableTemplate { private final Directions direction; private final String delByLabelTemplate; - protected Edge(String store, Directions direction) { + public Edge(String store, Directions direction) { + this(store, direction, TYPES_MAPPING); + } + + public Edge(String store, Directions direction, + Map typesMapping) { super(joinTableName(store, table(direction))); this.direction = direction; @@ -231,12 +271,12 @@ protected Edge(String store, Directions direction) { "DELETE FROM %s WHERE %s = ?;", this.table(), formatKey(HugeKeys.LABEL)); - this.define = new TableDefine(); - this.define.column(HugeKeys.OWNER_VERTEX, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.OWNER_VERTEX, SMALL_TEXT); this.define.column(HugeKeys.DIRECTION, TINYINT); this.define.column(HugeKeys.LABEL, DATATYPE_SL); - this.define.column(HugeKeys.SORT_VALUES, VARCHAR); - this.define.column(HugeKeys.OTHER_VERTEX, VARCHAR); + this.define.column(HugeKeys.SORT_VALUES, SMALL_TEXT); + this.define.column(HugeKeys.OTHER_VERTEX, SMALL_TEXT); this.define.column(HugeKeys.PROPERTIES, LARGE_JSON); this.define.keys(HugeKeys.OWNER_VERTEX, HugeKeys.DIRECTION, HugeKeys.LABEL, HugeKeys.SORT_VALUES, @@ -244,7 +284,7 @@ protected Edge(String store, Directions direction) { } @Override - protected List idColumnValue(Id id) { + public List idColumnValue(Id id) { EdgeId edgeId; if (!(id instanceof EdgeId)) { String[] idParts = EdgeId.split(id); @@ -299,7 +339,7 @@ private void deleteEdgesByLabel(Session session, Id label) { } @Override - protected BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { + public BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { // Merge edges into vertex // TODO: merge rows before calling row2Entry() @@ -366,22 +406,24 @@ public static class SecondaryIndex extends Index { public static final String TABLE = "secondary_indexes"; public SecondaryIndex(String store) { - this(store, TABLE); + this(store, TABLE, TYPES_MAPPING); } - public SecondaryIndex(String store, String table) { + + public SecondaryIndex(String store, String table, + Map typesMapping) { super(joinTableName(store, table)); - this.define = new TableDefine(); - this.define.column(HugeKeys.FIELD_VALUES, VARCHAR); + this.define = new TableDefine(typesMapping); + this.define.column(HugeKeys.FIELD_VALUES, SMALL_TEXT); this.define.column(HugeKeys.INDEX_LABEL_ID, DATATYPE_IL); - this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR); + this.define.column(HugeKeys.ELEMENT_IDS, SMALL_TEXT); this.define.keys(HugeKeys.FIELD_VALUES, HugeKeys.INDEX_LABEL_ID, HugeKeys.ELEMENT_IDS); } @Override - protected final String entryId(MysqlBackendEntry entry) { + public final String entryId(MysqlBackendEntry entry) { String fieldValues = entry.column(HugeKeys.FIELD_VALUES); Integer labelId = entry.column(HugeKeys.INDEX_LABEL_ID); return SplicingIdGenerator.concat(fieldValues, labelId.toString()); @@ -393,7 +435,7 @@ public static class SearchIndex extends SecondaryIndex { public static final String TABLE = "search_indexes"; public SearchIndex(String store) { - super(store, TABLE); + super(store, TABLE, TYPES_MAPPING); } } @@ -402,19 +444,23 @@ public static class RangeIndex extends Index { public static final String TABLE = "range_indexes"; public RangeIndex(String store) { + this(store, TYPES_MAPPING); + } + + public RangeIndex(String store, Map typesMapping) { super(joinTableName(store, TABLE)); - this.define = new TableDefine(); + this.define = new TableDefine(typesMapping); this.define.column(HugeKeys.INDEX_LABEL_ID, DATATYPE_IL); this.define.column(HugeKeys.FIELD_VALUES, DOUBLE); - this.define.column(HugeKeys.ELEMENT_IDS, VARCHAR); + this.define.column(HugeKeys.ELEMENT_IDS, SMALL_TEXT); this.define.keys(HugeKeys.INDEX_LABEL_ID, HugeKeys.FIELD_VALUES, HugeKeys.ELEMENT_IDS); } @Override - protected final String entryId(MysqlBackendEntry entry) { + public final String entryId(MysqlBackendEntry entry) { Double fieldValue = entry.column(HugeKeys.FIELD_VALUES); Integer labelId = entry.column(HugeKeys.INDEX_LABEL_ID); return SplicingIdGenerator.concat(labelId.toString(), diff --git a/hugegraph-postgresql/pom.xml b/hugegraph-postgresql/pom.xml new file mode 100644 index 0000000000..c9680e49f6 --- /dev/null +++ b/hugegraph-postgresql/pom.xml @@ -0,0 +1,31 @@ + + + + hugegraph + com.baidu.hugegraph + 0.10.0 + + 4.0.0 + + hugegraph-postgresql + + + + com.baidu.hugegraph + hugegraph-core + ${project.version} + + + com.baidu.hugegraph + hugegraph-mysql + ${project.version} + + + org.postgresql + postgresql + 42.1.4 + + + diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java new file mode 100644 index 0000000000..6d2d6b57dd --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import com.baidu.hugegraph.backend.store.mysql.MysqlOptions; +import com.baidu.hugegraph.config.ConfigOption; + +import static com.baidu.hugegraph.config.OptionChecker.disallowEmpty; + +public class PostgresqlOptions extends MysqlOptions { + + private PostgresqlOptions() { + super(); + } + + private static volatile PostgresqlOptions instance; + + public static synchronized PostgresqlOptions instance() { + if (instance == null) { + instance = new PostgresqlOptions(); + instance.registerOptions(); + } + return instance; + } + + public static final ConfigOption JDBC_DRIVER = + new ConfigOption<>( + "jdbc.driver", + "The JDBC driver class to connect database.", + disallowEmpty(), + "org.postgresql.Driver" + ); + + public static final ConfigOption JDBC_URL = + new ConfigOption<>( + "jdbc.url", + "The url of database in JDBC format.", + disallowEmpty(), + "jdbc:postgresql://127.0.0.1:5432/" + ); + + public static final ConfigOption JDBC_USERNAME = + new ConfigOption<>( + "jdbc.username", + "The username to login database.", + disallowEmpty(), + "postgres" + ); +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java new file mode 100644 index 0000000000..4a787e35c9 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.sql.SQLException; + +import org.apache.logging.log4j.util.Strings; +import org.postgresql.core.Utils; + +import com.baidu.hugegraph.HugeException; +import com.baidu.hugegraph.backend.id.IdUtil; +import com.baidu.hugegraph.backend.serializer.TableBackendEntry; +import com.baidu.hugegraph.backend.store.BackendEntry; +import com.baidu.hugegraph.backend.store.mysql.MysqlSerializer; +import com.baidu.hugegraph.structure.HugeIndex; +import com.baidu.hugegraph.type.define.HugeKeys; + +public class PostgresqlSerializer extends MysqlSerializer { + + @Override + public BackendEntry writeIndex(HugeIndex index) { + TableBackendEntry entry = newBackendEntry(index); + /* + * When field-values is null and elementIds size is 0, it is + * meaningful for deletion of index data in secondary/range index. + */ + if (index.fieldValues() == null && index.elementIds().size() == 0) { + entry.column(HugeKeys.INDEX_LABEL_ID, index.indexLabel().asLong()); + } else { + Object value = index.fieldValues(); + if (value != null && value.equals("\u0000")) { + value = Strings.EMPTY; + } + entry.column(HugeKeys.FIELD_VALUES, value); + entry.column(HugeKeys.INDEX_LABEL_ID, index.indexLabel().asLong()); + entry.column(HugeKeys.ELEMENT_IDS, + IdUtil.writeString(index.elementId())); + entry.subId(index.elementId()); + } + return entry; + } + + @Override + protected String escapeString(String value) { + if (value.equals("\u0000")) { + return "\'\'"; + } + StringBuilder builder = new StringBuilder(8 + value.length()); + builder.append('\''); + try { + Utils.escapeLiteral(builder, value, false); + } catch (SQLException e) { + throw new HugeException("Failed to escape '%s'", e, value); + } + builder.append('\''); + return builder.toString(); + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java new file mode 100644 index 0000000000..638d63c061 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java @@ -0,0 +1,72 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.sql.Connection; +import java.sql.SQLException; + +import org.postgresql.util.PSQLException; +import org.slf4j.Logger; + +import com.baidu.hugegraph.backend.BackendException; +import com.baidu.hugegraph.backend.store.mysql.MysqlSessions; +import com.baidu.hugegraph.backend.store.mysql.MysqlStore; +import com.baidu.hugegraph.config.HugeConfig; +import com.baidu.hugegraph.util.Log; + +public class PostgresqlSessions extends MysqlSessions { + + private static final Logger LOG = Log.logger(MysqlStore.class); + + private static final String COCKROACH_DB_CREATE = + "CREATE DATABASE %s ENCODING='UTF-8'"; + private static final String POSTGRESQL_DB_CREATE = COCKROACH_DB_CREATE + + " TEMPLATE=template0 LC_COLLATE='C' LC_CTYPE='C';"; + + public PostgresqlSessions(HugeConfig config, String database, String store) { + super(config, database, store); + } + + @Override + public void createDatabase() { + // Create database with non-database-session + LOG.debug("Create database: {}", this.database()); + + String sql = String.format(POSTGRESQL_DB_CREATE, this.database()); + try (Connection conn = this.openWithoutDB(0)) { + try { + conn.createStatement().execute(sql); + } catch (PSQLException e) { + // CockroackDB not support 'template' args of CREATE DATABASE + if (e.getMessage().contains("syntax error at or near " + + "\"template\"")) { + sql = String.format(COCKROACH_DB_CREATE, this.database()); + conn.createStatement().execute(sql); + } + } + } catch (SQLException e) { + if (!e.getMessage().endsWith("already exists")) { + throw new BackendException("Failed to create database '%s'", e, + this.database()); + } + // Ignore exception if database already exists + } + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java new file mode 100644 index 0000000000..5bd11786a8 --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStore.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import com.baidu.hugegraph.backend.store.BackendStoreProvider; +import com.baidu.hugegraph.backend.store.mysql.MysqlStore; +import com.baidu.hugegraph.config.HugeConfig; + +public abstract class PostgresqlStore extends MysqlStore { + + public PostgresqlStore(BackendStoreProvider provider, + String database, String name) { + super(provider, database, name); + } + + @Override + protected PostgresqlSessions openSessionPool(HugeConfig config) { + return new PostgresqlSessions(config, this.database(), this.store()); + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java new file mode 100644 index 0000000000..85f68e022e --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlStoreProvider.java @@ -0,0 +1,139 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.store.BackendStore; +import com.baidu.hugegraph.backend.store.BackendStoreProvider; +import com.baidu.hugegraph.backend.store.mysql.MysqlSessions; +import com.baidu.hugegraph.backend.store.mysql.MysqlStoreProvider; +import com.baidu.hugegraph.backend.store.mysql.MysqlTable; +import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.Directions; + +public class PostgresqlStoreProvider extends MysqlStoreProvider { + + @Override + protected BackendStore newSchemaStore(String store) { + return new PostgresqlSchemaStore(this, this.database(), store); + } + + @Override + protected BackendStore newGraphStore(String store) { + return new PostgresqlGraphStore(this, this.database(), store); + } + + @Override + public String type() { + return "postgresql"; + } + + @Override + public String version() { + return "1.0"; + } + + public static class PostgresqlSchemaStore extends PostgresqlStore { + + private final PostgresqlTables.Counters counters; + + public PostgresqlSchemaStore(BackendStoreProvider provider, + String database, String store) { + super(provider, database, store); + + this.counters = new PostgresqlTables.Counters(); + + registerTableManager(HugeType.VERTEX_LABEL, + new PostgresqlTables.VertexLabel()); + registerTableManager(HugeType.EDGE_LABEL, + new PostgresqlTables.EdgeLabel()); + registerTableManager(HugeType.PROPERTY_KEY, + new PostgresqlTables.PropertyKey()); + registerTableManager(HugeType.INDEX_LABEL, + new PostgresqlTables.IndexLabel()); + } + + @Override + protected Collection tables() { + List tables = new ArrayList<>(super.tables()); + tables.add(this.counters); + return tables; + } + + @Override + public void increaseCounter(HugeType type, long increment) { + this.checkSessionConnected(); + MysqlSessions.Session session = this.session(type); + this.counters.increaseCounter(session, type, increment); + } + + @Override + public long getCounter(HugeType type) { + this.checkSessionConnected(); + MysqlSessions.Session session = this.session(type); + return this.counters.getCounter(session, type); + } + } + + public static class PostgresqlGraphStore extends PostgresqlStore { + + public PostgresqlGraphStore(BackendStoreProvider provider, + String database, String store) { + super(provider, database, store); + + registerTableManager(HugeType.VERTEX, + new PostgresqlTables.Vertex(store)); + registerTableManager(HugeType.EDGE_OUT, + new PostgresqlTables.Edge(store, + Directions.OUT)); + registerTableManager(HugeType.EDGE_IN, + new PostgresqlTables.Edge(store, + Directions.IN)); + registerTableManager(HugeType.SECONDARY_INDEX, + new PostgresqlTables.SecondaryIndex(store)); + registerTableManager(HugeType.RANGE_INDEX, + new PostgresqlTables.RangeIndex(store)); + registerTableManager(HugeType.SEARCH_INDEX, + new PostgresqlTables.SearchIndex(store)); + } + + @Override + public Id nextId(HugeType type) { + throw new UnsupportedOperationException( + "PostgresqlGraphStore.nextId()"); + } + + @Override + public void increaseCounter(HugeType type, long increment) { + throw new UnsupportedOperationException( + "PostgresqlGraphStore.increaseCounter()"); + } + + @Override + public long getCounter(HugeType type) { + throw new UnsupportedOperationException( + "PostgresqlGraphStore.getCounter()"); + } + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java new file mode 100644 index 0000000000..112036085c --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTable.java @@ -0,0 +1,132 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.util.Strings; + +import com.baidu.hugegraph.backend.store.mysql.MysqlBackendEntry; +import com.baidu.hugegraph.backend.store.mysql.MysqlTable; +import com.baidu.hugegraph.type.define.HugeKeys; + +public abstract class PostgresqlTable extends MysqlTable { + + private String insertTemplate = null; + private String orderByKeys = null; + + public PostgresqlTable(String table) { + super(table); + } + + protected String buildDropTemplate() { + return String.format("DROP TABLE IF EXISTS %s CASCADE;", this.table()); + } + + protected String buildTruncateTemplate() { + return String.format("TRUNCATE TABLE %s CASCADE;", this.table()); + } + + @Override + protected String engine() { + return Strings.EMPTY; + } + + @Override + protected List buildInsertObjects(MysqlBackendEntry.Row entry) { + List objects = new ArrayList<>(); + objects.addAll(super.buildInsertObjects(entry)); + objects.addAll(super.buildInsertObjects(entry)); + return objects; + } + + @Override + protected String buildInsertTemplate(MysqlBackendEntry.Row entry) { + if (this.insertTemplate != null) { + return this.insertTemplate; + } + + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO ").append(this.table()).append(" ("); + + int i = 0; + int size = entry.columns().size(); + for (HugeKeys key : entry.columns().keySet()) { + insert.append(formatKey(key)); + if (++i != size) { + insert.append(", "); + } + } + insert.append(") VALUES ("); + + for (i = 0; i < size; i++) { + insert.append("?"); + if (i != size - 1) { + insert.append(", "); + } + } + insert.append(")"); + + i = 0; + size = this.tableDefine().keys().size(); + insert.append(" ON CONFLICT ("); + for (HugeKeys key : this.tableDefine().keys()) { + insert.append(formatKey(key)); + if (++i != size) { + insert.append(", "); + } + } + insert.append(")"); + + i = 0; + size = entry.columns().keySet().size(); + insert.append(" DO UPDATE SET "); + for (HugeKeys key : entry.columns().keySet()) { + insert.append(formatKey(key)).append(" = ?"); + if (++i != size) { + insert.append(", "); + } + } + + this.insertTemplate = insert.toString(); + return this.insertTemplate; + } + + // Set order-by to keep results order consistence for PostgreSQL result + protected String orderByKeys() { + if (this.orderByKeys != null) { + return this.orderByKeys; + } + int i = 0; + int size = this.tableDefine().keys().size(); + StringBuilder select = new StringBuilder(" ORDER BY "); + for (HugeKeys hugeKey : this.tableDefine().keys()) { + String key = formatKey(hugeKey); + select.append(key).append(" "); + select.append("ASC "); + if (++i != size) { + select.append(", "); + } + } + this.orderByKeys = select.toString(); + return this.orderByKeys; + } +} diff --git a/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java new file mode 100644 index 0000000000..b8514c330e --- /dev/null +++ b/hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlTables.java @@ -0,0 +1,203 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.backend.store.postgresql; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +import com.baidu.hugegraph.backend.BackendException; +import com.baidu.hugegraph.backend.id.Id; +import com.baidu.hugegraph.backend.store.BackendEntry; +import com.baidu.hugegraph.backend.store.TableDefine; +import com.baidu.hugegraph.backend.store.mysql.MysqlBackendEntry; +import com.baidu.hugegraph.backend.store.mysql.MysqlSessions.Session; +import com.baidu.hugegraph.backend.store.mysql.MysqlTables; +import com.baidu.hugegraph.backend.store.mysql.MysqlTables.MysqlTableTemplate; +import com.baidu.hugegraph.type.HugeType; +import com.baidu.hugegraph.type.define.Directions; +import com.baidu.hugegraph.type.define.HugeKeys; + +import jersey.repackaged.com.google.common.collect.ImmutableMap; + +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.BOOLEAN; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.DOUBLE; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.INT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.LARGE_TEXT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.MID_TEXT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.SMALL_TEXT; +import static com.baidu.hugegraph.backend.store.mysql.MysqlTables.TINYINT; + +public class PostgresqlTables { + + private static final Map TYPES_MAPPING = + ImmutableMap.builder() + .put(BOOLEAN, "BOOL") + .put(TINYINT, "INT") + .put(INT, "INT") + .put(DOUBLE, "FLOAT") + .put(SMALL_TEXT, "VARCHAR(255)") + .put(MID_TEXT, "VARCHAR(1024)") + .put(LARGE_TEXT, "VARCHAR(65533)") + .build(); + + public static class PostgresqlTableTemplate extends PostgresqlTable { + + protected MysqlTableTemplate template; + + public PostgresqlTableTemplate(MysqlTableTemplate template) { + super(template.table()); + this.template = template; + } + + @Override + public TableDefine tableDefine() { + return this.template.tableDefine(); + } + } + + public static class Counters extends PostgresqlTableTemplate { + + public Counters() { + super(new MysqlTables.Counters(TYPES_MAPPING)); + } + + public long getCounter(Session session, HugeType type) { + MysqlTables.Counters table = (MysqlTables.Counters) this.template; + return table.getCounter(session, type); + } + + public void increaseCounter(Session session, HugeType type, + long increment) { + String update = String.format( + "INSERT INTO %s (%s, %s) VALUES ('%s', %s) " + + "ON CONFLICT (%s) DO UPDATE SET ID = %s.ID + %s;", + this.table(), formatKey(HugeKeys.SCHEMA_TYPE), + formatKey(HugeKeys.ID), type.name(), increment, + formatKey(HugeKeys.SCHEMA_TYPE), + this.table(), increment); + try { + session.execute(update); + } catch (SQLException e) { + throw new BackendException( + "Failed to update counters with type '%s'", e, type); + } + } + } + + public static class VertexLabel extends PostgresqlTableTemplate { + + public VertexLabel() { + super(new MysqlTables.VertexLabel(TYPES_MAPPING)); + } + } + + public static class EdgeLabel extends PostgresqlTableTemplate { + + public EdgeLabel() { + super(new MysqlTables.EdgeLabel(TYPES_MAPPING)); + } + } + + public static class PropertyKey extends PostgresqlTableTemplate { + + public PropertyKey() { + super(new MysqlTables.PropertyKey(TYPES_MAPPING)); + } + } + + public static class IndexLabel extends PostgresqlTableTemplate { + + public IndexLabel() { + super(new MysqlTables.IndexLabel(TYPES_MAPPING)); + } + } + + public static class Vertex extends PostgresqlTableTemplate { + + public static final String TABLE = "vertices"; + + public Vertex(String store) { + super(new MysqlTables.Vertex(store, TYPES_MAPPING)); + } + } + + public static class Edge extends PostgresqlTableTemplate { + + public Edge(String store, Directions direction) { + super(new MysqlTables.Edge(store, direction, TYPES_MAPPING)); + } + + @Override + protected List idColumnValue(Id id) { + MysqlTables.Edge table = (MysqlTables.Edge) this.template; + return table.idColumnValue(id); + } + + @Override + public void delete(Session session, MysqlBackendEntry.Row entry) { + MysqlTables.Edge table = (MysqlTables.Edge) this.template; + table.delete(session, entry); + } + + @Override + protected BackendEntry mergeEntries(BackendEntry e1, BackendEntry e2) { + MysqlTables.Edge table = (MysqlTables.Edge) this.template; + return table.mergeEntries(e1, e2); + } + } + + public static class SecondaryIndex extends PostgresqlTableTemplate { + + public static final String TABLE = MysqlTables.SecondaryIndex.TABLE; + + public SecondaryIndex(String store) { + super(new MysqlTables.SecondaryIndex(store, TABLE, TYPES_MAPPING)); + } + + public SecondaryIndex(String store, String table) { + super(new MysqlTables.SecondaryIndex(store, table, TYPES_MAPPING)); + } + + protected final String entryId(MysqlBackendEntry entry) { + return ((MysqlTables.SecondaryIndex) this.template).entryId(entry); + } + } + + public static class SearchIndex extends SecondaryIndex { + + public static final String TABLE = MysqlTables.SearchIndex.TABLE; + + public SearchIndex(String store) { + super(store, TABLE); + } + } + + public static class RangeIndex extends PostgresqlTableTemplate { + + public RangeIndex(String store) { + super(new MysqlTables.RangeIndex(store, TYPES_MAPPING)); + } + + protected final String entryId(MysqlBackendEntry entry) { + return ((MysqlTables.RangeIndex) this.template).entryId(entry); + } + } +} diff --git a/hugegraph-test/pom.xml b/hugegraph-test/pom.xml index fdc9d0e4d4..ae5d845772 100644 --- a/hugegraph-test/pom.xml +++ b/hugegraph-test/pom.xml @@ -47,6 +47,11 @@ hugegraph-mysql ${project.version} + + com.baidu.hugegraph + hugegraph-postgresql + ${project.version} + com.baidu.hugegraph hugegraph-dist @@ -293,5 +298,15 @@ hbase + + postgresql + + false + + + postgresql + postgresql + + diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java index bff59662df..7175785e85 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/MetricsApiTest.java @@ -68,6 +68,7 @@ public void testMetricsBackend() { case "memory": case "mysql": case "hbase": + case "postgresql": String except = (String) assertMapContains(graph, "exception"); Assert.assertTrue(except, except.contains(notSupport)); break; diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java index 040c22c0de..75663669ff 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/api/TaskApiTest.java @@ -56,7 +56,7 @@ public void prepareSchema() { public void testList() { int taskId = this.rebuild(); - Response r = client().get(path); + Response r = client().get(path, ImmutableMap.of("limit", -1)); String content = assertResponseStatus(200, r); List> tasks = assertJsonContains(content, "tasks"); assertArrayContains(tasks, "id", taskId); diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index 21bb51ccc3..95fbd270d2 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -2394,7 +2394,7 @@ public void testQueryEdgeByPropertyWithEmptyString() { Assert.assertEquals("", edge.value("tool")); edge = graph.traversal().E().has("tool", "").has("place", "park") - .has("reason", "jeer").next(); + .has("reason", "jeer").next(); Assert.assertEquals(1, (int) edge.value("id")); } @@ -2593,17 +2593,20 @@ public void testQueryEdgeByPageResultsMatched() { String page = PageState.PAGE_NONE; int size = 20; + Set pageAll = new HashSet<>(); for (int i = 0; i < 100 / size; i++) { iter = graph.traversal().E() .has("~page", page).limit(size); - List vertexes = IteratorUtils.asList(iter); - Assert.assertEquals(size, vertexes.size()); + @SuppressWarnings("unchecked") + List edges = IteratorUtils.asList(iter); + Assert.assertEquals(size, edges.size()); - List expected = all.subList(i * size, (i + 1) * size); - Assert.assertEquals(expected, vertexes); + pageAll.addAll(edges); page = TraversalUtil.page(iter); } + Assert.assertEquals(100, pageAll.size()); + Assert.assertTrue(all.containsAll(pageAll)); Assert.assertNull(page); } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java index 98c1822bb5..4d06ad20f3 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexCoreTest.java @@ -3280,17 +3280,20 @@ public void testQueryByPageResultsMatched() { String page = PageState.PAGE_NONE; int size = 20; + Set pageAll = new HashSet<>(); for (int i = 0; i < 100 / size; i++) { iter = graph.traversal().V() .has("~page", page).limit(size); - List vertexes = IteratorUtils.asList(iter); + @SuppressWarnings("unchecked") + List vertexes = IteratorUtils.asList(iter); Assert.assertEquals(size, vertexes.size()); - List expected = all.subList(i * size, (i + 1) * size); - Assert.assertEquals(expected, vertexes); + pageAll.addAll(vertexes); page = TraversalUtil.page(iter); } + Assert.assertEquals(100, pageAll.size()); + Assert.assertTrue(all.containsAll(pageAll)); Assert.assertNull(page); } diff --git a/hugegraph-test/src/main/resources/hugegraph.properties b/hugegraph-test/src/main/resources/hugegraph.properties index 479584a48c..9dbc8ecc97 100644 --- a/hugegraph-test/src/main/resources/hugegraph.properties +++ b/hugegraph-test/src/main/resources/hugegraph.properties @@ -37,6 +37,7 @@ jdbc.username=root jdbc.password= jdbc.reconnect_max_times=3 jdbc.reconnect_interval=3 +jdbc.sslmode=disable # palo backend config palo.host=localhost diff --git a/pom.xml b/pom.xml index 0c2a9bf326..4c2ac6ef61 100644 --- a/pom.xml +++ b/pom.xml @@ -124,6 +124,7 @@ hugegraph-mysql hugegraph-palo hugegraph-hbase + hugegraph-postgresql