Support PostgreSQL and CockroachDB backends

implemented: #441

Change-Id: I979affd9f8b2916e6696672fc6e3fda75f9b76da

.travis.yml

@@ -5,6 +5,9 @@ jdk:
 
 sudo: required
 
+addons:
+  postgresql: "9.5"
+
 cache:
   directories:
     - $HOME/.m2
@@ -80,6 +83,7 @@ env:
     - BACKEND=mysql
     - BACKEND=hbase
     - BACKEND=rocksdb
+    - BACKEND=postgresql
   global:
     - RELEASE_BRANCH=^release-.*$
     - RELEASE_TAG=^v[0-9]\..*$

hugegraph-dist/pom.xml

@@ -59,6 +59,11 @@
             <artifactId>hugegraph-hbase</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.baidu.hugegraph</groupId>
+            <artifactId>hugegraph-postgresql</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>airline</artifactId>

hugegraph-dist/src/assembly/static/conf/hugegraph.properties

@@ -33,6 +33,7 @@ cassandra.password=
 
 
 # mysql backend config
+#jdbc.driver=com.mysql.jdbc.Driver
 #jdbc.url=jdbc:mysql://127.0.0.1:3306
 #jdbc.username=root
 #jdbc.password=

hugegraph-dist/src/assembly/travis/install-backend.sh

@@ -10,4 +10,6 @@ elif [ "$BACKEND" == "hbase" ]; then
     $TRAVIS_DIR/install-hbase.sh
 elif [ "$BACKEND" == "mysql" ]; then
     $TRAVIS_DIR/install-mysql.sh
+elif [ "$BACKEND" == "postgresql" ]; then
+    $TRAVIS_DIR/install-postgresql.sh
 fi

hugegraph-dist/src/assembly/travis/install-postgresql.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ev
+
+TRAVIS_DIR=`dirname $0`
+CONF=$TRAVIS_DIR/../../../../hugegraph-test/src/main/resources/hugegraph.properties
+
+POSTGRESQL_DRIVER=org.postgresql.Driver
+POSTGRESQL_URL=jdbc:postgresql://localhost:5432/
+POSTGRESQL_USERNAME=postgres
+
+# Set PostgreSQL configurations
+sed -i "s/jdbc.driver=.*/jdbc.driver=$POSTGRESQL_DRIVER/" $CONF
+sed -i "s?jdbc.url=.*?jdbc.url=$POSTGRESQL_URL?" $CONF
+sed -i "s/jdbc.username=.*/jdbc.username=$POSTGRESQL_USERNAME/" $CONF
+
+sudo service postgresql restart

hugegraph-dist/src/assembly/travis/start-server.sh

@@ -7,15 +7,29 @@ BASE_DIR=hugegraph-$VERSION
 BIN=$BASE_DIR/bin
 CONF=$BASE_DIR/conf/hugegraph.properties
 
+# PostgreSQL configurations
+POSTGRESQL_DRIVER=org.postgresql.Driver
+POSTGRESQL_URL=jdbc:postgresql://localhost:5432/
+POSTGRESQL_USERNAME=postgres
+
 declare -A backend_serializer_map=(["memory"]="text" ["cassandra"]="cassandra" \
                                    ["scylladb"]="scylladb" ["mysql"]="mysql" \
-                                   ["hbase"]="hbase" ["rocksdb"]="binary")
+                                   ["hbase"]="hbase" ["rocksdb"]="binary" \
+                                   ["postgresql"]="postgresql")
 
 SERIALIZER=${backend_serializer_map[$BACKEND]}
 
+# Set backend and serializer
 sed -i "s/backend=.*/backend=$BACKEND/" $CONF
 sed -i "s/serializer=.*/serializer=$SERIALIZER/" $CONF
 
+# Set PostgreSQL configurations if needed
+if [ "$BACKEND" == "postgresql" ]; then
+    sed -i "s/#jdbc.driver=.*/jdbc.driver=$POSTGRESQL_DRIVER/" $CONF
+    sed -i "s?#jdbc.url=.*?jdbc.url=$POSTGRESQL_URL?" $CONF
+    sed -i "s/#jdbc.username=.*/jdbc.username=$POSTGRESQL_USERNAME/" $CONF
+fi
+
 # Append schema.sync_deletion=true to config file
 echo "schema.sync_deletion=true" >> $CONF
 

hugegraph-dist/src/main/java/com/baidu/hugegraph/dist/RegisterUtil.java

@@ -89,6 +89,9 @@ private static void registerBackend(String backend) {
             case "palo":
                 registerPalo();
                 break;
+            case "postgresql":
+                registerPostgresql();
+                break;
             default:
                 throw new HugeException("Unsupported backend type '%s'", backend);
         }
@@ -165,6 +168,18 @@ public static void registerPalo() {
                 "com.baidu.hugegraph.backend.store.palo.PaloStoreProvider");
     }
 
+    public static void registerPostgresql() {
+        // Register config
+        OptionSpace.register("postgresql",
+                             "com.baidu.hugegraph.backend.store.postgresql.PostgresqlOptions");
+        // Register serializer
+        SerializerFactory.register("postgresql",
+                                   "com.baidu.hugegraph.backend.store.postgresql.PostgresqlSerializer");
+        // Register backend
+        BackendProviderFactory.register("postgresql",
+                                        "com.baidu.hugegraph.backend.store.postgresql.PostgresqlStoreProvider");
+    }
+
     public static void registerServer() {
         OptionSpace.register("server", "com.baidu.hugegraph.config.ServerOptions");
     }

hugegraph-dist/src/main/resources/backend.properties

@@ -1 +1 @@
-backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase]
+backends=[cassandra, scylladb, rocksdb, mysql, palo, hbase, postgresql]

hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlSessions.java

@@ -147,8 +147,11 @@ public void createDatabase() {
         try (Connection conn = this.openWithoutDB(0)) {
             conn.createStatement().execute(sql);
         } catch (SQLException e) {
-            throw new BackendException("Failed to create database '%s'",
-                                       this.database);
+            if (!e.getMessage().endsWith("already exists")) {
+                throw new BackendException("Failed to create database '%s'", e,
+                                           this.database);
+            }
+            // Ignore exception if database already exists
         }
     }
 

hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlStore.java

@@ -57,7 +57,7 @@ public abstract class MysqlStore extends AbstractBackendStore<Session> {
 
     private final Map<HugeType, MysqlTable> tables;
 
-    private MysqlSessions sessions;
+    protected MysqlSessions sessions;
 
     public MysqlStore(final BackendStoreProvider provider,
                       final String database, final String store) {
@@ -114,7 +114,8 @@ public synchronized void open(HugeConfig config) {
         try {
             this.sessions.open(config);
         } catch (Exception e) {
-            if (!e.getMessage().startsWith("Unknown database")) {
+            if (!e.getMessage().startsWith("Unknown database") &&
+                !e.getMessage().endsWith("does not exist")) {
                 throw new BackendException("Failed connect with mysql, " +
                                            "please ensure it's ok", e);
             }

hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTable.java

@@ -521,7 +521,7 @@ protected void wrapOffset(StringBuilder select, Query query) {
         select.append(";");
     }
 
-    private static Object serializeValue(Object value) {
+    protected static Object serializeValue(Object value) {
         if (value instanceof Id) {
             value = ((Id) value).asObject();
         }

hugegraph-mysql/src/main/java/com/baidu/hugegraph/backend/store/mysql/MysqlTables.java

@@ -106,8 +106,8 @@ public void increaseCounter(Session session,
                                     HugeType type, long increment) {
             String update = String.format(
                             "INSERT INTO %s VALUES ('%s', %s) " +
-                            "ON DUPLICATE KEY UPDATE " +
-                            "ID = ID + 1;", TABLE, type.name(), increment);
+                            "ON DUPLICATE KEY UPDATE ID = ID + %s;",
+                            TABLE, type.name(), increment, increment);
             try {
                 session.execute(update);
             } catch (SQLException e) {

hugegraph-postgresql/pom.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>hugegraph</artifactId>
+        <groupId>com.baidu.hugegraph</groupId>
+        <version>0.9.2</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>hugegraph-postgresql</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.baidu.hugegraph</groupId>
+            <artifactId>hugegraph-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.baidu.hugegraph</groupId>
+            <artifactId>hugegraph-mysql</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>42.1.4</version>
+        </dependency>
+    </dependencies>
+</project>

hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlFeatures.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.backend.store.postgresql;
+
+import com.baidu.hugegraph.backend.store.mysql.MysqlFeatures;
+
+public class PostgresqlFeatures extends MysqlFeatures {}

hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlOptions.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.backend.store.postgresql;
+
+import com.baidu.hugegraph.config.ConfigOption;
+import com.baidu.hugegraph.config.OptionHolder;
+
+import static com.baidu.hugegraph.config.OptionChecker.*;
+
+public class PostgresqlOptions extends OptionHolder {
+
+    private PostgresqlOptions() {
+        super();
+    }
+
+    private static volatile PostgresqlOptions instance;
+
+    public static synchronized PostgresqlOptions instance() {
+        if (instance == null) {
+            instance = new PostgresqlOptions();
+            instance.registerOptions();
+        }
+        return instance;
+    }
+
+    public static final ConfigOption<String> JDBC_DRIVER =
+            new ConfigOption<>(
+                    "jdbc.driver",
+                    "The JDBC driver class to connect database.",
+                    disallowEmpty(),
+                    "org.postgresql.Driver"
+            );
+
+    public static final ConfigOption<String> JDBC_URL =
+            new ConfigOption<>(
+                    "jdbc.url",
+                    "The url of database in JDBC format.",
+                    disallowEmpty(),
+                    "jdbc:postgresql://127.0.0.1:5432"
+            );
+
+    public static final ConfigOption<String> SSL_MODE =
+            new ConfigOption<>(
+                    "jdbc.ssl_mode",
+                    "The url of database in JDBC format.",
+                    disallowEmpty(),
+                    "disable"
+            );
+
+    public static final ConfigOption<String> JDBC_USERNAME =
+            new ConfigOption<>(
+                    "jdbc.username",
+                    "The username to login database.",
+                    disallowEmpty(),
+                    "root"
+            );
+
+    public static final ConfigOption<String> JDBC_PASSWORD =
+            new ConfigOption<>(
+                    "jdbc.password",
+                    "The password corresponding to jdbc.username.",
+                    null,
+                    ""
+            );
+}

hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSerializer.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.backend.store.postgresql;
+
+import com.baidu.hugegraph.backend.store.mysql.MysqlSerializer;
+
+public class PostgresqlSerializer extends MysqlSerializer {}

hugegraph-postgresql/src/main/java/com/baidu/hugegraph/backend/store/postgresql/PostgresqlSessions.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.backend.store.postgresql;
+
+import com.baidu.hugegraph.backend.store.mysql.MysqlSessions;
+import com.baidu.hugegraph.config.HugeConfig;
+
+public class PostgresqlSessions extends MysqlSessions {
+
+    public PostgresqlSessions(HugeConfig config, String database, String store) {
+        super(config, database, store);
+    }
+
+    @Override
+    protected String buildCreateDatabase(String database) {
+        return String.format("CREATE DATABASE %s ENCODING='UTF-8' " +
+                             "TEMPLATE=template0 LC_COLLATE='C' LC_CTYPE='C';", database);
+    }
+}