Skip to content

Commit

Permalink
Merge branch 'master' into release-vaughn
Browse files Browse the repository at this point in the history
  • Loading branch information
zyxxoo authored Dec 16, 2023
2 parents 20704bb + 4346b44 commit 3afbe22
Show file tree
Hide file tree
Showing 79 changed files with 2,289 additions and 969 deletions.
2 changes: 2 additions & 0 deletions .github/configs/settings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/check-dependencies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
branches:
- /^release-.*$/
pull_request:

permissions:
contents: read

Expand All @@ -17,7 +17,7 @@ jobs:
SCRIPT_DEPENDENCY: hugegraph-server/hugegraph-dist/scripts/dependency
steps:
- name: Checkout source
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
Expand All @@ -44,7 +44,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: 'Dependency Review'
uses: actions/dependency-review-action@v3
# Refer: https://github.com/actions/dependency-review-action
Expand Down
43 changes: 20 additions & 23 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,48 +30,44 @@ jobs:
JAVA_VERSION: [ '8', '11' ]

steps:
- name: Install JDK ${{ matrix.JAVA_VERSION }}
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.JAVA_VERSION }}
distribution: 'zulu'

- name: Cache Maven packages
uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2

- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 2

- name: Compile
run: |
mvn clean compile -U -Dmaven.javadoc.skip=true -ntp
- name: Install JDK 8
# TODO: Remove this step after install-backend.sh updated
- name: Install Java8 for backend
uses: actions/setup-java@v3
with:
java-version: '8'
distribution: 'zulu'
- name: Prepare env and service

- name: Prepare backend environment
run: |
$TRAVIS_DIR/install-backend.sh $BACKEND
$TRAVIS_DIR/install-backend.sh $BACKEND && jps -l
- name: Install JDK ${{ matrix.JAVA_VERSION }}
- name: Install Java ${{ matrix.JAVA_VERSION }}
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.JAVA_VERSION }}
distribution: 'zulu'

- name: use staged maven repo settings
- name: Cache Maven packages
uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2

- name: Use staged maven repo settings
if: ${{ env.USE_STAGE == 'true' }}
run: |
cp $HOME/.m2/settings.xml /tmp/settings.xml
mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml
cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml
- name: Compile
run: |
mvn clean compile -U -Dmaven.javadoc.skip=true -ntp
# - name: Run unit test
# run: |
Expand Down Expand Up @@ -99,4 +95,5 @@ jobs:
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ${{ env.REPORT_DIR }}/*.xml
6 changes: 3 additions & 3 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ on:
push:
branches: [ master, release-* ]
pull_request:
# The branches below must be a subset of the branches above
# branches: [ master ] # enable in all PR
# The branches below must be a subset of the branches above
# branches: [ master ] # enable in all PRs
schedule:
- cron: '33 0 * * 5'

Expand All @@ -27,7 +27,7 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Setup Java JDK
uses: actions/setup-java@v3
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/licence-checker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ on:
jobs:
check-license:
runs-on: ubuntu-latest
env:
USE_STAGE: 'true' # Whether to include the stage repository.
steps:
- uses: actions/checkout@v4

Expand Down
5 changes: 5 additions & 0 deletions hugegraph-pd/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# HugeGraph PD

HugeGraph PD is a meta server responsible for service discovery, partition information storage, and node scheduling.

> Note: Currently, the contents of this folder are empty. Starting from revision 1.5.0, the code of HugeGraph PD will be adapted to this location (WIP).
6 changes: 4 additions & 2 deletions hugegraph-server/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ FROM maven:3.9.0-eclipse-temurin-11 AS build

COPY . /pkg
WORKDIR /pkg
RUN mvn package -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true && pwd && ls -l
ARG MAVEN_ARGS

RUN mvn package $MAVEN_ARGS -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true && pwd && ls -l

# 2nd stage: runtime env
FROM openjdk:11-slim
# TODO: get the version from the pom.xml
ENV version=1.5.0
ENV version=1.2.0
COPY --from=build /pkg/hugegraph-server/apache-hugegraph-incubating-$version/ /hugegraph-server
LABEL maintainer="HugeGraph Docker Maintainers <[email protected]>"

Expand Down
11 changes: 11 additions & 0 deletions hugegraph-server/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# HugeGraph Server

HugeGraph Server consists of two layers of functionality: the graph engine layer, and the storage layer.

- Graph Engine Layer:
- REST Server: Provides a RESTful API for querying graph/schema information, supports the [Gremlin](https://tinkerpop.apache.org/gremlin.html) and [Cypher](https://en.wikipedia.org/wiki/Cypher) query languages, and offers APIs for service monitoring and operations.
- Graph Engine: Supports both OLTP and OLAP graph computation types, with OLTP implementing the [Apache TinkerPop3](https://tinkerpop.apache.org) framework.
- Backend Interface: Implements the storage of graph data to the backend.

- Storage Layer:
- Storage Backend: Supports multiple built-in storage backends (RocksDB/MySQL/HBase/...) and allows users to extend custom backends without modifying the existing source code.
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,16 @@ public void filter(ContainerRequestContext context) throws IOException {
return;
}

GlobalMasterInfo globalMasterInfo = manager.globalMasterInfo();
if (globalMasterInfo == null || !globalMasterInfo.isFeatureSupport()) {
GlobalMasterInfo globalNodeInfo = manager.globalNodeRoleInfo();
if (globalNodeInfo == null || !globalNodeInfo.supportElection()) {
return;
}
GlobalMasterInfo.NodeInfo masterNodeInfo = globalMasterInfo.nodeInfo();
if (masterNodeInfo == null || masterNodeInfo.isMaster() ||
StringUtils.isEmpty(masterNodeInfo.url())) {
GlobalMasterInfo.NodeInfo masterInfo = globalNodeInfo.masterInfo();
if (masterInfo == null || masterInfo.isMaster() ||
StringUtils.isEmpty(masterInfo.nodeUrl())) {
return;
}
String url = masterNodeInfo.url();
String url = masterInfo.nodeUrl();

URI redirectUri;
try {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/

package org.apache.hugegraph.api.gremlin;

import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.pool.PoolStats;
import org.apache.hugegraph.util.ExecutorUtil;
import org.glassfish.jersey.apache.connector.ApacheClientProperties;
import org.glassfish.jersey.apache.connector.ApacheConnectorProvider;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.client.JerseyClientBuilder;
import org.glassfish.jersey.message.GZipEncoder;

import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.WebTarget;

/**
* This class is a simplified class of AbstractRestClient from hugegraph-common.
* For some reason, we replace the rest implementation from jersey to okhttp.
* But GremlinClient still uses jersey-client to forward request, so we copy the old
* AbstractRestClient from hugegraph-common and rename the name to AbstractJerseyRestClient.
* Because we don't need the full feature of AbstractRestClient, so we reduce some useless code.
*/
public abstract class AbstractJerseyRestClient {

/**
* Time unit: hour
*/
private static final long TTL = 24L;
/**
* Time unit: ms
*/
private static final long IDLE_TIME = 40L * 1000L;
private static final String PROPERTY_MAX_TOTAL = "maxTotal";
private static final String PROPERTY_MAX_PER_ROUTE = "maxPerRoute";
private static final String PROPERTY_IDLE_TIME = "idleTime";
private static final String CONNECTION_MANAGER = ApacheClientProperties.CONNECTION_MANAGER;

private final Client client;
private final WebTarget webTarget;
private final PoolingHttpClientConnectionManager pool;
private ScheduledExecutorService cleanExecutor;

public AbstractJerseyRestClient(String url, int timeout, int maxTotal, int maxPerRoute) {
this(url, new ConfigBuilder().configTimeout(timeout)
.configPool(maxTotal, maxPerRoute)
.build());
}

public AbstractJerseyRestClient(String url, ClientConfig config) {
this.pool = configConnectionManager(config);

this.client = JerseyClientBuilder.newClient(config);
this.client.register(GZipEncoder.class);
this.webTarget = this.client.target(url);

cleanThreadPoolExecutor(config);
}

private static PoolingHttpClientConnectionManager configConnectionManager(ClientConfig conf) {
/*
* Using httpclient with connection pooling, and configuring the
* jersey connector. But the jersey that has been released in the maven central
* repository seems to have a bug: https://github.com/jersey/jersey/pull/3752
*/
PoolingHttpClientConnectionManager pool =
new PoolingHttpClientConnectionManager(TTL, TimeUnit.HOURS);
Integer maxTotal = (Integer) conf.getProperty(PROPERTY_MAX_TOTAL);
Integer maxPerRoute = (Integer) conf.getProperty(PROPERTY_MAX_PER_ROUTE);

if (maxTotal != null) {
pool.setMaxTotal(maxTotal);
}
if (maxPerRoute != null) {
pool.setDefaultMaxPerRoute(maxPerRoute);
}
conf.property(CONNECTION_MANAGER, pool);
conf.connectorProvider(new ApacheConnectorProvider());
return pool;
}

private void cleanThreadPoolExecutor(ClientConfig config) {
this.cleanExecutor = ExecutorUtil.newScheduledThreadPool("conn-clean-worker-%d");
Number idleTimeProp = (Number) config.getProperty(PROPERTY_IDLE_TIME);
final long idleTime = idleTimeProp == null ? IDLE_TIME : idleTimeProp.longValue();
final long checkPeriod = idleTime / 2L;
this.cleanExecutor.scheduleWithFixedDelay(() -> {
PoolStats stats = this.pool.getTotalStats();
int using = stats.getLeased() + stats.getPending();
if (using > 0) {
// Do clean only when all connections are idle
return;
}
// Release connections when all clients are inactive
this.pool.closeIdleConnections(idleTime, TimeUnit.MILLISECONDS);
this.pool.closeExpiredConnections();
}, checkPeriod, checkPeriod, TimeUnit.MILLISECONDS);
}

protected WebTarget getWebTarget() {
return this.webTarget;
}

public void close() {
try {
if (this.pool != null) {
this.pool.close();
this.cleanExecutor.shutdownNow();
}
} finally {
this.client.close();
}
}

private static class ConfigBuilder {

private final ClientConfig config;

ConfigBuilder() {
this.config = new ClientConfig();
}

public ConfigBuilder configTimeout(int timeout) {
this.config.property(ClientProperties.CONNECT_TIMEOUT, timeout);
this.config.property(ClientProperties.READ_TIMEOUT, timeout);
return this;
}

public ConfigBuilder configPool(int maxTotal, int maxPerRoute) {
this.config.property(PROPERTY_MAX_TOTAL, maxTotal);
this.config.property(PROPERTY_MAX_PER_ROUTE, maxPerRoute);
return this;
}

public ClientConfig build() {
return this.config;
}
}
}
Loading

0 comments on commit 3afbe22

Please sign in to comment.