From fc8be06ad51addfb39c704c0921203fbe7971610 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Wed, 27 Nov 2024 08:58:09 +0800 Subject: [PATCH] [improve] update victoriametrics and greptime store (#2836) Signed-off-by: tomsun28 Signed-off-by: Logic Co-authored-by: shown Co-authored-by: Logic Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Zhang Yuxuan <1373529784@qq.com> --- .../src/main/resources/application.yml | 6 +- .../src/main/resources/define/app-mariadb.yml | 1 - .../src/main/resources/define/app-mysql.yml | 1 - hertzbeat-warehouse/pom.xml | 6 - .../greptime/GreptimeDbDataStorage.java | 425 ++++++------------ .../history/greptime/GreptimeProperties.java | 8 +- .../store/history/vm/PromQlQueryContent.java | 99 ++++ .../vm/VictoriaMetricsClusterDataStorage.java | 146 +++--- .../vm/VictoriaMetricsDataStorage.java | 143 +++--- home/docs/start/greptime-init.md | 14 +- .../current/start/greptime-init.md | 13 +- pom.xml | 2 +- script/application.yml | 39 +- script/sureness.yml | 6 +- .../routes/bulletin/bulletin.component.html | 2 +- web-app/src/assets/app-data.json | 4 +- web-app/src/assets/i18n/en-US.json | 3 +- web-app/src/assets/i18n/zh-CN.json | 3 +- web-app/src/assets/i18n/zh-TW.json | 5 +- 19 files changed, 396 insertions(+), 530 deletions(-) create mode 100644 hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/PromQlQueryContent.java diff --git a/hertzbeat-manager/src/main/resources/application.yml b/hertzbeat-manager/src/main/resources/application.yml index 82728fc5fc1..dfe141f5716 100644 --- a/hertzbeat-manager/src/main/resources/application.yml +++ b/hertzbeat-manager/src/main/resources/application.yml @@ -144,11 +144,11 @@ warehouse: greptime: enabled: false grpc-endpoints: localhost:4001 - url: jdbc:mysql://localhost:4002/hertzbeat?connectionTimeZone=Asia/Shanghai&forceConnectionTimeZoneToSession=true - driver-class-name: com.mysql.cj.jdbc.Driver + http-endpoint: http://localhost:4000 + # if you config other database name, you should create them first + database: public username: greptime password: greptime - expire-time: 30d iot-db: enabled: false host: 127.0.0.1 diff --git a/hertzbeat-manager/src/main/resources/define/app-mariadb.yml b/hertzbeat-manager/src/main/resources/define/app-mariadb.yml index c3186bbb24a..4d53c5c57a7 100644 --- a/hertzbeat-manager/src/main/resources/define/app-mariadb.yml +++ b/hertzbeat-manager/src/main/resources/define/app-mariadb.yml @@ -107,7 +107,6 @@ metrics: # field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field - field: version type: 1 - label: true i18n: zh-CN: 版本 en-US: Version diff --git a/hertzbeat-manager/src/main/resources/define/app-mysql.yml b/hertzbeat-manager/src/main/resources/define/app-mysql.yml index 43d5f1a06e8..dc565eb4622 100644 --- a/hertzbeat-manager/src/main/resources/define/app-mysql.yml +++ b/hertzbeat-manager/src/main/resources/define/app-mysql.yml @@ -131,7 +131,6 @@ metrics: # field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field - field: version type: 1 - label: true i18n: zh-CN: 版本 en-US: Version diff --git a/hertzbeat-warehouse/pom.xml b/hertzbeat-warehouse/pom.xml index 491b6349a1c..caa9d0c56de 100644 --- a/hertzbeat-warehouse/pom.xml +++ b/hertzbeat-warehouse/pom.xml @@ -103,12 +103,6 @@ - - - mysql - mysql-connector-java - ${mysql-jdbcdriver.version} - org.apache.kafka diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeDbDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeDbDataStorage.java index 1e039188f25..df99916e1f2 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeDbDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeDbDataStorage.java @@ -17,9 +17,6 @@ package org.apache.hertzbeat.warehouse.store.history.greptime; -import com.mysql.cj.jdbc.Driver; -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; import io.greptime.GreptimeDB; import io.greptime.models.AuthInfo; import io.greptime.models.DataType; @@ -31,14 +28,14 @@ import io.greptime.options.GreptimeOptions; import java.math.BigDecimal; import java.math.RoundingMode; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.DriverPropertyInfo; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Collections; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAmount; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -46,15 +43,25 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.hertzbeat.common.constants.CommonConstants; import org.apache.hertzbeat.common.entity.dto.Value; import org.apache.hertzbeat.common.entity.message.CollectRep; import org.apache.hertzbeat.common.util.JsonUtil; +import org.apache.hertzbeat.common.util.TimePeriodUtil; import org.apache.hertzbeat.warehouse.store.history.AbstractHistoryDataStorage; +import org.apache.hertzbeat.warehouse.store.history.vm.PromQlQueryContent; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; /** * GreptimeDB data storage, only supports GreptimeDB version >= v0.5 @@ -63,180 +70,101 @@ @ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true") @Slf4j public class GreptimeDbDataStorage extends AbstractHistoryDataStorage { - - private static final String CONSTANT_DB_TTL = "30d"; - - private static final String QUERY_HISTORY_SQL = "SELECT CAST (ts AS Int64) ts, instance, `%s` FROM `%s` WHERE ts >= now() - interval '%s' and monitor_id = %s order by ts desc;"; - - @SuppressWarnings("checkstyle:LineLength") - private static final String QUERY_HISTORY_WITH_INSTANCE_SQL = "SELECT CAST (ts AS Int64) ts, instance, `%s` FROM `%s` WHERE ts >= now() - interval '%s' and monitor_id = %s and instance = '%s' order by ts desc;"; - - private static final String QUERY_INSTANCE_SQL = "SELECT DISTINCT instance FROM `%s` WHERE ts >= now() - interval '1 WEEK'"; - - @SuppressWarnings("checkstyle:LineLength") - private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = "SELECT CAST (ts AS Int64) ts, first_value(`%s`) range '4h' first, avg(`%s`) range '4h' avg, min(`%s`) range '4h' min, max(`%s`) range '4h' max FROM `%s` WHERE instance = '%s' AND ts >= now() - interval '%s' ALIGN '4h'"; - - private static final String TABLE_NOT_EXIST = "not found"; - - private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS `%s` WITH(ttl='%s')"; - - private static final Runnable INSTANCE_EXCEPTION_PRINT = () -> { - if (log.isErrorEnabled()) { - log.error(""" - \t---------------GreptimeDB Init Failed--------------- - \t--------------Please Config GreptimeDB-------------- - t-----------Can Not Use Metric History Now----------- - """); - } - }; - - private HikariDataSource hikariDataSource; - + + private static final String BASIC = "Basic"; + private static final String QUERY_RANGE_PATH = "/v1/prometheus/api/v1/query_range"; + private static final String LABEL_KEY_NAME = "__name__"; + private static final String LABEL_KEY_FIELD = "__field__"; + private static final String LABEL_KEY_INSTANCE = "instance"; + private static final String SPILT = "_"; + private GreptimeDB greptimeDb; - - public GreptimeDbDataStorage(GreptimeProperties greptimeProperties) { + + private final GreptimeProperties greptimeProperties; + + private final RestTemplate restTemplate; + + public GreptimeDbDataStorage(GreptimeProperties greptimeProperties, RestTemplate restTemplate) { if (greptimeProperties == null) { log.error("init error, please config Warehouse GreptimeDB props in application.yml"); throw new IllegalArgumentException("please config Warehouse GreptimeDB props"); } - - serverAvailable = initGreptimeDbClient(greptimeProperties) && initGreptimeDbDataSource(greptimeProperties); + this.restTemplate = restTemplate; + this.greptimeProperties = greptimeProperties; + serverAvailable = initGreptimeDbClient(greptimeProperties); } - - private void initGreptimeDb(final GreptimeProperties greptimeProperties) throws SQLException { - final DriverPropertyInfo[] properties = new Driver().getPropertyInfo(greptimeProperties.url(), null); - final String host = ObjectUtils.requireNonEmpty(properties[0].value); - final String port = ObjectUtils.requireNonEmpty(properties[1].value); - final String dbName = ObjectUtils.requireNonEmpty(properties[2].value); - - String ttl = greptimeProperties.expireTime(); - if (ttl == null || StringUtils.isBlank(ttl.trim())) { - ttl = CONSTANT_DB_TTL; - } - - try (final Connection tempConnection = DriverManager.getConnection("jdbc:mysql://" + host + ":" + port, - greptimeProperties.username(), greptimeProperties.password()); - final PreparedStatement pstmt = tempConnection - .prepareStatement(String.format(CONSTANTS_CREATE_DATABASE, dbName, ttl))) { - log.info("[warehouse greptime] try to create database `{}` if not exists", dbName); - pstmt.execute(); - } - } - + private boolean initGreptimeDbClient(GreptimeProperties greptimeProperties) { String endpoints = greptimeProperties.grpcEndpoints(); try { - final DriverPropertyInfo[] properties = new Driver().getPropertyInfo(greptimeProperties.url(), null); - final String dbName = ObjectUtils.requireNonEmpty(properties[2].value); - - GreptimeOptions opts = GreptimeOptions.newBuilder(endpoints.split(","), dbName) // - .writeMaxRetries(3) // + GreptimeOptions opts = GreptimeOptions.newBuilder(endpoints.split(","), greptimeProperties.database()) + .writeMaxRetries(3) .authInfo(new AuthInfo(greptimeProperties.username(), greptimeProperties.password())) - .routeTableRefreshPeriodSeconds(30) // + .routeTableRefreshPeriodSeconds(30) .build(); - this.greptimeDb = GreptimeDB.create(opts); } catch (Exception e) { log.error("[warehouse greptime] Fail to start GreptimeDB client"); return false; } - - return true; - } - - private boolean initGreptimeDbDataSource(final GreptimeProperties greptimeProperties) { - try { - initGreptimeDb(greptimeProperties); - } catch (Exception e) { - if (log.isErrorEnabled()) { - log.error(e.getMessage(), e); - } - - INSTANCE_EXCEPTION_PRINT.run(); - return false; - } - - final HikariConfig config = new HikariConfig(); - // jdbc properties - config.setJdbcUrl(greptimeProperties.url()); - config.setUsername(greptimeProperties.username()); - config.setPassword(greptimeProperties.password()); - config.setDriverClassName(greptimeProperties.driverClassName()); - // minimum number of idle connection - config.setMinimumIdle(10); - // maximum number of connection in the pool - config.setMaximumPoolSize(10); - // maximum wait milliseconds for get connection from pool - config.setConnectionTimeout(30000); - // maximum lifetime for each connection - config.setMaxLifetime(0); - // max idle time for recycle idle connection - config.setIdleTimeout(0); - // validation query - config.setConnectionTestQuery("select 1"); - try { - this.hikariDataSource = new HikariDataSource(config); - } catch (Exception e) { - INSTANCE_EXCEPTION_PRINT.run(); - return false; - } + return true; } - + @Override public void saveData(CollectRep.MetricsData metricsData) { if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) { return; } if (metricsData.getValuesList().isEmpty()) { - log.info("[warehouse greptime] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse greptime] flush metrics data {} {}is null, ignore.", metricsData.getId(), metricsData.getMetrics()); return; } String monitorId = String.valueOf(metricsData.getId()); String tableName = getTableName(metricsData.getApp(), metricsData.getMetrics()); TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(tableName); - - tableSchemaBuilder.addTag("monitor_id", DataType.String) // - .addTag("instance", DataType.String) // + + tableSchemaBuilder.addTag("instance", DataType.String) .addTimestamp("ts", DataType.TimestampMillisecond); - + List fieldsList = metricsData.getFieldsList(); for (CollectRep.Field field : fieldsList) { // handle field type - if (field.getType() == CommonConstants.TYPE_NUMBER) { - tableSchemaBuilder.addField(field.getName(), DataType.Float64); - } else if (field.getType() == CommonConstants.TYPE_STRING) { - tableSchemaBuilder.addField(field.getName(), DataType.String); + if (field.getLabel()) { + tableSchemaBuilder.addTag(field.getName(), DataType.String); + } else { + if (field.getType() == CommonConstants.TYPE_NUMBER) { + tableSchemaBuilder.addField(field.getName(), DataType.Float64); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + tableSchemaBuilder.addField(field.getName(), DataType.String); + } } } Table table = Table.from(tableSchemaBuilder.build()); - try { long now = System.currentTimeMillis(); - Object[] values = new Object[3 + fieldsList.size()]; + Object[] values = new Object[2 + fieldsList.size()]; values[0] = monitorId; - values[2] = now; + values[1] = now; for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { - Map labels = new HashMap<>(8); for (int i = 0; i < fieldsList.size(); i++) { if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { CollectRep.Field field = fieldsList.get(i); - if (field.getType() == CommonConstants.TYPE_NUMBER) { - values[3 + i] = Double.parseDouble(valueRow.getColumns(i)); - } else if (field.getType() == CommonConstants.TYPE_STRING) { - values[3 + i] = valueRow.getColumns(i); - } if (field.getLabel()) { - labels.put(field.getName(), String.valueOf(values[3 + i])); + values[2 + i] = valueRow.getColumns(i); + } else { + if (field.getType() == CommonConstants.TYPE_NUMBER) { + values[2 + i] = Double.parseDouble(valueRow.getColumns(i)); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + values[2 + i] = valueRow.getColumns(i); + } } } else { - values[3 + i] = null; + values[2 + i] = null; } } - values[1] = JsonUtil.toJson(labels); table.addRow(values); } - CompletableFuture> writeFuture = greptimeDb.write(table); try { Result result = writeFuture.get(10, TimeUnit.SECONDS); @@ -252,168 +180,97 @@ public void saveData(CollectRep.MetricsData metricsData) { log.error("[warehouse greptime]--Error: {}", e.getMessage(), e); } } - + @Override public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String label, String history) { - Map> instanceValuesMap = new HashMap<>(8); - if (!isServerAvailable()) { - INSTANCE_EXCEPTION_PRINT.run(); - return instanceValuesMap; - } - - String table = getTableName(app, metrics); - - String interval = history2interval(history); - String selectSql = label == null ? String.format(QUERY_HISTORY_SQL, metric, table, interval, monitorId) - : String.format(QUERY_HISTORY_WITH_INSTANCE_SQL, metric, table, interval, monitorId, label); - - if (log.isDebugEnabled()) { - log.debug("[warehouse greptime] getHistoryMetricData SQL: {}", selectSql); - } - - try (Connection connection = hikariDataSource.getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(selectSql)) { - while (resultSet.next()) { - long ts = resultSet.getLong(1); - if (ts == 0) { - if (log.isErrorEnabled()) { - log.error("[warehouse greptime] getHistoryMetricData query result timestamp is 0, ignore. {}.", - selectSql); - } - continue; - } - String instanceValue = resultSet.getString(2); - if (instanceValue == null || StringUtils.isBlank(instanceValue)) { - instanceValue = ""; - } - double value = resultSet.getDouble(3); - String strValue = double2decimalString(value); - - List valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); - valueList.add(new Value(strValue, ts)); - } - return instanceValuesMap; - } catch (SQLException sqlException) { - String msg = sqlException.getMessage(); - if (msg != null && !msg.contains(TABLE_NOT_EXIST)) { - if (log.isWarnEnabled()) { - log.warn("[warehouse greptime] failed to getHistoryMetricData: {}", sqlException.getMessage()); - } - } - } catch (Exception e) { - if (log.isErrorEnabled()) { - log.error("[warehouse greptime] failed to getHistoryMetricData:{}", e.getMessage(), e); - } - } - return instanceValuesMap; - } - - private String getTableName(String app, String metrics) { - return app + "_" + metrics; - } - - @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, - String metric, String label, String history) { - if (!isServerAvailable()) { - INSTANCE_EXCEPTION_PRINT.run(); - return Collections.emptyMap(); - } - String table = getTableName(app, metrics); - List instances = new LinkedList<>(); - if (label != null && !StringUtils.isBlank(label)) { - instances.add(label); + String name = getTableName(app, metrics); + String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + name + "\"" + + "," + LABEL_KEY_INSTANCE + "=\"" + monitorId + "\""; + if (!CommonConstants.PROMETHEUS.equals(app)) { + timeSeriesSelector = timeSeriesSelector + "," + LABEL_KEY_FIELD + "=\"" + metric + "\""; } - if (instances.isEmpty()) { - String selectSql = String.format(QUERY_INSTANCE_SQL, table); - if (log.isDebugEnabled()) { - log.debug("[warehouse greptime] getHistoryIntervalMetricData sql: {}", selectSql); + Map> instanceValuesMap = new HashMap<>(8); + try { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.setAccept(List.of(MediaType.APPLICATION_JSON)); + if (StringUtils.hasText(greptimeProperties.username()) + && StringUtils.hasText(greptimeProperties.password())) { + String authStr = greptimeProperties.username() + ":" + greptimeProperties.password(); + String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + headers.add(HttpHeaders.AUTHORIZATION, BASIC + " " + encodedAuth); } - - try (Connection connection = hikariDataSource.getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(selectSql)) { - while (resultSet.next()) { - String instanceValue = resultSet.getString(1); - if (instanceValue == null || StringUtils.isBlank(instanceValue)) { - instances.add("''"); - } else { - instances.add(instanceValue); - } + Instant now = Instant.now(); + long start; + try { + if (NumberUtils.isParsable(history)) { + start = NumberUtils.toLong(history); + start = (ZonedDateTime.now().toEpochSecond() - start); + } else { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); + assert temporalAmount != null; + Instant dateTime = now.minus(temporalAmount); + start = dateTime.getEpochSecond(); } } catch (Exception e) { - if (log.isErrorEnabled()) { - log.error("[warehouse greptime] failed to query instances{}", e.getMessage(), e); - } + log.error("history time error: {}. use default: 6h", e.getMessage()); + start = now.minus(6, ChronoUnit.HOURS).getEpochSecond(); } - } - - Map> instanceValuesMap = new HashMap<>(instances.size()); - for (String instanceValue : instances) { - String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, metric, metric, metric, - table, instanceValue, history2interval(history)); - - if (log.isDebugEnabled()) { - log.debug("[warehouse greptime] getHistoryIntervalMetricData sql: {}", selectSql); + long end = now.getEpochSecond(); + String step = "60s"; + if (end - start < Duration.ofDays(7).getSeconds() && end - start > Duration.ofDays(1).getSeconds()) { + step = "1h"; + } else if (end - start >= Duration.ofDays(7).getSeconds()) { + step = "4h"; } - - List values = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); - try (Connection connection = hikariDataSource.getConnection(); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(selectSql)) { - while (resultSet.next()) { - long ts = resultSet.getLong(1); - if (ts == 0) { - if (log.isErrorEnabled()) { - log.error( - "[warehouse greptime] getHistoryIntervalMetricData query result timestamp is 0, ignore. {}.", - selectSql); + HttpEntity httpEntity = new HttpEntity<>(headers); + URI uri = UriComponentsBuilder.fromHttpUrl(greptimeProperties.httpEndpoint() + QUERY_RANGE_PATH) + .queryParam(URLEncoder.encode("query", StandardCharsets.UTF_8), URLEncoder.encode("{" + timeSeriesSelector + "}", StandardCharsets.UTF_8)) + .queryParam("start", start) + .queryParam("end", end) + .queryParam("step", step) + .build(true).toUri(); + ResponseEntity responseEntity = restTemplate.exchange(uri, + HttpMethod.GET, httpEntity, PromQlQueryContent.class); + if (responseEntity.getStatusCode().is2xxSuccessful()) { + log.debug("query metrics data from victoria-metrics success. {}", uri); + if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null + && responseEntity.getBody().getData().getResult() != null) { + List contents = responseEntity.getBody().getData().getResult(); + for (PromQlQueryContent.ContentData.Content content : contents) { + Map labels = content.getMetric(); + labels.remove(LABEL_KEY_NAME); + labels.remove(LABEL_KEY_INSTANCE); + String labelStr = JsonUtil.toJson(labels); + if (content.getValues() != null && !content.getValues().isEmpty()) { + List valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>()); + for (Object[] valueArr : content.getValues()) { + long timestamp = ((Double) valueArr[0]).longValue(); + String value = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + // read timestamp here is s unit + valueList.add(new Value(value, timestamp * 1000)); + } } - continue; } - double origin = resultSet.getDouble(2); - String originStr = double2decimalString(origin); - double avg = resultSet.getDouble(3); - String avgStr = double2decimalString(avg); - double min = resultSet.getDouble(4); - String minStr = double2decimalString(min); - double max = resultSet.getDouble(5); - String maxStr = double2decimalString(max); - Value value = Value.builder().origin(originStr).mean(avgStr).min(minStr).max(maxStr).time(ts) - .build(); - values.add(value); - } - resultSet.close(); - } catch (Exception e) { - if (log.isErrorEnabled()) { - log.error("[warehouse greptime] failed to getHistoryIntervalMetricData: {}", e.getMessage(), e); } + } else { + log.error("query metrics data from greptime failed. {}", responseEntity); } + } catch (Exception e) { + log.error(e.getMessage(), e); } return instanceValuesMap; } - - // TODO(dennis): we can remove it when - // https://github.com/GreptimeTeam/greptimedb/issues/4168 is fixed. - // default 6h-6 hours: s-seconds, M-minutes, h-hours, d-days, w-weeks - private String history2interval(String history) { - if (history == null) { - return null; - } - history = history.trim().toLowerCase(); - - // Be careful, the order matters. - return history.replaceAll("d", " day") // - .replaceAll("s", " second") // - .replaceAll("w", " week") // - .replaceAll("h", " hour")// - .replaceAll("m", " minute"); + + private String getTableName(String app, String metrics) { + return app + SPILT + metrics; } - - private String double2decimalString(double d) { - return BigDecimal.valueOf(d).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + + @Override + public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, + String metric, String label, String history) { + return getHistoryMetricData(monitorId, app, metrics, metric, label, history); } @Override @@ -422,9 +279,5 @@ public void destroy() { this.greptimeDb.shutdownGracefully(); this.greptimeDb = null; } - if (this.hikariDataSource != null) { - this.hikariDataSource.close(); - hikariDataSource = null; - } } } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeProperties.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeProperties.java index 8b9bcf2619a..c48d4af0d24 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeProperties.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/greptime/GreptimeProperties.java @@ -26,16 +26,12 @@ /** * GrepTimeDB configuration information */ - @ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE + SignConstants.DOT + WarehouseConstants.STORE + SignConstants.DOT + WarehouseConstants.HistoryName.GREPTIME) public record GreptimeProperties(@DefaultValue("false") boolean enabled, - @DefaultValue("127.0.0.1:4001") String grpcEndpoints, - @DefaultValue("jdbc:mysql://127.0.0.1:4002/hertzbeat?connectionTimeZone=Asia/Shanghai&forceConnectionTimeZoneToSession=true") String url, - @DefaultValue("com.mysql.cj.jdbc.Driver") String driverClassName, String username, String password, - // Database TTL, default is 30 days. - @DefaultValue("30d") String expireTime) { + @DefaultValue("127.0.0.1:4001") String grpcEndpoints, @DefaultValue("http://127.0.0.1:4000") String httpEndpoint, + @DefaultValue("public") String database, String username, String password) { } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/PromQlQueryContent.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/PromQlQueryContent.java new file mode 100644 index 00000000000..6620416f1b3 --- /dev/null +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/PromQlQueryContent.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hertzbeat.warehouse.store.history.vm; + +import java.util.List; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * promql query content + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class PromQlQueryContent { + + private String status; + + private ContentData data; + + /** + * content data + */ + @Data + @AllArgsConstructor + @NoArgsConstructor + public static final class ContentData { + + private String resultType; + + private List result; + + /** + * content + */ + @Data + @AllArgsConstructor + @NoArgsConstructor + public static final class Content { + + /** + * metric contains metric name plus labels for a particular time series + */ + private Map metric; + + /** + * values contains raw sample values for the given time series + * value-timestamp + * [1700993195,"436960986"] + */ + private Object[] value; + + /** + * values contains raw sample values for the given time series + * value-timestamp list + * [[1700993195,"436960986"],[1700993195,"436960986"]...] + */ + private List values; + } + } +} diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsClusterDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsClusterDataStorage.java index ec521be9736..6e1a298ed1e 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsClusterDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsClusterDataStorage.java @@ -125,7 +125,8 @@ public void saveData(CollectRep.MetricsData metricsData) { return; } if (metricsData.getValuesList().isEmpty()) { - log.info("[warehouse victoria-metrics] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse victoria-metrics] flush metrics data {} {} {} is null, ignore.", + metricsData.getId(), metricsData.getApp(), metricsData.getMetrics()); return; } Map defaultLabels = new HashMap<>(8); @@ -134,8 +135,8 @@ public void saveData(CollectRep.MetricsData metricsData) { if (metricsData.getApp().startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) { isPrometheusAuto = true; defaultLabels.remove(MONITOR_METRICS_KEY); - defaultLabels.put(LABEL_KEY_JOB, - metricsData.getApp().substring(CommonConstants.PROMETHEUS_APP_PREFIX.length())); + defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp() + .substring(CommonConstants.PROMETHEUS_APP_PREFIX.length())); } else { defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()); } @@ -145,6 +146,7 @@ public void saveData(CollectRep.MetricsData metricsData) { Long[] timestamp = new Long[]{metricsData.getTime()}; Map fieldsValue = new HashMap<>(fields.size()); Map labels = new HashMap<>(fields.size()); + List contentList = new LinkedList<>(); for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { fieldsValue.clear(); labels.clear(); @@ -172,34 +174,47 @@ public void saveData(CollectRep.MetricsData metricsData) { if (!isPrometheusAuto) { labels.put(MONITOR_METRIC_KEY, entry.getKey()); } - VictoriaMetricsContent content = VictoriaMetricsContent.builder().metric(labels) - .values(new Double[]{entry.getValue()}).timestamps(timestamp).build(); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - if (StringUtils.hasText(vmInsertProps.username()) && StringUtils.hasText( - vmInsertProps.password())) { - String authStr = vmInsertProps.username() + ":" + vmInsertProps.password(); - String encodedAuth = new String( - Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8); - headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC - + SignConstants.BLANK + encodedAuth); - } - HttpEntity httpEntity = new HttpEntity<>(content, headers); - ResponseEntity responseEntity = restTemplate.postForEntity( - vmInsertProps.url() + IMPORT_PATH, httpEntity, String.class); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - log.debug("insert metrics data to victoria-metrics success. {}", content); - } else { - log.error("insert metrics data to victoria-metrics failed. {}", content); - } + VictoriaMetricsDataStorage.VictoriaMetricsContent content = VictoriaMetricsDataStorage.VictoriaMetricsContent.builder() + .metric(new HashMap<>(labels)) + .values(new Double[]{entry.getValue()}) + .timestamps(timestamp) + .build(); + contentList.add(content); } catch (Exception e) { - log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e); + log.error("combine metrics data error: {}.", e.getMessage(), e); } } } } + if (contentList.isEmpty()) { + log.info("[warehouse victoria-metrics] flush metrics data {} is empty, ignore.", metricsData.getId()); + return; + } + try { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + if (StringUtils.hasText(vmInsertProps.username()) + && StringUtils.hasText(vmInsertProps.password())) { + String authStr = vmInsertProps.username() + ":" + vmInsertProps.password(); + String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth); + } + StringBuilder stringBuilder = new StringBuilder(); + for (VictoriaMetricsDataStorage.VictoriaMetricsContent content : contentList) { + stringBuilder.append(JsonUtil.toJson(content)).append("\n"); + } + HttpEntity httpEntity = new HttpEntity<>(stringBuilder.toString(), headers); + ResponseEntity responseEntity = restTemplate.postForEntity(vmInsertProps.url() + IMPORT_PATH, + httpEntity, String.class); + if (responseEntity.getStatusCode().is2xxSuccessful()) { + log.debug("insert metrics data to victoria-metrics success."); + } else { + log.error("insert metrics data to victoria-metrics failed. {}", responseEntity.getBody()); + } + } catch (Exception e){ + log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e); + } } @Override @@ -330,15 +345,15 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str URLEncoder.encode("{" + timeSeriesSelector + "}", StandardCharsets.UTF_8)) .queryParam("step", "4h").queryParam("start", startTime).queryParam("end", endTime).build(true) .toUri(); - ResponseEntity responseEntity = restTemplate.exchange(uri, HttpMethod.GET, - httpEntity, VictoriaMetricsQueryContent.class); + ResponseEntity responseEntity = restTemplate.exchange(uri, HttpMethod.GET, + httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { log.debug("query metrics data from victoria-metrics success. {}", uri); if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData() + List contents = responseEntity.getBody().getData() .getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -368,13 +383,13 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str URLEncoder.encode("max_over_time({" + timeSeriesSelector + "})", StandardCharsets.UTF_8)) .queryParam("step", "4h").queryParam("start", startTime).queryParam("end", endTime).build(true) .toUri(); - responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, VictoriaMetricsQueryContent.class); + responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData() + List contents = responseEntity.getBody().getData() .getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -404,13 +419,13 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str URLEncoder.encode("min_over_time({" + timeSeriesSelector + "})", StandardCharsets.UTF_8)) .queryParam("step", "4h").queryParam("start", startTime).queryParam("end", endTime).build(true) .toUri(); - responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, VictoriaMetricsQueryContent.class); + responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData() + List contents = responseEntity.getBody().getData() .getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -440,13 +455,13 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str URLEncoder.encode("avg_over_time({" + timeSeriesSelector + "})", StandardCharsets.UTF_8)) .queryParam("step", "4h").queryParam("start", startTime).queryParam("end", endTime).build(true) .toUri(); - responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, VictoriaMetricsQueryContent.class); + responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData() + List contents = responseEntity.getBody().getData() .getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -501,59 +516,4 @@ public static final class VictoriaMetricsContent { */ private Long[] timestamps; } - - /** - * victoria metrics query content - */ - @Data - @Builder - @AllArgsConstructor - @NoArgsConstructor - public static final class VictoriaMetricsQueryContent { - - private String status; - - private ContentData data; - - /** - * content data - */ - @Data - @AllArgsConstructor - @NoArgsConstructor - public static final class ContentData { - - private String resultType; - - private List result; - - /** - * content - */ - @Data - @AllArgsConstructor - @NoArgsConstructor - public static final class Content { - - /** - * metric contains metric name plus labels for a particular time series - */ - private Map metric; - - /** - * values contains raw sample values for the given time series - * value-timestamp - * [1700993195,"436960986"] - */ - private Object[] value; - - /** - * values contains raw sample values for the given time series - * value-timestamp list - * [[1700993195,"436960986"],[1700993195,"436960986"]...] - */ - private List values; - } - } - } } diff --git a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsDataStorage.java b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsDataStorage.java index 7dd905f9998..259a485067e 100644 --- a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsDataStorage.java +++ b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/vm/VictoriaMetricsDataStorage.java @@ -131,7 +131,8 @@ public void saveData(CollectRep.MetricsData metricsData) { return; } if (metricsData.getValuesList().isEmpty()) { - log.info("[warehouse victoria-metrics] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse victoria-metrics] flush metrics data {} {} {} is null, ignore.", + metricsData.getId(), metricsData.getApp(), metricsData.getMetrics()); return; } Map defaultLabels = new HashMap<>(8); @@ -141,16 +142,17 @@ public void saveData(CollectRep.MetricsData metricsData) { isPrometheusAuto = true; defaultLabels.remove(MONITOR_METRICS_KEY); defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp() - .substring(CommonConstants.PROMETHEUS_APP_PREFIX.length())); + .substring(CommonConstants.PROMETHEUS_APP_PREFIX.length())); } else { defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()); } defaultLabels.put(LABEL_KEY_INSTANCE, String.valueOf(metricsData.getId())); - + List fields = metricsData.getFieldsList(); Long[] timestamp = new Long[]{metricsData.getTime()}; Map fieldsValue = new HashMap<>(fields.size()); Map labels = new HashMap<>(fields.size()); + List contentList = new LinkedList<>(); for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { fieldsValue.clear(); labels.clear(); @@ -172,41 +174,53 @@ public void saveData(CollectRep.MetricsData metricsData) { if (entry.getKey() != null && entry.getValue() != null) { try { labels.putAll(defaultLabels); - String labelName = isPrometheusAuto ? metricsData.getMetrics() + String labelName = isPrometheusAuto ? metricsData.getMetrics() : metricsData.getMetrics() + SPILT + entry.getKey(); labels.put(LABEL_KEY_NAME, labelName); if (!isPrometheusAuto) { labels.put(MONITOR_METRIC_KEY, entry.getKey()); } VictoriaMetricsContent content = VictoriaMetricsContent.builder() - .metric(labels) + .metric(new HashMap<>(labels)) .values(new Double[]{entry.getValue()}) .timestamps(timestamp) .build(); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - if (StringUtils.hasText(victoriaMetricsProp.username()) - && StringUtils.hasText(victoriaMetricsProp.password())) { - String authStr = victoriaMetricsProp.username() + SignConstants.DOUBLE_MARK + victoriaMetricsProp.password(); - String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); - headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC - + SignConstants.BLANK + encodedAuth); - } - HttpEntity httpEntity = new HttpEntity<>(content, headers); - ResponseEntity responseEntity = restTemplate.postForEntity(victoriaMetricsProp.url() + IMPORT_PATH, - httpEntity, String.class); - if (responseEntity.getStatusCode().is2xxSuccessful()) { - log.debug("insert metrics data to victoria-metrics success. {}", content); - } else { - log.error("insert metrics data to victoria-metrics failed. {}", content); - } + contentList.add(content); } catch (Exception e) { - log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e); + log.error("combine metrics data error: {}.", e.getMessage(), e); } - + } } } + if (contentList.isEmpty()) { + log.info("[warehouse victoria-metrics] flush metrics data {} is empty, ignore.", metricsData.getId()); + return; + } + try { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + if (StringUtils.hasText(victoriaMetricsProp.username()) + && StringUtils.hasText(victoriaMetricsProp.password())) { + String authStr = victoriaMetricsProp.username() + ":" + victoriaMetricsProp.password(); + String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth); + } + StringBuilder stringBuilder = new StringBuilder(); + for (VictoriaMetricsContent content : contentList) { + stringBuilder.append(JsonUtil.toJson(content)).append("\n"); + } + HttpEntity httpEntity = new HttpEntity<>(stringBuilder.toString(), headers); + ResponseEntity responseEntity = restTemplate.postForEntity(victoriaMetricsProp.url() + IMPORT_PATH, + httpEntity, String.class); + if (responseEntity.getStatusCode().is2xxSuccessful()) { + log.debug("insert metrics data to victoria-metrics success."); + } else { + log.error("insert metrics data to victoria-metrics failed. {}", responseEntity.getBody()); + } + } catch (Exception e){ + log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e); + } } @Override @@ -334,14 +348,14 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str .queryParam("start", startTime) .queryParam("end", endTime) .build(true).toUri(); - ResponseEntity responseEntity = restTemplate.exchange(uri, - HttpMethod.GET, httpEntity, VictoriaMetricsQueryContent.class); + ResponseEntity responseEntity = restTemplate.exchange(uri, + HttpMethod.GET, httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { log.debug("query metrics data from victoria-metrics success. {}", uri); if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData().getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + List contents = responseEntity.getBody().getData().getResult(); + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -371,12 +385,12 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str .queryParam("end", endTime) .build(true).toUri(); responseEntity = restTemplate.exchange(uri, - HttpMethod.GET, httpEntity, VictoriaMetricsQueryContent.class); + HttpMethod.GET, httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData().getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + List contents = responseEntity.getBody().getData().getResult(); + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -406,12 +420,12 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str .queryParam("end", endTime) .build(true).toUri(); responseEntity = restTemplate.exchange(uri, - HttpMethod.GET, httpEntity, VictoriaMetricsQueryContent.class); + HttpMethod.GET, httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData().getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + List contents = responseEntity.getBody().getData().getResult(); + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -441,12 +455,12 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str .queryParam("end", endTime) .build(true).toUri(); responseEntity = restTemplate.exchange(uri, - HttpMethod.GET, httpEntity, VictoriaMetricsQueryContent.class); + HttpMethod.GET, httpEntity, PromQlQueryContent.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null && responseEntity.getBody().getData().getResult() != null) { - List contents = responseEntity.getBody().getData().getResult(); - for (VictoriaMetricsQueryContent.ContentData.Content content : contents) { + List contents = responseEntity.getBody().getData().getResult(); + for (PromQlQueryContent.ContentData.Content content : contents) { Map labels = content.getMetric(); labels.remove(LABEL_KEY_NAME); labels.remove(LABEL_KEY_JOB); @@ -499,59 +513,4 @@ public static final class VictoriaMetricsContent { */ private Long[] timestamps; } - - /** - * victoria metrics query content - */ - @Data - @Builder - @AllArgsConstructor - @NoArgsConstructor - public static final class VictoriaMetricsQueryContent { - - private String status; - - private ContentData data; - - /** - * content data - */ - @Data - @AllArgsConstructor - @NoArgsConstructor - public static final class ContentData { - - private String resultType; - - private List result; - - /** - * content - */ - @Data - @AllArgsConstructor - @NoArgsConstructor - public static final class Content { - - /** - * metric contains metric name plus labels for a particular time series - */ - private Map metric; - - /** - * values contains raw sample values for the given time series - * value-timestamp - * [1700993195,"436960986"] - */ - private Object[] value; - - /** - * values contains raw sample values for the given time series - * value-timestamp list - * [[1700993195,"436960986"],[1700993195,"436960986"]...] - */ - private List values; - } - } - } } diff --git a/home/docs/start/greptime-init.md b/home/docs/start/greptime-init.md index a04823dfc2d..b1c7ae88acc 100644 --- a/home/docs/start/greptime-init.md +++ b/home/docs/start/greptime-init.md @@ -28,9 +28,9 @@ After the installation you can check if the Docker version normally output at th 2. Install GreptimeDB with Docker ```shell - $ docker run -p 127.0.0.1:4000-4003:4000-4003 \ + $ docker run -d -p 127.0.0.1:4000-4003:4000-4003 \ v "$(pwd)/greptimedb:/tmp/greptimedb" \ - --name greptime --rm \ + --name greptime \ greptime/greptimedb:latest standalone start \ --http-addr 0.0.0.0:4000 \ --rpc-addr 0.0.0.0:4001 \ @@ -51,21 +51,19 @@ use```$ docker ps``` to check if the database started successfully ```yaml warehouse: store: - # disable jpa jpa: enabled: false - # enable greptime greptime: enabled: true grpc-endpoints: localhost:4001 - url: jdbc:mysql://localhost:4002/hertzbeat?connectionTimeZone=Asia/Shanghai&forceConnectionTimeZoneToSession=true - driver-class-name: com.mysql.cj.jdbc.Driver + http-endpoint: http://localhost:4000 + database: public username: greptime password: greptime - expire-time: 30d ``` - The default database is `hertzbeat` in the `url`, and it will be created automatically. The `expire-time` specifies the TTL(time-to-live) of the auto-created database, it's 30 days by default. + The default database is `public`, if you specify another database name, you need to create it in `greptimeDB` in advance. + eg: Create a database named `hertzbeat` with a validity period of 90 days SQL: `CREATE DATABASE IF NOT EXISTS hertzbeat WITH(ttl='90d')` 2. Restart HertzBeat diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/greptime-init.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/greptime-init.md index 7c40e2a8255..74a35c7f1c8 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/greptime-init.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/greptime-init.md @@ -27,9 +27,9 @@ Docker 工具自身的下载请参考 [Docker官网文档](https://docs.docker.c 2. Docker安装GreptimeDB ```shell - $ docker run -p 127.0.0.1:4000-4003:4000-4003 \ + $ docker run -d -p 127.0.0.1:4000-4003:4000-4003 \ -v "$(pwd)/greptimedb:/tmp/greptimedb" \ - --name greptime --rm \ + --name greptime \ greptime/greptimedb:latest standalone start \ --http-addr 0.0.0.0:4000 \ --rpc-addr 0.0.0.0:4001 \ @@ -52,20 +52,19 @@ Docker 工具自身的下载请参考 [Docker官网文档](https://docs.docker.c ```yaml warehouse: store: - # 关闭默认JPA jpa: enabled: false greptime: enabled: true grpc-endpoints: localhost:4001 - url: jdbc:mysql://localhost:4002/hertzbeat?connectionTimeZone=Asia/Shanghai&forceConnectionTimeZoneToSession=true - driver-class-name: com.mysql.cj.jdbc.Driver + http-endpoint: http://localhost:4000 + database: public username: greptime password: greptime - expire-time: 30d ``` - 默认数据库是 URL 中配置的 `hertzbeat` ，将自动创建。 `expire-time` 是自动创建的数据库的 TTL （数据过期）时间，默认为 30 天。 + 默认数据库是内置的 `public` ，若制定其它数据库名称，需要在 `greptimeDB` 提前创建。 + eg: 创建名称为 `hertzbeat` 数据有效期90天的数据库 SQL: `CREATE DATABASE IF NOT EXISTS hertzbeat WITH(ttl='90d')` 2. 重启 HertzBeat diff --git a/pom.xml b/pom.xml index 61336071683..05974f113fa 100644 --- a/pom.xml +++ b/pom.xml @@ -168,7 +168,7 @@ 2.23 3.0.5 3.0.0 - 0.7.3 + 0.9.1 8.0.33 diff --git a/script/application.yml b/script/application.yml index a96a85293fb..26d1ff74345 100644 --- a/script/application.yml +++ b/script/application.yml @@ -33,7 +33,10 @@ spring: exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration, org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration freemarker: enabled: false - + servlet: + multipart: + max-file-size: 100MB + max-request-size: 100MB management: health: @@ -81,7 +84,7 @@ spring: eclipselink: logging: level: SEVERE - + flyway: enabled: true clean-disabled: true @@ -89,13 +92,12 @@ spring: baseline-version: 1 locations: - classpath:db/migration/{vendor} - + mail: - # Mail server address, eg: qq-mailbox is smtp.qq.com, qq-exmail is smtp.exmail.qq.com + # Mail server address, eg: qq-mailbox is smtp.qq.com, qq-exmail is smtp.exmail.qq.com host: smtp.qq.com username: tancloud@qq.com # Attention this is not email account password, this requires an email authorization code - # 请注意此非邮箱账户密码 此需填写邮箱授权码 password: your-password # Mailbox smtp server port, eg: qq-mailbox is 465, qq-exmail is 587 port: 465 @@ -105,6 +107,8 @@ spring: socketFactoryClass: javax.net.ssl.SSLSocketFactory ssl: enable: true + starttls: + enable: false common: queue: @@ -119,14 +123,12 @@ common: warehouse: store: # store history metrics data, enable only one below - # 存储历史数据方式, 下方只能 enabled 启用一种方式 jpa: enabled: true # The maximum retention time for history records, after which records will be deleted expire-time: 1h # The maximum number of history records retained, if this number is exceeded, half of the data in this configuration item will be deleted # (please set this configuration reasonably as history records can affect performance when it is large) - # 历史数据的最大保留条数，超过此数量时，将会删除一半于此配量的数据（由于历史数据较大时会影响性能，请合理设置此配置） max-history-record-num: 6000 victoria-metrics: enabled: false @@ -141,7 +143,12 @@ warehouse: password: taosdata greptime: enabled: false - endpoint: localhost:4001 + grpc-endpoints: localhost:4001 + http-endpoint: http://localhost:4000 + # if you config other database name, you should create them first + database: public + username: greptime + password: greptime iot-db: enabled: false host: 127.0.0.1 @@ -158,16 +165,19 @@ warehouse: password: root expire-time: '30d' replication: 1 - - # store real-time metrics data, enable only one below - # 存储实时数据方式, 下方只能 enabled 启用一种方式 + # store real-time metrics data, enable only one below + real-time: memory: enabled: true init-size: 16 redis: enabled: false - host: 127.0.0.1 - port: 6379 + # redis mode: single, sentinel, cluster. Default is single + mode: single + # separate each address with comma when using cluster mode, eg: 127.0.0.1:6379,127.0.0.1:6380 + address: 127.0.0.1:6379 + # enter master name when using sentinel mode + masterName: mymaster password: 123456 # redis db index, default: DB0 db: 0 @@ -185,7 +195,7 @@ alerter: telegram-webhook-url: https://api.telegram.org/bot%s/sendMessage # discord discord-webhook-url: https://discord.com/api/v9/channels/%s/messages - # server酱 + # serverChan server-chan-webhook-url: https://sctapi.ftqq.com/%s.send # gotify gotify-webhook-url: http://127.0.0.1/message?token=%s @@ -194,3 +204,4 @@ scheduler: server: enabled: true port: 1158 + diff --git a/script/sureness.yml b/script/sureness.yml index cb4024c5a9c..9ebd316af1e 100644 --- a/script/sureness.yml +++ b/script/sureness.yml @@ -58,11 +58,15 @@ resourceRole: - /api/status/page/**===post===[admin,user] - /api/status/page/**===put===[admin,user] - /api/status/page/**===delete===[admin] + - /api/grafana/**===get===[admin,user,guest] + - /api/grafana/**===post===[admin,user] + - /api/grafana/**===put===[admin,user] + - /api/grafana/**===delete===[admin] - /api/bulletin/**===get===[admin,user,guest] - /api/bulletin/**===post===[admin,user] - /api/bulletin/**===put===[admin,user] - /api/bulletin/**===delete===[admin] - + # config the resource restful api that need bypass auth protection # rule: api===method # eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth. diff --git a/web-app/src/app/routes/bulletin/bulletin.component.html b/web-app/src/app/routes/bulletin/bulletin.component.html index b602a708b0e..d04f22ec21c 100644 --- a/web-app/src/app/routes/bulletin/bulletin.component.html +++ b/web-app/src/app/routes/bulletin/bulletin.component.html @@ -18,7 +18,7 @@ -->