diff --git a/Changelog b/Changelog index 72f777b..a524ab9 100644 --- a/Changelog +++ b/Changelog @@ -1,82 +1,82 @@ -2.11.0 (November 28th, 2023) - -- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) -- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) -- Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions - -- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) - -2.10.0 (November 2nd, 2022) - -- Fix: healthcheck over sth exposed port -- Fix: Dockerfile to include initial packages upgrade -- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) - -2.9.0 (March 25th, 2022) - -- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) -- Add: Docker healthcheck for STH API -- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) -- Add: allow connect with mongo uri without auth -- Fix: return 500 when DB is not connected (#570) -- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) -- Fix: update logs about get raw and aggregated data (#556) -- Update codebase to use ES6 - - Remove JSHint and jshint overrides - - Add esLint using standard tamia presets - - Replace var with let/const - - Fix or disable eslint errors -- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption -- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) -- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile - -2.8.0 (May 27th, 2020) - -- Fix: Filetype issue fixed for hLimit and hOffset (#539) -- Fix: Updated temporary csv filename for uniqueness (#532) -- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life -- Make optional PM2 usage in docker entrypoint -- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) - -2.7.0 (November 11th, 2019) - -- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) -- Add: CORS support (#500) -- Fix: check header response before use it (CSV response has no header) -- Fix: handler for reply a CSV: use new Hapi API (#513) -- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) -- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues - -2.6.0 (June 4th, 2019) - -- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security -- Add: logging feature to count number of requests attended and number of requests processed with error (#310) -- Fix: race condition causes erroneous count to 0 in some cases (#493) - -2.5.0 (December 19th, 2018) - -- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) -- Add: use NodeJS 8 in Dockerfile -- Add: use PM2 in Dockerfile -- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 -- Upgrade: logops dependence from ~1.0.5 to 2.1.0 -- Upgrade: joi dependence from ~5.1.0 to 14.0.6 -- Upgrade: boom dependence from ~2.7.2 to 7.2.2 -- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 -- Upgrade: request development dependence from ~2.79.0 to 2.88.0 -- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 -- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) - -2.4.0 (August 16th, 2018) - -- Upgrade lodash dependence to 4.17.5 -- Upgrade mongodb dependence to 2.2.35 -- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] -- Fix: Content Disposition header encoding issue (#433) -- Add count to query and related fiware-total-count header [#428] -- Using precise dependencies (~=) in packages.json -- Upgrade hapi to 11.1.3 - -2.3.0 (October 18th, 2017) - +2.11.0 (November 28th, 2023) + +- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) +- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) +- Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions + +- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) + +2.10.0 (November 2nd, 2022) + +- Fix: healthcheck over sth exposed port +- Fix: Dockerfile to include initial packages upgrade +- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) + +2.9.0 (March 25th, 2022) + +- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) +- Add: Docker healthcheck for STH API +- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) +- Add: allow connect with mongo uri without auth +- Fix: return 500 when DB is not connected (#570) +- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) +- Fix: update logs about get raw and aggregated data (#556) +- Update codebase to use ES6 + - Remove JSHint and jshint overrides + - Add esLint using standard tamia presets + - Replace var with let/const + - Fix or disable eslint errors +- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption +- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) +- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile + +2.8.0 (May 27th, 2020) + +- Fix: Filetype issue fixed for hLimit and hOffset (#539) +- Fix: Updated temporary csv filename for uniqueness (#532) +- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life +- Make optional PM2 usage in docker entrypoint +- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) + +2.7.0 (November 11th, 2019) + +- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) +- Add: CORS support (#500) +- Fix: check header response before use it (CSV response has no header) +- Fix: handler for reply a CSV: use new Hapi API (#513) +- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) +- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues + +2.6.0 (June 4th, 2019) + +- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security +- Add: logging feature to count number of requests attended and number of requests processed with error (#310) +- Fix: race condition causes erroneous count to 0 in some cases (#493) + +2.5.0 (December 19th, 2018) + +- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) +- Add: use NodeJS 8 in Dockerfile +- Add: use PM2 in Dockerfile +- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 +- Upgrade: logops dependence from ~1.0.5 to 2.1.0 +- Upgrade: joi dependence from ~5.1.0 to 14.0.6 +- Upgrade: boom dependence from ~2.7.2 to 7.2.2 +- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 +- Upgrade: request development dependence from ~2.79.0 to 2.88.0 +- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 +- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) + +2.4.0 (August 16th, 2018) + +- Upgrade lodash dependence to 4.17.5 +- Upgrade mongodb dependence to 2.2.35 +- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] +- Fix: Content Disposition header encoding issue (#433) +- Add count to query and related fiware-total-count header [#428] +- Using precise dependencies (~=) in packages.json +- Upgrade hapi to 11.1.3 + +2.3.0 (October 18th, 2017) + - FEATURE update node version to 4.8.4 \ No newline at end of file diff --git a/sth_default.conf b/sth_default.conf index fbcb1ad..ab4f314 100644 --- a/sth_default.conf +++ b/sth_default.conf @@ -1,157 +1,157 @@ -#!/usr/bin/env bash - -# Ansible managed: modified by IoT devops team - -# Short Term History configuration variables. -# STH is configured defining the folowing variables. -# All variables are optional, so, if one of this variables is not defined -# default value is used. - -# See https://github.com/telefonicaid/IoT-STH#running-the-sth-server -# for further information - -# Note: is important to export environment variables with export - -# The host where the STH server will be started. -# Default value: "localhost". -export STH_HOST="0.0.0.0" - -# The port where the STH server will be listening. -# Default value: 8666. -export STH_PORT="8666" - -# The service to be used if not sent by the Orion Context Broker in the notifications. -# Default value: "testservice". -export DEFAULT_SERVICE="testservice" - -# The service path to be used if not sent by the Orion Context Broker in the notifications. -# Default value: "/testservicepath". -export DEFAULT_SERVICE_PATH="/testservicepath" - -# A flag indicating if the empty results should be removed from the response. -# Default value: "true". -export FILTER_OUT_EMPTY="true" - -# Array of resolutions the STH component should aggregate values for. -# Valid resolution values are: 'month', 'day', 'hour', 'minute' and 'second' -export AGGREGATION_BY="['day', 'hour', 'minute']" - -# The temporal directory to be used by the STH to temporarily store files (right now .csv files) before returning them. -# Default value: "temp". -export TEMPORAL_DIR="temp" - -# The STH component supports 3 alternative data models when storing the raw and aggregated data -# into the database: 1) one collection per attribute, 2) one collection per entity and -# 3) one collection per service path. The possible values are: "collection-per-attribute", -# "collection-per-entity" and "collection-per-service-path" respectively. Default value: -# "collection-per-entity". -export DATA_MODEL='collection-per-entity' - -# The username to use for the database connection. -# Default value: "". -export DB_USERNAME="" - -# The password to use for the database connection. -# Default value: "". -export DB_PASSWORD="" - -# The URI to use for the database connection. -# This does not include the 'mongo://' protocol part. -# Two or more host are comma separated. -# Default value: "localhost:27017". -export DB_URI="localhost:27017" - -# The name of the MongoDB replica set. -# Default value: "". -export REPLICA_SET="" - -# The prefix to be added to the service for the creation of the databases. -# More information below. -# Default value: "sth_". -export DB_PREFIX="sth_" - -# The prefix to be added to the collections in the databases. More information below. -# Default value: "sth_". -export COLLECTION_PREFIX="sth_" - -# The default MongoDB pool size of database connections. -# Default value: "5". -export POOL_SIZE="5" - -# The write concern policy to apply when writing data to the MongoDB database. -# Write concenr more info: http://docs.mongodb.org/manual/core/write-concern/ -# Default value: "1". -export WRITE_CONCERN="1" - -# Flag indicating if the raw and/or aggregated data should be persisted. -# Valid values are: "only-raw", "only-aggregated" and "both". -# Default value: "both". -export SHOULD_STORE="both" - -# Data from the raw and aggregated data collections will be removed if older than the value specified in seconds. -# In case of raw data the reference time is the one stored in the recvTime property whereas in the case of the -# aggregated data the reference of time is the one stored in the _id.origin property. -# Set the value to 0 not to apply this time-based truncation policy. Default value: "0". -export TRUNCATION_EXPIRE_AFTER_SECONDS="0" - -# The oldest raw data (according to insertion time) will be removed if the size of the raw data collection gets bigger -# than the value specified in bytes. -# Set the value to 0 not to apply this truncation policy. Take into consideration than the "size" configuration -# parameter is mandatory in case size collection truncation is desired as required by MongoDB. -# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not -# currently support updating documents in capped collections which increase the size of the documents. -# Notice also that in case of the raw data, the size-based truncation policy takes precedence over the TTL one. -# More concretely, if "size" is set, the value of "exporeAfterSeconds" is ignored for the raw data collections since -# currently MongoDB does not support TTL in capped collections. -# Default value: "0". -export TRUNCATION_SIZE="0" - -# The oldest raw data (according to insertion time) will be removed if the number of documents in the raw data -# collections goes beyond the specified value. Set the value to 0 not to apply this truncation policy. -# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not -# currently support updating documents in capped collections which increase the size of the documents. -# Default value: "0". -export TRUNCATION_MAX="0" - -# Attribute values to one or more blank spaces should be ignored and not processed either as raw data or for the -# aggregated computations -# Default value: "true". -export IGNORE_BLANK_SPACES="true" - -# Database and collection names have to respect the limitations imposed by MongoDB (see -# https://docs.mongodb.com/manual/reference/limits/). To it, the STH provides 2 main mechanisms: mappings and -# encoding which can be configured using the next 2 configuration parameters. -# The mappings mechanism will substitute the original services, service paths, entity and attribute names and types -# by the ones defined in the configuration file. If enabled, the mappings mechanism will be the one applied. -export NAME_MAPPING ="{\"enabled\": \"false\", \"configFile\": \"./name-mapping.json\"}" - -# Database and collection names should be encoded to avoid the restrictions imposed by MongoDB and stated at -# https://docs.mongodb.com/manual/reference/limits/ -# The encoding criteria is the following one: -# 1. Encode the forbidden characters using an escaping character (x) and a numerical Unicode code for each character. -# For instance, the / character will be encoded as x002f. -# 2. Database and collection names already using the above encoding must be escaped prepending another x, -# for instance, the text x002a will be encoded as xx002a. -# 3. The uppercase characters included in database names will be encoded using the mechanism stated in 1. -# 4. Collection names starting with 'system.' will be encoded as 'xsystem.'. For instance, system.myData will be -# encoded as xsystem.myData. -# Default value: "true" (although we will set it to false until the Cygnus counterpart is ready and landed) -export NAME_ENCODING="false" - -# The log level to use. Possible values are: "DEBUG", "INFO", "WARN", "ERROR" and "FATAL". -# Default value: "INFO". -export LOGOPS_LEVEL="INFO" - -# The log format to use. Possible values are: "json" (writes logs as JSON), "dev" (for development, used when the -# NODE_ENV variable is set to 'development') and "pipe" (write logs separating the contents with pipes). -# Default value: "pipe". -export LOGOPS_FORMAT="pipe" - -# The time in seconds between proof of life logging messages informing that -# the server is up and running normally. -# Default value: "60". -export PROOF_OF_LIFE_INTERVAL="60" - -# The time in seconds between processed requests statistics appear in the logs -# Default value: "60". -export PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL="60" +#!/usr/bin/env bash + +# Ansible managed: modified by IoT devops team + +# Short Term History configuration variables. +# STH is configured defining the folowing variables. +# All variables are optional, so, if one of this variables is not defined +# default value is used. + +# See https://github.com/telefonicaid/IoT-STH#running-the-sth-server +# for further information + +# Note: is important to export environment variables with export + +# The host where the STH server will be started. +# Default value: "localhost". +export STH_HOST="0.0.0.0" + +# The port where the STH server will be listening. +# Default value: 8666. +export STH_PORT="8666" + +# The service to be used if not sent by the Orion Context Broker in the notifications. +# Default value: "testservice". +export DEFAULT_SERVICE="testservice" + +# The service path to be used if not sent by the Orion Context Broker in the notifications. +# Default value: "/testservicepath". +export DEFAULT_SERVICE_PATH="/testservicepath" + +# A flag indicating if the empty results should be removed from the response. +# Default value: "true". +export FILTER_OUT_EMPTY="true" + +# Array of resolutions the STH component should aggregate values for. +# Valid resolution values are: 'month', 'day', 'hour', 'minute' and 'second' +export AGGREGATION_BY="['day', 'hour', 'minute']" + +# The temporal directory to be used by the STH to temporarily store files (right now .csv files) before returning them. +# Default value: "temp". +export TEMPORAL_DIR="temp" + +# The STH component supports 3 alternative data models when storing the raw and aggregated data +# into the database: 1) one collection per attribute, 2) one collection per entity and +# 3) one collection per service path. The possible values are: "collection-per-attribute", +# "collection-per-entity" and "collection-per-service-path" respectively. Default value: +# "collection-per-entity". +export DATA_MODEL='collection-per-entity' + +# The username to use for the database connection. +# Default value: "". +export DB_USERNAME="" + +# The password to use for the database connection. +# Default value: "". +export DB_PASSWORD="" + +# The URI to use for the database connection. +# This does not include the 'mongo://' protocol part. +# Two or more host are comma separated. +# Default value: "localhost:27017". +export DB_URI="localhost:27017" + +# The name of the MongoDB replica set. +# Default value: "". +export REPLICA_SET="" + +# The prefix to be added to the service for the creation of the databases. +# More information below. +# Default value: "sth_". +export DB_PREFIX="sth_" + +# The prefix to be added to the collections in the databases. More information below. +# Default value: "sth_". +export COLLECTION_PREFIX="sth_" + +# The default MongoDB pool size of database connections. +# Default value: "5". +export POOL_SIZE="5" + +# The write concern policy to apply when writing data to the MongoDB database. +# Write concenr more info: http://docs.mongodb.org/manual/core/write-concern/ +# Default value: "1". +export WRITE_CONCERN="1" + +# Flag indicating if the raw and/or aggregated data should be persisted. +# Valid values are: "only-raw", "only-aggregated" and "both". +# Default value: "both". +export SHOULD_STORE="both" + +# Data from the raw and aggregated data collections will be removed if older than the value specified in seconds. +# In case of raw data the reference time is the one stored in the recvTime property whereas in the case of the +# aggregated data the reference of time is the one stored in the _id.origin property. +# Set the value to 0 not to apply this time-based truncation policy. Default value: "0". +export TRUNCATION_EXPIRE_AFTER_SECONDS="0" + +# The oldest raw data (according to insertion time) will be removed if the size of the raw data collection gets bigger +# than the value specified in bytes. +# Set the value to 0 not to apply this truncation policy. Take into consideration than the "size" configuration +# parameter is mandatory in case size collection truncation is desired as required by MongoDB. +# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not +# currently support updating documents in capped collections which increase the size of the documents. +# Notice also that in case of the raw data, the size-based truncation policy takes precedence over the TTL one. +# More concretely, if "size" is set, the value of "exporeAfterSeconds" is ignored for the raw data collections since +# currently MongoDB does not support TTL in capped collections. +# Default value: "0". +export TRUNCATION_SIZE="0" + +# The oldest raw data (according to insertion time) will be removed if the number of documents in the raw data +# collections goes beyond the specified value. Set the value to 0 not to apply this truncation policy. +# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not +# currently support updating documents in capped collections which increase the size of the documents. +# Default value: "0". +export TRUNCATION_MAX="0" + +# Attribute values to one or more blank spaces should be ignored and not processed either as raw data or for the +# aggregated computations +# Default value: "true". +export IGNORE_BLANK_SPACES="true" + +# Database and collection names have to respect the limitations imposed by MongoDB (see +# https://docs.mongodb.com/manual/reference/limits/). To it, the STH provides 2 main mechanisms: mappings and +# encoding which can be configured using the next 2 configuration parameters. +# The mappings mechanism will substitute the original services, service paths, entity and attribute names and types +# by the ones defined in the configuration file. If enabled, the mappings mechanism will be the one applied. +export NAME_MAPPING ="{\"enabled\": \"false\", \"configFile\": \"./name-mapping.json\"}" + +# Database and collection names should be encoded to avoid the restrictions imposed by MongoDB and stated at +# https://docs.mongodb.com/manual/reference/limits/ +# The encoding criteria is the following one: +# 1. Encode the forbidden characters using an escaping character (x) and a numerical Unicode code for each character. +# For instance, the / character will be encoded as x002f. +# 2. Database and collection names already using the above encoding must be escaped prepending another x, +# for instance, the text x002a will be encoded as xx002a. +# 3. The uppercase characters included in database names will be encoded using the mechanism stated in 1. +# 4. Collection names starting with 'system.' will be encoded as 'xsystem.'. For instance, system.myData will be +# encoded as xsystem.myData. +# Default value: "true" (although we will set it to false until the Cygnus counterpart is ready and landed) +export NAME_ENCODING="false" + +# The log level to use. Possible values are: "DEBUG", "INFO", "WARN", "ERROR" and "FATAL". +# Default value: "INFO". +export LOGOPS_LEVEL="INFO" + +# The log format to use. Possible values are: "json" (writes logs as JSON), "dev" (for development, used when the +# NODE_ENV variable is set to 'development') and "pipe" (write logs separating the contents with pipes). +# Default value: "pipe". +export LOGOPS_FORMAT="pipe" + +# The time in seconds between proof of life logging messages informing that +# the server is up and running normally. +# Default value: "60". +export PROOF_OF_LIFE_INTERVAL="60" + +# The time in seconds between processed requests statistics appear in the logs +# Default value: "60". +export PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL="60"