Merge branch 'master' into issue432_aggrMethod_partial

.github/workflows/ci.yml

@@ -87,7 +87,7 @@ jobs:
     needs: unit-test
     services:
       mongodb:
-        image: mongo:4.4
+        image: mongo:6.0
         ports:
         - 27017:27017
     steps:

.readthedocs.yml

@@ -1,9 +1,13 @@
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
 mkdocs:
   configuration: mkdocs.yml
 
 python:
-  version: 3.8
   install:
     - requirements: doc/requirements.txt

CHANGES_NEXT_RELEASE

@@ -1,3 +1,3 @@
 - Add: aggrMethod accept multiple values separated by comma (#432, partial)
 - Add: aggrMethod 'all' to get all the possible aggregations (#432, partial)
-- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions)
+- Remove: RPM stuff

Changelog

@@ -0,0 +1,81 @@
+2.11.0 (November 28th, 2023)
+
+- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608)
+- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions)
+- Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions
+- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606)
+
+2.10.0 (November 2nd, 2022)
+
+- Fix: healthcheck over sth exposed port 
+- Fix: Dockerfile to include initial packages upgrade
+- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions)
+
+2.9.0 (March 25th, 2022)
+
+- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576)
+- Add: Docker healthcheck for STH API
+- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559)
+- Add: allow connect with mongo uri without auth
+- Fix: return 500 when DB is not connected (#570)
+- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0)
+- Fix: update logs about get raw and aggregated data (#556)
+- Update codebase to use ES6
+    -  Remove JSHint and jshint overrides
+    -  Add esLint using standard tamia presets
+    -  Replace var with let/const
+    -  Fix or disable eslint errors
+- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption
+- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567)
+- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile
+
+2.8.0 (May 27th, 2020)
+
+- Fix: Filetype issue fixed for hLimit and hOffset (#539)
+- Fix: Updated temporary csv filename for uniqueness (#532)
+- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life
+- Make optional PM2 usage in docker entrypoint
+- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE)
+
+2.7.0 (November 11th, 2019)
+
+- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118)
+- Add: CORS support (#500)
+- Fix: check header response before use it (CSV response has no header)
+- Fix: handler for reply a CSV: use new Hapi API (#513)
+- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514)
+- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues
+
+2.6.0 (June 4th, 2019)
+
+- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security
+- Add: logging feature to count number of requests attended and number of requests processed with error (#310)
+- Fix: race condition causes erroneous count to 0 in some cases (#493)
+
+2.5.0 (December 19th, 2018)
+
+- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions)
+- Add: use NodeJS 8 in Dockerfile
+- Add: use PM2 in Dockerfile
+- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0
+- Upgrade: logops dependence from ~1.0.5 to 2.1.0
+- Upgrade: joi dependence from ~5.1.0 to 14.0.6
+- Upgrade: boom dependence from ~2.7.2 to 7.2.2
+- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0
+- Upgrade: request development dependence from ~2.79.0 to 2.88.0
+- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0
+- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module)
+
+2.4.0 (August 16th, 2018)
+
+- Upgrade lodash dependence to 4.17.5
+- Upgrade mongodb dependence to 2.2.35
+- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431]
+- Fix: Content Disposition header encoding issue (#433)
+- Add count to query and related fiware-total-count header [#428]
+- Using precise dependencies (~=) in packages.json
+- Upgrade hapi to 11.1.3
+
+2.3.0 (October 18th, 2017)
+
+- FEATURE update node version to 4.8.4

README.md

@@ -2,11 +2,13 @@
 
 [![](https://nexus.lab.fiware.org/repository/raw/public/badges/chapters/core.svg)](https://www.fiware.org/developers/catalogue/)
 [![License](https://img.shields.io/github/license/telefonicaid/fiware-sth-comet.svg)](https://opensource.org/licenses/AGPL-3.0)
-[![Docker badge](https://img.shields.io/docker/pulls/fiware/sth-comet.svg)](https://hub.docker.com/r/fiware/sth-comet-ngsi/)
 [![](https://img.shields.io/badge/tag-fiware--sth-comet-orange.svg?logo=stackoverflow)](http://stackoverflow.com/questions/tagged/fiware-sth-comet)
 [![Support badge](https://img.shields.io/badge/support-askbot-yellowgreen.svg)](https://ask.fiware.org/questions/scope%3Aall/tags%3Asth-comet/)
 [![Join the chat at https://gitter.im/telefonicaid/fiware-sth-comet](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/telefonicaid/fiware-sth-comet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 <br/>
+[![Quay badge](https://img.shields.io/badge/quay.io-fiware%2Fsth--comet-grey?logo=red%20hat&labelColor=EE0000)](https://quay.io/repository/fiware/sth-comet)
+[![Docker badge](https://img.shields.io/badge/docker-telefonicaiot%2Ffiware--sth--comet-blue?logo=docker)](https://registry.hub.docker.com/r/telefonicaiot/fiware-sth-comet)
+<br/>
 [![Documentation badge](https://readthedocs.org/projects/fiware-sth-comet/badge/?version=latest)](https://fiware-sth-comet.readthedocs.io)
 [![CI](https://github.com/telefonicaid/fiware-sth-comet/workflows/CI/badge.svg)](https://github.com/telefonicaid/fiware-sth-comet/actions?query=workflow%3ACI)
 [![Coverage Status](https://coveralls.io/repos/github/telefonicaid/fiware-sth-comet/badge.svg?branch=master)](https://coveralls.io/github/telefonicaid/fiware-sth-comet?branch=master)
@@ -25,8 +27,8 @@ interfaces.
 This project is part of [FIWARE](https://www.fiware.org/). For more information check the FIWARE Catalogue entry for the
 [Core Context Management](https://github.com/Fiware/catalogue/tree/master/core).
 
-| :books: [Documentation](https://fiware-sth-comet.readthedocs.io) | :mortar_board: [Academy](https://fiware-academy.readthedocs.io/en/latest/core/sth-comet) | :whale: [Docker Hub](https://hub.docker.com/r/fiware/sth-comet/) | :dart: [Roadmap](https://github.com/telefonicaid/fiware-sth-comet/blob/master/doc/roadmap.md) |
-| --------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------- |
+| :books: [Documentation](https://fiware-sth-comet.readthedocs.io) | :mortar_board: [Academy](https://fiware-academy.readthedocs.io/en/latest/core/sth-comet) | <img style="height:1em" src="https://quay.io/static/img/quay_favicon.png"/> [quay.io](https://quay.io/repository/fiware/sth-comet) | :whale: [Docker Hub](https://registry.hub.docker.com/r/telefonicaiot/fiware-sth-comet) | :dart: [Roadmap](https://github.com/telefonicaid/fiware-sth-comet/blob/master/doc/roadmap.md) |
+| --------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | --- |
 
 
 # Contents

doc/manuals/db_indexes.md

@@ -16,6 +16,16 @@ It is recommended to create the following index in this collection:
 The performance difference can be dramatic for large sets of data. For instance, for a collection with around ~3000000
 query execution time can drop from 3 seconds to 1 millisecond.
 
+Since version 3.0.0 of cygnus an index named `cyg_raw_opt` is created in this way but depending on datamodel:
+
+| datamodel         | keys                                                             |
+| :---------------- | :--------------------------------------------------------------- |
+| dm-by-servicepath | recvTime, entityId, entityType, attrName, attrType, attrValue    |
+| dm-by-entity      | recvTime, attrName, attrType, attrValue                          |
+| dm-by-attribute   | recvTime, attrType, attrValue                                    |
+
+Note that datamodel others that the ones above are not allowed by Cygnus.
+
 ## Indexes in the aggregated data collection
 
 It is recommended to create the following index in this collection:
@@ -26,3 +36,13 @@ It is recommended to create the following index in this collection:
 
 The performance difference can be dramatic for large sets of data. For instance, for a collection with around ~6000000
 query execution time can drop from 28 seconds to 20 millisecond.
+
+Since version 3.0.0 of cygnus index named `cyg_agg_opt` is created in this way but depending on datamodel:
+
+| datamodel          | keys                                               |
+| :----------------- | :------------------------------------------------- |
+| dm-by-servicepath  | entityId, entityType, attrName, resolution, origin |
+| dm-by-entity       | attrName, resolution, origin                       |
+| dm-by-attribute    | resolution, origin                                 |
+
+Note that datamodel others that the ones above are not allowed by Cygnus.

doc/manuals/installation.md

@@ -3,8 +3,7 @@
 The STH component provides 3 alternatives for its installation:
 
 1.  Cloning the GitHub repository
-2.  Using a RPM package
-3.  Automatic deployment using Docker
+2.  Automatic deployment using Docker
 
 ## Cloning the GitHub repository
 
@@ -24,118 +23,6 @@ npm install
 
 The STH component server is ready to be started as a Node application.
 
-## Using a RPM package
-
-In the next sections we detail the steps to follow to install, update and remove the STH component using a RPM package.
-
-### Package generation
-
-**Prerequisites:** To generate the RPM package from the STH component sources it is needed to have the rpm build tools
-(`rpmbuild` executable), Node and the npm utilities, as well as an Internet connection to download the required Node
-modules.
-
-To generate the RPM package for the STH component, execute the following command from the root of the STH component:
-
-```bash
-./rpm/create-rpm.sh -v <version> -r <release>
-```
-
-If everything goes fine, a new RPM package will be created and stored in the following location:
-`./rpm/RPMS/x86_64/fiware-sth-comet-<version>-<release>.x86_64.rpm`.
-
-Execute the next command to get additional information about the RPM package creation script:
-
-```bash
-./rpm/create-rpm.sh -h
-```
-
-### Installation, upgrade and removal
-
-**Prerequisites:** Node is needed to install the generated STH component RPM package.
-
-To install or upgrade the STH component, execute:
-
-```bash
-sudo rpm -Uvh fiware-sth-comet-<version>-<release>.x86_64.rpm
-```
-
-After the installation, the following files and directories are created:
-
-```text
-/etc/init.d
-└── sth
-
-/etc/logrotate.d
-└── logrotate-sth-daily
-
-/var/log/sth
-
-/var/run/sth
-
-/opt/sth
-├── conf
-│   └── <empty> Here is where instances are configured
-├── node_modules
-│   └── <node modules directory structure and files>
-├── package.json
-└── src
-    └── <STH SW files>
-```
-
-To remove a previous STH component installation, execute:
-
-```bash
-sudo rpm -e fiware-sth-comet
-```
-
-### Configuration
-
-STH is able to start multiple instances using the
-[`sth`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/SOURCES/etc/init.d/sth) service script by
-adding and configuring certain files as detailed next.
-
-To start multiple instances, one configuration file per instance has to be included in the `/opt/sth/conf` directory. It
-is important to note that the default installation includes preconfigured instances.
-
-It is important to change the `STH_PORT` value included in the configuration files to a value not used by other STH
-instances/services. It is also a good practice to change the `LOG_FILE_NAME` value to avoid getting the logs from
-several instances mixed.
-
-The [`sth`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/SOURCES/etc/init.d/sth) service script
-includes the following operations:
-
--   **start**: if no `<instance>` (see below) is provided, the script starts an instance per configuration file found in
-    the `/opt/sth/conf` directory matching the `sth_*.conf` template. If `<instance>` is provided, a configuration file
-    named `sth_<instance>.conf` is searched in the `/opt/sth/conf` directory and the corresponding instance is started.
--   **stop**: if no `<instance>` (see below) is provided, the script stops all the instances by listing all pid files
-    under `/var/run/sth` matching the pattern `sth_*.pid`. If `<instance>` is provided, the scripts stops the instance
-    with the associated pid file `/var/run/sth/sth_<instance>.pid`
--   **status**: The status operation shows information about one or more running instances following the same procedure
-    detailed in the `stop` operation.
--   **restart**: The restart operation executes a `stop` operation followed by a `start` operation according to the
-    procedure detailed in those operations.
-
-```bash
-# start
-sudo /sbin/service sth start [<instance>]
-# stop
-sudo /sbin/service sth stop [<instance>]
-# status
-sudo /sbin/service sth status [<instance>]
-# restart
-sudo /sbin/service sth restart [<instance>]
-```
-
-An example
-[`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/EXAMPLES/sth_default.conf) file
-has been included in the component repository in GitHub to guide the STH instance configuration.
-
-<!-- textlint-disable write-good -->
-
-Last but not least, the STH process (a `node` process) runs the as `sth` user.
-
-<!-- textlint-enable write-good -->
-
 ## Automatic deployment using Docker
 
 To ease the testing and deployment of the STH component, there also exists Docker images hosted at the

doc/manuals/running.md

@@ -9,7 +9,7 @@ To run the STH server, please execute the following command from the directory w
 The STH component provides the user with 2 mechanisms to configure the component to the concrete needs of the user:
 
 -   Environment variables, which can be set assigning values to them or using the
-    [`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/EXAMPLES/sth_default.conf)
+    [`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/sth_default.conf)
     file if a packaged version of the STH component is used.
 -   The [`config.js`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/config.js) file located at the root
     of the STH component code, a JSON formatted file including the configuration properties.
@@ -101,6 +101,7 @@ The environment variables accepted by the script (for which there exists counter
 -   `PROOF_OF_LIFE_INTERVAL`: The time in seconds between proof of life logging messages informing that the server is up
     and running normally. Default value: "60". `PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL`: The time in seconds between
     processed requests statistics appear in the logs. Default value: "60".
+-   `CORS_ENABLED`: Boolean attribute (`true`|`false`) to enable cors configuration. Optional. Default value: "false".
 
 For example, to start the STH server listening on port 7777, connecting to a MongoDB instance listening on
 mymongo.com:27777 and without filtering out the empty results, use:

doc/requirements.txt

@@ -1,4 +1,4 @@
-mkdocs==1.2.3
+mkdocs==1.2.4
 Pygments==2.15.0
 Markdown==3.3.4
-jinja2==3.0.0
+jinja2==3.1.3

lib/configuration/sthConfiguration.js

@@ -223,7 +223,10 @@ if (ENV.STH_PORT && !isNaN(ENV.STH_PORT)) {
     );
 }
 
-if (config && config.cors && config.cors.enabled) {
+if (ENV.CORS_ENABLED && !(ENV.CORS_ENABLED == null)) {
+    module.exports.corsEnabled = JSON.parse(ENV.CORS_ENABLED);
+    sthLogger.info(module.exports.LOGGING_CONTEXT.STARTUP, 'CORS value set to: ' + module.exports.corsEnabled);
+} else if (config && config.cors && config.cors.enabled) {
     module.exports.corsEnabled = JSON.parse(config.cors.enabled);
     sthLogger.info(module.exports.LOGGING_CONTEXT.STARTUP, 'CORS value set to: ' + module.exports.corsEnabled);
 } else {

lib/database/model/sthDatabaseNameCodecTool.js

@@ -389,7 +389,7 @@ function copyDatabase(options, databaseName, callback) {
     const adminDB = sthDatabase.connection.admin();
     const listCommand = {
         listCollections: 1,
-        nameOnly: 1
+        nameOnly: true
     };
     var sourceDB = databaseName;
     var targetDB = options.encode

lib/database/model/sthDatabaseNameMapperTool.js

@@ -392,7 +392,7 @@ function copyDatabase(options, databaseName, callback) {
     const adminDB = sthDatabase.connection.admin();
     const listCommand = {
         listCollections: 1,
-        nameOnly: 1
+        nameOnly: true
     };
     var sourceDB = databaseName;
     var targetDB = options.encode

lib/database/sthDatabase.js

@@ -220,7 +220,7 @@ function isAggregated(collectionName) {
 function setTTLPolicy(collection) {
     // Set the TTL policy if required
     if (sthConfig.TRUNCATION_EXPIRE_AFTER_SECONDS > 0) {
-        if (!isAggregated(collection)) {
+        if (!isAggregated(collection.collectionName)) {
             if (sthConfig.TRUNCATION_SIZE === 0) {
                 collection.ensureIndex(
                     {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "fiware-sth-comet",
   "description": "The Short Time Historic (STH, aka. Comet) is a component of the FIWARE ecosystem in charge of providing aggregated time series information about the evolution in time of entity attribute values registered using the Orion Context Broker, an implementation of the publish/subscribe context management system exposing NGSI9 and NGSI10 interfaces.",
-  "version": "2.10.0-next",
+  "version": "2.11.0-next",
   "licence": "AGPL-3.0",
   "homepage": "https://github.com/telefonicaid/fiware-sth-comet",
   "author": {