From 481c301892ca1e7be4c3d19360550990a548c5b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 12:20:42 +0100 Subject: [PATCH 1/4] REMOVE rpm stuff --- CHANGES_NEXT_RELEASE | 1 + Changelog | 81 +++++ doc/manuals/installation.md | 115 +------ doc/manuals/running.md | 2 +- rpm/SPECS/sth.spec | 234 ------------- rpm/create-rpm.sh | 89 ----- rpm/get_version_string.sh | 86 ----- .../sth_default.conf => sth_default.conf | 314 +++++++++--------- 8 files changed, 241 insertions(+), 681 deletions(-) create mode 100644 Changelog delete mode 100644 rpm/SPECS/sth.spec delete mode 100755 rpm/create-rpm.sh delete mode 100755 rpm/get_version_string.sh rename rpm/EXAMPLES/sth_default.conf => sth_default.conf (97%) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29b..49d9d404 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Remove: RPM stuff \ No newline at end of file diff --git a/Changelog b/Changelog new file mode 100644 index 00000000..b962c432 --- /dev/null +++ b/Changelog @@ -0,0 +1,81 @@ +2.11.0 (November 28th, 2023) + +- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) +- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) +- Fix: the way in which list collection is done at MongoDB (ensure�ing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions +- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) + +2.10.0 (November 2nd, 2022) + +- Fix: healthcheck over sth exposed port +- Fix: Dockerfile to include initial packages upgrade +- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) + +2.9.0 (March 25th, 2022) + +- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) +- Add: Docker healthcheck for STH API +- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) +- Add: allow connect with mongo uri without auth +- Fix: return 500 when DB is not connected (#570) +- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) +- Fix: update logs about get raw and aggregated data (#556) +- Update codebase to use ES6 + - Remove JSHint and jshint overrides + - Add esLint using standard tamia presets + - Replace var with let/const + - Fix or disable eslint errors +- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption +- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) +- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile + +2.8.0 (May 27th, 2020) + +- Fix: Filetype issue fixed for hLimit and hOffset (#539) +- Fix: Updated temporary csv filename for uniqueness (#532) +- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life +- Make optional PM2 usage in docker entrypoint +- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) + +2.7.0 (November 11th, 2019) + +- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) +- Add: CORS support (#500) +- Fix: check header response before use it (CSV response has no header) +- Fix: handler for reply a CSV: use new Hapi API (#513) +- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) +- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues + +2.6.0 (June 4th, 2019) + +- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security +- Add: logging feature to count number of requests attended and number of requests processed with error (#310) +- Fix: race condition causes erroneous count to 0 in some cases (#493) + +2.5.0 (December 19th, 2018) + +- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) +- Add: use NodeJS 8 in Dockerfile +- Add: use PM2 in Dockerfile +- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 +- Upgrade: logops dependence from ~1.0.5 to 2.1.0 +- Upgrade: joi dependence from ~5.1.0 to 14.0.6 +- Upgrade: boom dependence from ~2.7.2 to 7.2.2 +- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 +- Upgrade: request development dependence from ~2.79.0 to 2.88.0 +- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 +- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) + +2.4.0 (August 16th, 2018) + +- Upgrade lodash dependence to 4.17.5 +- Upgrade mongodb dependence to 2.2.35 +- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] +- Fix: Content Disposition header encoding issue (#433) +- Add count to query and related fiware-total-count header [#428] +- Using precise dependencies (~=) in packages.json +- Upgrade hapi to 11.1.3 + +2.3.0 (October 18th, 2017) + +- FEATURE update node version to 4.8.4 \ No newline at end of file diff --git a/doc/manuals/installation.md b/doc/manuals/installation.md index 75ce7060..f1230992 100644 --- a/doc/manuals/installation.md +++ b/doc/manuals/installation.md @@ -3,8 +3,7 @@ The STH component provides 3 alternatives for its installation: 1. Cloning the GitHub repository -2. Using a RPM package -3. Automatic deployment using Docker +2. Automatic deployment using Docker ## Cloning the GitHub repository @@ -24,118 +23,6 @@ npm install The STH component server is ready to be started as a Node application. -## Using a RPM package - -In the next sections we detail the steps to follow to install, update and remove the STH component using a RPM package. - -### Package generation - -**Prerequisites:** To generate the RPM package from the STH component sources it is needed to have the rpm build tools -(`rpmbuild` executable), Node and the npm utilities, as well as an Internet connection to download the required Node -modules. - -To generate the RPM package for the STH component, execute the following command from the root of the STH component: - -```bash -./rpm/create-rpm.sh -v -r -``` - -If everything goes fine, a new RPM package will be created and stored in the following location: -`./rpm/RPMS/x86_64/fiware-sth-comet--.x86_64.rpm`. - -Execute the next command to get additional information about the RPM package creation script: - -```bash -./rpm/create-rpm.sh -h -``` - -### Installation, upgrade and removal - -**Prerequisites:** Node is needed to install the generated STH component RPM package. - -To install or upgrade the STH component, execute: - -```bash -sudo rpm -Uvh fiware-sth-comet--.x86_64.rpm -``` - -After the installation, the following files and directories are created: - -```text -/etc/init.d -└── sth - -/etc/logrotate.d -└── logrotate-sth-daily - -/var/log/sth - -/var/run/sth - -/opt/sth -├── conf -│ └── Here is where instances are configured -├── node_modules -│ └── -├── package.json -└── src - └── -``` - -To remove a previous STH component installation, execute: - -```bash -sudo rpm -e fiware-sth-comet -``` - -### Configuration - -STH is able to start multiple instances using the -[`sth`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/SOURCES/etc/init.d/sth) service script by -adding and configuring certain files as detailed next. - -To start multiple instances, one configuration file per instance has to be included in the `/opt/sth/conf` directory. It -is important to note that the default installation includes preconfigured instances. - -It is important to change the `STH_PORT` value included in the configuration files to a value not used by other STH -instances/services. It is also a good practice to change the `LOG_FILE_NAME` value to avoid getting the logs from -several instances mixed. - -The [`sth`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/SOURCES/etc/init.d/sth) service script -includes the following operations: - -- **start**: if no `` (see below) is provided, the script starts an instance per configuration file found in - the `/opt/sth/conf` directory matching the `sth_*.conf` template. If `` is provided, a configuration file - named `sth_.conf` is searched in the `/opt/sth/conf` directory and the corresponding instance is started. -- **stop**: if no `` (see below) is provided, the script stops all the instances by listing all pid files - under `/var/run/sth` matching the pattern `sth_*.pid`. If `` is provided, the scripts stops the instance - with the associated pid file `/var/run/sth/sth_.pid` -- **status**: The status operation shows information about one or more running instances following the same procedure - detailed in the `stop` operation. -- **restart**: The restart operation executes a `stop` operation followed by a `start` operation according to the - procedure detailed in those operations. - -```bash -# start -sudo /sbin/service sth start [] -# stop -sudo /sbin/service sth stop [] -# status -sudo /sbin/service sth status [] -# restart -sudo /sbin/service sth restart [] -``` - -An example -[`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/EXAMPLES/sth_default.conf) file -has been included in the component repository in GitHub to guide the STH instance configuration. - - - -Last but not least, the STH process (a `node` process) runs the as `sth` user. - - - ## Automatic deployment using Docker To ease the testing and deployment of the STH component, there also exists Docker images hosted at the diff --git a/doc/manuals/running.md b/doc/manuals/running.md index 67ab745d..4d19b02c 100644 --- a/doc/manuals/running.md +++ b/doc/manuals/running.md @@ -9,7 +9,7 @@ To run the STH server, please execute the following command from the directory w The STH component provides the user with 2 mechanisms to configure the component to the concrete needs of the user: - Environment variables, which can be set assigning values to them or using the - [`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/rpm/EXAMPLES/sth_default.conf) + [`sth_default.conf`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/sth_default.conf) file if a packaged version of the STH component is used. - The [`config.js`](https://github.com/telefonicaid/fiware-sth-comet/blob/master/config.js) file located at the root of the STH component code, a JSON formatted file including the configuration properties. diff --git a/rpm/SPECS/sth.spec b/rpm/SPECS/sth.spec deleted file mode 100644 index 2623e758..00000000 --- a/rpm/SPECS/sth.spec +++ /dev/null @@ -1,234 +0,0 @@ -Summary: Short Time Historic (STH) -Name: fiware-sth-comet -Version: %{_product_version} -Release: %{_product_release} -License: AGPLv3 -BuildRoot: %{_topdir}/BUILDROOT/ -BuildArch: x86_64 -Requires: nodejs >= 0.10.42 -Requires: logrotate -Requires(post): /sbin/chkconfig, /usr/sbin/useradd, npm -Requires(preun): /sbin/chkconfig, /sbin/service -Requires(postun): /sbin/service -Group: Applications/Engineering -Vendor: Telefonica I+D -BuildRequires: npm - -# HISTORY CHANGES -# CJMM-DevOps 2016/03/21 - -%description -The Short Time Historic (STH, aka. Comet) is a component of the FIWARE ecosystem in charge of providing aggregated time series information about the evolution in time of entity attribute values registered using the Orion Context Broker, an implementation of the publish/subscribe context management system exposing NGSI9 and NGSI10 interfaces. - -# System folders -%define _srcdir $RPM_BUILD_ROOT/../../.. -%define _project_user sth -%define _service_name sth -%define _install_dir /opt/sth -%define _sth_log_dir /var/log/sth -%define _sth_pid_dir /var/run/sth - -# RPM Building folder -%define _build_root_project %{buildroot}%{_install_dir} -# -------------------------------------------------------------------------------------------- # -# prep section, setup macro: -# -------------------------------------------------------------------------------------------- # -%prep -echo "[INFO] Preparing installation" -# Create rpm/BUILDROOT folder -rm -Rf $RPM_BUILD_ROOT && mkdir -p $RPM_BUILD_ROOT -[ -d %{_build_root_project} ] || mkdir -p %{_build_root_project} - -# Copy src files -cp -R %{_srcdir}/lib \ - %{_srcdir}/bin \ - %{_srcdir}/config.js \ - %{_srcdir}/package.json \ - %{_build_root_project} - -[ -f %{_srcdir}/npm-shrinkwrap.json ] && /bin/cp %{_srcdir}/npm-shrinkwrap.json %{_build_root_project} - -cp -R %{_topdir}/SOURCES/etc %{buildroot} - -# Create conf dir -mkdir -p %{_build_root_project}/conf - -# -------------------------------------------------------------------------------------------- # -# Build section: -# -------------------------------------------------------------------------------------------- # -%build -echo "[INFO] Building RPM" -cd %{_build_root_project} - -# Only production modules. We have found that --force is required to make this work for Node v8 -rm -fR node_modules/ -npm cache clear --force -npm install --production - -# -------------------------------------------------------------------------------------------- # -# pre-install section: -# -------------------------------------------------------------------------------------------- # -%pre -echo "[INFO] Creating %{_project_user} user" -grep ^%{_project_user}: /etc/passwd -RET_VAL=$? -if [ "$RET_VAL" != "0" ]; then - /usr/sbin/useradd -s "/bin/bash" -d %{_install_dir} %{_project_user} - RET_VAL=$? - if [ "$RET_VAL" != "0" ]; then - echo "[ERROR] Unable create %{_project_user} user" \ - exit $RET_VAL - fi - -fi - -# -------------------------------------------------------------------------------------------- # -# post-install section: -# -------------------------------------------------------------------------------------------- # -%post -echo "[INFO] Configuring application" - - echo "[INFO] Creating the home Short Time Historic (STH) directory" - mkdir -p _install_dir - echo "[INFO] Creating log & run directory" - mkdir -p %{_sth_log_dir} - chown -R %{_project_user}:%{_project_user} %{_sth_log_dir} - chown -R %{_project_user}:%{_project_user} _install_dir - chmod g+s %{_sth_log_dir} - setfacl -d -m g::rwx %{_sth_log_dir} - setfacl -d -m o::rx %{_sth_log_dir} - - mkdir -p %{_sth_pid_dir} - chown -R %{_project_user}:%{_project_user} %{_sth_pid_dir} - chown -R %{_project_user}:%{_project_user} _install_dir - chmod g+s %{_sth_pid_dir} - setfacl -d -m g::rwx %{_sth_pid_dir} - setfacl -d -m o::rx %{_sth_pid_dir} - - echo "[INFO] Configuring application service" - cd /etc/init.d - chkconfig --add %{_service_name} - -echo "Done" - -# -------------------------------------------------------------------------------------------- # -# pre-uninstall section: -# -------------------------------------------------------------------------------------------- # -%preun - -echo "[INFO] stoping service %{_service_name}" -service %{_service_name} stop &> /dev/null - -if [ $1 == 0 ]; then - - echo "[INFO] Removing application log files" - # Log - [ -d %{_sth_log_dir} ] && rm -rfv %{_sth_log_dir} - - echo "[INFO] Removing application run files" - # Log - [ -d %{_sth_pid_dir} ] && rm -rfv %{_sth_pid_dir} - - echo "[INFO] Removing application files" - # Installed files - [ -d %{_install_dir} ] && rm -rfv %{_install_dir} - - echo "[INFO] Removing application user" - userdel -fr %{_project_user} - - echo "[INFO] Removing application service" - chkconfig --del %{_service_name} - rm -Rf /etc/init.d/%{_service_name} - echo "Done" -fi - -# -------------------------------------------------------------------------------------------- # -# post-uninstall section: -# clean section: -# -------------------------------------------------------------------------------------------- # -%postun -%clean -rm -rf $RPM_BUILD_ROOT - -# -------------------------------------------------------------------------------------------- # -# Files to add to the RPM -# -------------------------------------------------------------------------------------------- # -%files -%defattr(755,%{_project_user},%{_project_user},755) -%config /etc/init.d/%{_service_name} -%config /etc/logrotate.d/logrotate-sth-daily.conf -%{_install_dir} - -%changelog -* Tue Nov 28 2023 Alvaro Vega 2.11.0 -- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) -- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) -- Fix: the way in which list collection is done at MongoDB (ensure¡ing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions -- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) - -* Wed Nov 02 2022 Alvaro Vega 2.10.0 -- Fix: healthcheck over sth exposed port -- Fix: Dockerfile to include initial packages upgrade -- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) - -* Fri Mar 25 2022 Fermin Galan 2.9.0 -- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) -- Add: Docker healthcheck for STH API -- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) -- Add: allow connect with mongo uri without auth -- Fix: return 500 when DB is not connected (#570) -- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) -- Fix: update logs about get raw and aggregated data (#556) -- Update codebase to use ES6 - - Remove JSHint and jshint overrides - - Add esLint using standard tamia presets - - Replace var with let/const - - Fix or disable eslint errors -- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption -- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) -- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile - -* Wed May 27 2020 Alvaro Vega 2.8.0 -- Fix: Filetype issue fixed for hLimit and hOffset (#539) -- Fix: Updated temporary csv filename for uniqueness (#532) -- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life -- Make optional PM2 usage in docker entrypoint -- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) - -* Mon Nov 11 2019 Fermin Galan 2.7.0 -- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) -- Add: CORS support (#500) -- Fix: check header response before use it (CSV response has no header) -- Fix: handler for reply a CSV: use new Hapi API (#513) -- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) -- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues - -* Tue Jun 04 2019 Fermin Galan 2.6.0 -- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security -- Add: logging feature to count number of requests attended and number of requests processed with error (#310) -- Fix: race condition causes erroneous count to 0 in some cases (#493) - -* Wed Dec 19 2018 Fermin Galan 2.5.0 -- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) -- Add: use NodeJS 8 in Dockerfile -- Add: use PM2 in Dockerfile -- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 -- Upgrade: logops dependence from ~1.0.5 to 2.1.0 -- Upgrade: joi dependence from ~5.1.0 to 14.0.6 -- Upgrade: boom dependence from ~2.7.2 to 7.2.2 -- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 -- Upgrade: request development dependence from ~2.79.0 to 2.88.0 -- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 -- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) - -* Thu Aug 16 2018 Alvaro Vega 2.4.0 -- Upgrade lodash dependence to 4.17.5 -- Upgrade mongodb dependence to 2.2.35 -- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] -- Fix: Content Disposition header encoding issue (#433) -- Add count to query and related fiware-total-count header [#428] -- Using precise dependencies (~=) in packages.json -- Upgrade hapi to 11.1.3 - -* Wed Oct 18 2017 Fermin Galan 2.3.0 -- FEATURE update node version to 4.8.4 diff --git a/rpm/create-rpm.sh b/rpm/create-rpm.sh deleted file mode 100755 index 77b82760..00000000 --- a/rpm/create-rpm.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U -# -# This file is part of the Short Time Historic (STH) component -# -# STH is free software: you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the License, -# or (at your option) any later version. -# -# STH is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with STH. -# If not, seehttp://www.gnu.org/licenses/. -# -# For those usages not covered by the GNU Affero General Public License -# please contact with: [german.torodelvalle@telefonica.com] - - -function usage() { - SCRIPT=$(basename $0) - - printf "\n" >&2 - printf "usage: ${SCRIPT} [options] \n" >&2 - printf "\n" >&2 - printf "Options:\n" >&2 - printf "\n" >&2 - printf " -h show usage\n" >&2 - printf " -v VERSION Mandatory parameter. Version for rpm product preferably in format x.y.z \n" >&2 - printf " -r RELEASE Mandatory parameter. Release for product. I.E. 0.ge58dffa \n" >&2 - printf "\n" >&2 -} - -while getopts ":v:r:u:a:h" opt - -do - case $opt in - v) - VERSION_ARG=${OPTARG} - ;; - r) - RELEASE_ARG=${OPTARG} - ;; - h) - usage - exit 0 - ;; - *) - echo "invalid argument: '${OPTARG}'" - exit 1 - ;; - esac -done - -BASE_DIR="$(cd ${0%/*} && pwd -P)/.." -RPM_BASE_DIR="${BASE_DIR}/rpm" - -if [[ ! -z ${VERSION_ARG} ]]; then - PRODUCT_VERSION=${VERSION_ARG} -else - echo "A product version must be specified with -v parameter." - usage - exit 2 -fi - -if [[ ! -z ${RELEASE_ARG} ]]; then - PRODUCT_RELEASE=${RELEASE_ARG} -else - echo "A product reslease must be specified with -r parameter." - usage - exit 2 -fi - -PROXY_USER="sth" - -# Clean previous builds of rpm -echo "INFO: Clean previous builds of rpm <${RPM_BASE_DIR}/{RPMS,BUILDROOT,BUILD,SRPMS}>" -rm -rf ${RPM_BASE_DIR}/{RPMS,BUILDROOT,BUILD,SRPMS} - -rpmbuild -ba ${RPM_BASE_DIR}/SPECS/sth.spec \ - --define "_topdir ${RPM_BASE_DIR}" \ - --define "_project_user ${PROXY_USER}" \ - --define "_product_version ${PRODUCT_VERSION}" \ - --define "_product_release ${PRODUCT_RELEASE}" - diff --git a/rpm/get_version_string.sh b/rpm/get_version_string.sh deleted file mode 100755 index f81c24d3..00000000 --- a/rpm/get_version_string.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# Bash lib to know the RPM version and revision from a GitHub repository -# Call method get_rpm_version_string to obtain them for rpmbuild -# The result appear on the vars ver and rel -# The requisites are tags similar to 0.1.0/KO. This tag must be created by 'git tag -a 0.1.0' -# The main purpose to use this script is to deploy CI on master branch. -# -# Steps to get version and release: -# 1 - source get_version_string.sh -# 2 - Execute any of the functions on the script -# - It will be use the command 'read ver rel < <(get_rpm_version_string)' in order to get version and release on different vars - -if [[ $(ps -hp $$ | grep bash) ]]; then - shopt -s extglob -elif [[ $(ps -hp $$ | grep zsh) ]]; then - setopt kshglob -fi - -get_branch() -{ - git rev-parse --abbrev-ref HEAD -} - -## Specific functions according the TID workflow -get_branch_type() -{ - local branch="$(get_branch)" - case $branch in - release/*) echo "release";; - master) echo "latest";; - *) echo "other";; - esac -} - -get_version_string() -{ - local branch branch_name describe_tags version ancestor release - case $(get_branch_type) in - master) - ## If we are in master use the total count of commits of the repo - total_commit_number=$(git rev-list --all --count) - short_hash=$(git rev-parse --short HEAD) - version="$(git describe --tags --long --match "[[:digit:]]*.[[:digit:]]*.[[:digit:]]*" 2>/dev/null)" - version="${version%-*-*}" - version="${version%KO}" - echo "${version}-${total_commit_number}-${short_hash}" - ;; - release) - ## in release branches the version is a tag named - branch_name="$(get_branch)" - branch_name="${branch_name#*/}" - describe_tags="$(git describe --tags --long --match ${branch_name} 2>/dev/null)" - version="${describe_tags%-*-*}" - version="${version%KO}" - release=${describe_tags#*.*.*-} - echo "${version}-${release}" - ;; - other) - ## We are in detached mode, use the last x-y-z tag - version="$(git describe --tags --long --match "[[:digit:]]*.[[:digit:]]*.[[:digit:]]*" 2>/dev/null)" - version="${version%-*-*}" - version="${version%KO}" - echo "${version}" - ;; - *) - # RMs don't stablish any standard here, we use branch name as version - version=$(get_branch) - # Using always master as parent branch does not describe correctly the number of revision - # for branches not starting there, but works as an incremental rev - ancestor="$(git merge-base $version master)" - version=${version#*/} - local res="$(git log --oneline ${ancestor}.. --pretty='format:%h')" - ## wc alone does not get the last line when there's no new line - [[ -z $res ]] && rel=0 || rel=$(echo "$res" | wc -l | tr -d ' ') - echo "${version}-${rel}-g$(git log --pretty=format:'%h' -1)" - esac -} - -get_rpm_version_string() { - local version_string ver rel - version_string="$(get_version_string)" - ver="${version_string%-*-*}" - rel="${version_string:$((${#ver}+1))}" - echo "${ver//[[:space:]-\/#]}" "${rel//[-]/.}" -} diff --git a/rpm/EXAMPLES/sth_default.conf b/sth_default.conf similarity index 97% rename from rpm/EXAMPLES/sth_default.conf rename to sth_default.conf index ab4f314f..fbcb1ada 100644 --- a/rpm/EXAMPLES/sth_default.conf +++ b/sth_default.conf @@ -1,157 +1,157 @@ -#!/usr/bin/env bash - -# Ansible managed: modified by IoT devops team - -# Short Term History configuration variables. -# STH is configured defining the folowing variables. -# All variables are optional, so, if one of this variables is not defined -# default value is used. - -# See https://github.com/telefonicaid/IoT-STH#running-the-sth-server -# for further information - -# Note: is important to export environment variables with export - -# The host where the STH server will be started. -# Default value: "localhost". -export STH_HOST="0.0.0.0" - -# The port where the STH server will be listening. -# Default value: 8666. -export STH_PORT="8666" - -# The service to be used if not sent by the Orion Context Broker in the notifications. -# Default value: "testservice". -export DEFAULT_SERVICE="testservice" - -# The service path to be used if not sent by the Orion Context Broker in the notifications. -# Default value: "/testservicepath". -export DEFAULT_SERVICE_PATH="/testservicepath" - -# A flag indicating if the empty results should be removed from the response. -# Default value: "true". -export FILTER_OUT_EMPTY="true" - -# Array of resolutions the STH component should aggregate values for. -# Valid resolution values are: 'month', 'day', 'hour', 'minute' and 'second' -export AGGREGATION_BY="['day', 'hour', 'minute']" - -# The temporal directory to be used by the STH to temporarily store files (right now .csv files) before returning them. -# Default value: "temp". -export TEMPORAL_DIR="temp" - -# The STH component supports 3 alternative data models when storing the raw and aggregated data -# into the database: 1) one collection per attribute, 2) one collection per entity and -# 3) one collection per service path. The possible values are: "collection-per-attribute", -# "collection-per-entity" and "collection-per-service-path" respectively. Default value: -# "collection-per-entity". -export DATA_MODEL='collection-per-entity' - -# The username to use for the database connection. -# Default value: "". -export DB_USERNAME="" - -# The password to use for the database connection. -# Default value: "". -export DB_PASSWORD="" - -# The URI to use for the database connection. -# This does not include the 'mongo://' protocol part. -# Two or more host are comma separated. -# Default value: "localhost:27017". -export DB_URI="localhost:27017" - -# The name of the MongoDB replica set. -# Default value: "". -export REPLICA_SET="" - -# The prefix to be added to the service for the creation of the databases. -# More information below. -# Default value: "sth_". -export DB_PREFIX="sth_" - -# The prefix to be added to the collections in the databases. More information below. -# Default value: "sth_". -export COLLECTION_PREFIX="sth_" - -# The default MongoDB pool size of database connections. -# Default value: "5". -export POOL_SIZE="5" - -# The write concern policy to apply when writing data to the MongoDB database. -# Write concenr more info: http://docs.mongodb.org/manual/core/write-concern/ -# Default value: "1". -export WRITE_CONCERN="1" - -# Flag indicating if the raw and/or aggregated data should be persisted. -# Valid values are: "only-raw", "only-aggregated" and "both". -# Default value: "both". -export SHOULD_STORE="both" - -# Data from the raw and aggregated data collections will be removed if older than the value specified in seconds. -# In case of raw data the reference time is the one stored in the recvTime property whereas in the case of the -# aggregated data the reference of time is the one stored in the _id.origin property. -# Set the value to 0 not to apply this time-based truncation policy. Default value: "0". -export TRUNCATION_EXPIRE_AFTER_SECONDS="0" - -# The oldest raw data (according to insertion time) will be removed if the size of the raw data collection gets bigger -# than the value specified in bytes. -# Set the value to 0 not to apply this truncation policy. Take into consideration than the "size" configuration -# parameter is mandatory in case size collection truncation is desired as required by MongoDB. -# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not -# currently support updating documents in capped collections which increase the size of the documents. -# Notice also that in case of the raw data, the size-based truncation policy takes precedence over the TTL one. -# More concretely, if "size" is set, the value of "exporeAfterSeconds" is ignored for the raw data collections since -# currently MongoDB does not support TTL in capped collections. -# Default value: "0". -export TRUNCATION_SIZE="0" - -# The oldest raw data (according to insertion time) will be removed if the number of documents in the raw data -# collections goes beyond the specified value. Set the value to 0 not to apply this truncation policy. -# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not -# currently support updating documents in capped collections which increase the size of the documents. -# Default value: "0". -export TRUNCATION_MAX="0" - -# Attribute values to one or more blank spaces should be ignored and not processed either as raw data or for the -# aggregated computations -# Default value: "true". -export IGNORE_BLANK_SPACES="true" - -# Database and collection names have to respect the limitations imposed by MongoDB (see -# https://docs.mongodb.com/manual/reference/limits/). To it, the STH provides 2 main mechanisms: mappings and -# encoding which can be configured using the next 2 configuration parameters. -# The mappings mechanism will substitute the original services, service paths, entity and attribute names and types -# by the ones defined in the configuration file. If enabled, the mappings mechanism will be the one applied. -export NAME_MAPPING ="{\"enabled\": \"false\", \"configFile\": \"./name-mapping.json\"}" - -# Database and collection names should be encoded to avoid the restrictions imposed by MongoDB and stated at -# https://docs.mongodb.com/manual/reference/limits/ -# The encoding criteria is the following one: -# 1. Encode the forbidden characters using an escaping character (x) and a numerical Unicode code for each character. -# For instance, the / character will be encoded as x002f. -# 2. Database and collection names already using the above encoding must be escaped prepending another x, -# for instance, the text x002a will be encoded as xx002a. -# 3. The uppercase characters included in database names will be encoded using the mechanism stated in 1. -# 4. Collection names starting with 'system.' will be encoded as 'xsystem.'. For instance, system.myData will be -# encoded as xsystem.myData. -# Default value: "true" (although we will set it to false until the Cygnus counterpart is ready and landed) -export NAME_ENCODING="false" - -# The log level to use. Possible values are: "DEBUG", "INFO", "WARN", "ERROR" and "FATAL". -# Default value: "INFO". -export LOGOPS_LEVEL="INFO" - -# The log format to use. Possible values are: "json" (writes logs as JSON), "dev" (for development, used when the -# NODE_ENV variable is set to 'development') and "pipe" (write logs separating the contents with pipes). -# Default value: "pipe". -export LOGOPS_FORMAT="pipe" - -# The time in seconds between proof of life logging messages informing that -# the server is up and running normally. -# Default value: "60". -export PROOF_OF_LIFE_INTERVAL="60" - -# The time in seconds between processed requests statistics appear in the logs -# Default value: "60". -export PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL="60" +#!/usr/bin/env bash + +# Ansible managed: modified by IoT devops team + +# Short Term History configuration variables. +# STH is configured defining the folowing variables. +# All variables are optional, so, if one of this variables is not defined +# default value is used. + +# See https://github.com/telefonicaid/IoT-STH#running-the-sth-server +# for further information + +# Note: is important to export environment variables with export + +# The host where the STH server will be started. +# Default value: "localhost". +export STH_HOST="0.0.0.0" + +# The port where the STH server will be listening. +# Default value: 8666. +export STH_PORT="8666" + +# The service to be used if not sent by the Orion Context Broker in the notifications. +# Default value: "testservice". +export DEFAULT_SERVICE="testservice" + +# The service path to be used if not sent by the Orion Context Broker in the notifications. +# Default value: "/testservicepath". +export DEFAULT_SERVICE_PATH="/testservicepath" + +# A flag indicating if the empty results should be removed from the response. +# Default value: "true". +export FILTER_OUT_EMPTY="true" + +# Array of resolutions the STH component should aggregate values for. +# Valid resolution values are: 'month', 'day', 'hour', 'minute' and 'second' +export AGGREGATION_BY="['day', 'hour', 'minute']" + +# The temporal directory to be used by the STH to temporarily store files (right now .csv files) before returning them. +# Default value: "temp". +export TEMPORAL_DIR="temp" + +# The STH component supports 3 alternative data models when storing the raw and aggregated data +# into the database: 1) one collection per attribute, 2) one collection per entity and +# 3) one collection per service path. The possible values are: "collection-per-attribute", +# "collection-per-entity" and "collection-per-service-path" respectively. Default value: +# "collection-per-entity". +export DATA_MODEL='collection-per-entity' + +# The username to use for the database connection. +# Default value: "". +export DB_USERNAME="" + +# The password to use for the database connection. +# Default value: "". +export DB_PASSWORD="" + +# The URI to use for the database connection. +# This does not include the 'mongo://' protocol part. +# Two or more host are comma separated. +# Default value: "localhost:27017". +export DB_URI="localhost:27017" + +# The name of the MongoDB replica set. +# Default value: "". +export REPLICA_SET="" + +# The prefix to be added to the service for the creation of the databases. +# More information below. +# Default value: "sth_". +export DB_PREFIX="sth_" + +# The prefix to be added to the collections in the databases. More information below. +# Default value: "sth_". +export COLLECTION_PREFIX="sth_" + +# The default MongoDB pool size of database connections. +# Default value: "5". +export POOL_SIZE="5" + +# The write concern policy to apply when writing data to the MongoDB database. +# Write concenr more info: http://docs.mongodb.org/manual/core/write-concern/ +# Default value: "1". +export WRITE_CONCERN="1" + +# Flag indicating if the raw and/or aggregated data should be persisted. +# Valid values are: "only-raw", "only-aggregated" and "both". +# Default value: "both". +export SHOULD_STORE="both" + +# Data from the raw and aggregated data collections will be removed if older than the value specified in seconds. +# In case of raw data the reference time is the one stored in the recvTime property whereas in the case of the +# aggregated data the reference of time is the one stored in the _id.origin property. +# Set the value to 0 not to apply this time-based truncation policy. Default value: "0". +export TRUNCATION_EXPIRE_AFTER_SECONDS="0" + +# The oldest raw data (according to insertion time) will be removed if the size of the raw data collection gets bigger +# than the value specified in bytes. +# Set the value to 0 not to apply this truncation policy. Take into consideration than the "size" configuration +# parameter is mandatory in case size collection truncation is desired as required by MongoDB. +# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not +# currently support updating documents in capped collections which increase the size of the documents. +# Notice also that in case of the raw data, the size-based truncation policy takes precedence over the TTL one. +# More concretely, if "size" is set, the value of "exporeAfterSeconds" is ignored for the raw data collections since +# currently MongoDB does not support TTL in capped collections. +# Default value: "0". +export TRUNCATION_SIZE="0" + +# The oldest raw data (according to insertion time) will be removed if the number of documents in the raw data +# collections goes beyond the specified value. Set the value to 0 not to apply this truncation policy. +# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not +# currently support updating documents in capped collections which increase the size of the documents. +# Default value: "0". +export TRUNCATION_MAX="0" + +# Attribute values to one or more blank spaces should be ignored and not processed either as raw data or for the +# aggregated computations +# Default value: "true". +export IGNORE_BLANK_SPACES="true" + +# Database and collection names have to respect the limitations imposed by MongoDB (see +# https://docs.mongodb.com/manual/reference/limits/). To it, the STH provides 2 main mechanisms: mappings and +# encoding which can be configured using the next 2 configuration parameters. +# The mappings mechanism will substitute the original services, service paths, entity and attribute names and types +# by the ones defined in the configuration file. If enabled, the mappings mechanism will be the one applied. +export NAME_MAPPING ="{\"enabled\": \"false\", \"configFile\": \"./name-mapping.json\"}" + +# Database and collection names should be encoded to avoid the restrictions imposed by MongoDB and stated at +# https://docs.mongodb.com/manual/reference/limits/ +# The encoding criteria is the following one: +# 1. Encode the forbidden characters using an escaping character (x) and a numerical Unicode code for each character. +# For instance, the / character will be encoded as x002f. +# 2. Database and collection names already using the above encoding must be escaped prepending another x, +# for instance, the text x002a will be encoded as xx002a. +# 3. The uppercase characters included in database names will be encoded using the mechanism stated in 1. +# 4. Collection names starting with 'system.' will be encoded as 'xsystem.'. For instance, system.myData will be +# encoded as xsystem.myData. +# Default value: "true" (although we will set it to false until the Cygnus counterpart is ready and landed) +export NAME_ENCODING="false" + +# The log level to use. Possible values are: "DEBUG", "INFO", "WARN", "ERROR" and "FATAL". +# Default value: "INFO". +export LOGOPS_LEVEL="INFO" + +# The log format to use. Possible values are: "json" (writes logs as JSON), "dev" (for development, used when the +# NODE_ENV variable is set to 'development') and "pipe" (write logs separating the contents with pipes). +# Default value: "pipe". +export LOGOPS_FORMAT="pipe" + +# The time in seconds between proof of life logging messages informing that +# the server is up and running normally. +# Default value: "60". +export PROOF_OF_LIFE_INTERVAL="60" + +# The time in seconds between processed requests statistics appear in the logs +# Default value: "60". +export PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL="60" From 545ba9dc7a4d4d49b21e6689473d4710f3a0c2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 12:34:42 +0100 Subject: [PATCH 2/4] Update Changelog --- Changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Changelog b/Changelog index b962c432..72f777b9 100644 --- a/Changelog +++ b/Changelog @@ -2,7 +2,8 @@ - Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) - Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) -- Fix: the way in which list collection is done at MongoDB (ensure�ing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions +- Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions + - Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) 2.10.0 (November 2nd, 2022) From f0496ad75eb25cb154ec575031ebf371fce7ef33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 12:36:05 +0100 Subject: [PATCH 3/4] FIX encoding --- Changelog | 162 ++++++++++++------------ sth_default.conf | 314 +++++++++++++++++++++++------------------------ 2 files changed, 238 insertions(+), 238 deletions(-) diff --git a/Changelog b/Changelog index 72f777b9..a524ab9d 100644 --- a/Changelog +++ b/Changelog @@ -1,82 +1,82 @@ -2.11.0 (November 28th, 2023) - -- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) -- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) -- Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions - -- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) - -2.10.0 (November 2nd, 2022) - -- Fix: healthcheck over sth exposed port -- Fix: Dockerfile to include initial packages upgrade -- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) - -2.9.0 (March 25th, 2022) - -- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) -- Add: Docker healthcheck for STH API -- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) -- Add: allow connect with mongo uri without auth -- Fix: return 500 when DB is not connected (#570) -- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) -- Fix: update logs about get raw and aggregated data (#556) -- Update codebase to use ES6 - - Remove JSHint and jshint overrides - - Add esLint using standard tamia presets - - Replace var with let/const - - Fix or disable eslint errors -- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption -- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) -- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile - -2.8.0 (May 27th, 2020) - -- Fix: Filetype issue fixed for hLimit and hOffset (#539) -- Fix: Updated temporary csv filename for uniqueness (#532) -- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life -- Make optional PM2 usage in docker entrypoint -- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) - -2.7.0 (November 11th, 2019) - -- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) -- Add: CORS support (#500) -- Fix: check header response before use it (CSV response has no header) -- Fix: handler for reply a CSV: use new Hapi API (#513) -- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) -- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues - -2.6.0 (June 4th, 2019) - -- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security -- Add: logging feature to count number of requests attended and number of requests processed with error (#310) -- Fix: race condition causes erroneous count to 0 in some cases (#493) - -2.5.0 (December 19th, 2018) - -- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) -- Add: use NodeJS 8 in Dockerfile -- Add: use PM2 in Dockerfile -- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 -- Upgrade: logops dependence from ~1.0.5 to 2.1.0 -- Upgrade: joi dependence from ~5.1.0 to 14.0.6 -- Upgrade: boom dependence from ~2.7.2 to 7.2.2 -- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 -- Upgrade: request development dependence from ~2.79.0 to 2.88.0 -- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 -- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) - -2.4.0 (August 16th, 2018) - -- Upgrade lodash dependence to 4.17.5 -- Upgrade mongodb dependence to 2.2.35 -- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] -- Fix: Content Disposition header encoding issue (#433) -- Add count to query and related fiware-total-count header [#428] -- Using precise dependencies (~=) in packages.json -- Upgrade hapi to 11.1.3 - -2.3.0 (October 18th, 2017) - +2.11.0 (November 28th, 2023) + +- Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) +- Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) +- Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions + +- Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) + +2.10.0 (November 2nd, 2022) + +- Fix: healthcheck over sth exposed port +- Fix: Dockerfile to include initial packages upgrade +- Set Nodejs 12 as minimum version in packages.json (effectively removing Nodev8 and Nodev10 from supported versions) + +2.9.0 (March 25th, 2022) + +- Add: graceful shutdown listening to SIGTERM and SIGHUP signals (#576) +- Add: Docker healthcheck for STH API +- Add: config reconnectTries and reconnectInterval configuration (env vars DB_RECONNECT_TRIES and DB_RECONNECT_INTERVAL) to allow mongo driver reconnect (#559) +- Add: allow connect with mongo uri without auth +- Fix: return 500 when DB is not connected (#570) +- Fix: ensure permissions over temp directory (used by CSV generation) in docker container (issue re-introduced in 2.8.0) +- Fix: update logs about get raw and aggregated data (#556) +- Update codebase to use ES6 + - Remove JSHint and jshint overrides + - Add esLint using standard tamia presets + - Replace var with let/const + - Fix or disable eslint errors +- Upgrade logops dep from 2.1.0 to 2.1.2 due to colors dependency corruption +- Upgrade: mongodb dependence from ~2.2.35 to ~3.6.12 (#567) +- Upgrade NodeJS version from 10.19.0 to 14-slim in Dockerfile + +2.8.0 (May 27th, 2020) + +- Fix: Filetype issue fixed for hLimit and hOffset (#539) +- Fix: Updated temporary csv filename for uniqueness (#532) +- Upgrade NodeJS version from 8.16.1 to 10.19.0 in Dockerfile due to Node 8 End-of-Life +- Make optional PM2 usage in docker entrypoint +- Add: MongoDB auth source configuration (env var DB_AUTH_SOURCE) + +2.7.0 (November 11th, 2019) + +- Add: NGSIv2 endpoints for the raw and aggregation API operations (#118) +- Add: CORS support (#500) +- Fix: check header response before use it (CSV response has no header) +- Fix: handler for reply a CSV: use new Hapi API (#513) +- Fix: Ensure permissions over temp directory (used by CSV generation) in docker container (#514) +- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues + +2.6.0 (June 4th, 2019) + +- Upgrade NodeJS version from 8.12.0 to 8.16.0 in Dockerfile due to improve security +- Add: logging feature to count number of requests attended and number of requests processed with error (#310) +- Fix: race condition causes erroneous count to 0 in some cases (#493) + +2.5.0 (December 19th, 2018) + +- Set Nodejs 8.12.0 as minimum version in packages.json (effectively removing Nodev4 and Nodev6 as supported versions) +- Add: use NodeJS 8 in Dockerfile +- Add: use PM2 in Dockerfile +- Upgrade: hapi dependence from ~11.1.3 to ~16.7.0 +- Upgrade: logops dependence from ~1.0.5 to 2.1.0 +- Upgrade: joi dependence from ~5.1.0 to 14.0.6 +- Upgrade: boom dependence from ~2.7.2 to 7.2.2 +- Upgrade: json-csv dependence from ~1.2.0 to 1.5.0 +- Upgrade: request development dependence from ~2.79.0 to 2.88.0 +- Upgrade: mocha development depencence from ~3.2.0 to 5.2.0 +- Remove: old unused development dependencies (chai, sinon, sinon-chai, grunt and grunt related module) + +2.4.0 (August 16th, 2018) + +- Upgrade lodash dependence to 4.17.5 +- Upgrade mongodb dependence to 2.2.35 +- Check & ensure hLimit<=lastN<=config.maxPageSize for raw data query. [#431] +- Fix: Content Disposition header encoding issue (#433) +- Add count to query and related fiware-total-count header [#428] +- Using precise dependencies (~=) in packages.json +- Upgrade hapi to 11.1.3 + +2.3.0 (October 18th, 2017) + - FEATURE update node version to 4.8.4 \ No newline at end of file diff --git a/sth_default.conf b/sth_default.conf index fbcb1ada..ab4f314f 100644 --- a/sth_default.conf +++ b/sth_default.conf @@ -1,157 +1,157 @@ -#!/usr/bin/env bash - -# Ansible managed: modified by IoT devops team - -# Short Term History configuration variables. -# STH is configured defining the folowing variables. -# All variables are optional, so, if one of this variables is not defined -# default value is used. - -# See https://github.com/telefonicaid/IoT-STH#running-the-sth-server -# for further information - -# Note: is important to export environment variables with export - -# The host where the STH server will be started. -# Default value: "localhost". -export STH_HOST="0.0.0.0" - -# The port where the STH server will be listening. -# Default value: 8666. -export STH_PORT="8666" - -# The service to be used if not sent by the Orion Context Broker in the notifications. -# Default value: "testservice". -export DEFAULT_SERVICE="testservice" - -# The service path to be used if not sent by the Orion Context Broker in the notifications. -# Default value: "/testservicepath". -export DEFAULT_SERVICE_PATH="/testservicepath" - -# A flag indicating if the empty results should be removed from the response. -# Default value: "true". -export FILTER_OUT_EMPTY="true" - -# Array of resolutions the STH component should aggregate values for. -# Valid resolution values are: 'month', 'day', 'hour', 'minute' and 'second' -export AGGREGATION_BY="['day', 'hour', 'minute']" - -# The temporal directory to be used by the STH to temporarily store files (right now .csv files) before returning them. -# Default value: "temp". -export TEMPORAL_DIR="temp" - -# The STH component supports 3 alternative data models when storing the raw and aggregated data -# into the database: 1) one collection per attribute, 2) one collection per entity and -# 3) one collection per service path. The possible values are: "collection-per-attribute", -# "collection-per-entity" and "collection-per-service-path" respectively. Default value: -# "collection-per-entity". -export DATA_MODEL='collection-per-entity' - -# The username to use for the database connection. -# Default value: "". -export DB_USERNAME="" - -# The password to use for the database connection. -# Default value: "". -export DB_PASSWORD="" - -# The URI to use for the database connection. -# This does not include the 'mongo://' protocol part. -# Two or more host are comma separated. -# Default value: "localhost:27017". -export DB_URI="localhost:27017" - -# The name of the MongoDB replica set. -# Default value: "". -export REPLICA_SET="" - -# The prefix to be added to the service for the creation of the databases. -# More information below. -# Default value: "sth_". -export DB_PREFIX="sth_" - -# The prefix to be added to the collections in the databases. More information below. -# Default value: "sth_". -export COLLECTION_PREFIX="sth_" - -# The default MongoDB pool size of database connections. -# Default value: "5". -export POOL_SIZE="5" - -# The write concern policy to apply when writing data to the MongoDB database. -# Write concenr more info: http://docs.mongodb.org/manual/core/write-concern/ -# Default value: "1". -export WRITE_CONCERN="1" - -# Flag indicating if the raw and/or aggregated data should be persisted. -# Valid values are: "only-raw", "only-aggregated" and "both". -# Default value: "both". -export SHOULD_STORE="both" - -# Data from the raw and aggregated data collections will be removed if older than the value specified in seconds. -# In case of raw data the reference time is the one stored in the recvTime property whereas in the case of the -# aggregated data the reference of time is the one stored in the _id.origin property. -# Set the value to 0 not to apply this time-based truncation policy. Default value: "0". -export TRUNCATION_EXPIRE_AFTER_SECONDS="0" - -# The oldest raw data (according to insertion time) will be removed if the size of the raw data collection gets bigger -# than the value specified in bytes. -# Set the value to 0 not to apply this truncation policy. Take into consideration than the "size" configuration -# parameter is mandatory in case size collection truncation is desired as required by MongoDB. -# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not -# currently support updating documents in capped collections which increase the size of the documents. -# Notice also that in case of the raw data, the size-based truncation policy takes precedence over the TTL one. -# More concretely, if "size" is set, the value of "exporeAfterSeconds" is ignored for the raw data collections since -# currently MongoDB does not support TTL in capped collections. -# Default value: "0". -export TRUNCATION_SIZE="0" - -# The oldest raw data (according to insertion time) will be removed if the number of documents in the raw data -# collections goes beyond the specified value. Set the value to 0 not to apply this truncation policy. -# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not -# currently support updating documents in capped collections which increase the size of the documents. -# Default value: "0". -export TRUNCATION_MAX="0" - -# Attribute values to one or more blank spaces should be ignored and not processed either as raw data or for the -# aggregated computations -# Default value: "true". -export IGNORE_BLANK_SPACES="true" - -# Database and collection names have to respect the limitations imposed by MongoDB (see -# https://docs.mongodb.com/manual/reference/limits/). To it, the STH provides 2 main mechanisms: mappings and -# encoding which can be configured using the next 2 configuration parameters. -# The mappings mechanism will substitute the original services, service paths, entity and attribute names and types -# by the ones defined in the configuration file. If enabled, the mappings mechanism will be the one applied. -export NAME_MAPPING ="{\"enabled\": \"false\", \"configFile\": \"./name-mapping.json\"}" - -# Database and collection names should be encoded to avoid the restrictions imposed by MongoDB and stated at -# https://docs.mongodb.com/manual/reference/limits/ -# The encoding criteria is the following one: -# 1. Encode the forbidden characters using an escaping character (x) and a numerical Unicode code for each character. -# For instance, the / character will be encoded as x002f. -# 2. Database and collection names already using the above encoding must be escaped prepending another x, -# for instance, the text x002a will be encoded as xx002a. -# 3. The uppercase characters included in database names will be encoded using the mechanism stated in 1. -# 4. Collection names starting with 'system.' will be encoded as 'xsystem.'. For instance, system.myData will be -# encoded as xsystem.myData. -# Default value: "true" (although we will set it to false until the Cygnus counterpart is ready and landed) -export NAME_ENCODING="false" - -# The log level to use. Possible values are: "DEBUG", "INFO", "WARN", "ERROR" and "FATAL". -# Default value: "INFO". -export LOGOPS_LEVEL="INFO" - -# The log format to use. Possible values are: "json" (writes logs as JSON), "dev" (for development, used when the -# NODE_ENV variable is set to 'development') and "pipe" (write logs separating the contents with pipes). -# Default value: "pipe". -export LOGOPS_FORMAT="pipe" - -# The time in seconds between proof of life logging messages informing that -# the server is up and running normally. -# Default value: "60". -export PROOF_OF_LIFE_INTERVAL="60" - -# The time in seconds between processed requests statistics appear in the logs -# Default value: "60". -export PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL="60" +#!/usr/bin/env bash + +# Ansible managed: modified by IoT devops team + +# Short Term History configuration variables. +# STH is configured defining the folowing variables. +# All variables are optional, so, if one of this variables is not defined +# default value is used. + +# See https://github.com/telefonicaid/IoT-STH#running-the-sth-server +# for further information + +# Note: is important to export environment variables with export + +# The host where the STH server will be started. +# Default value: "localhost". +export STH_HOST="0.0.0.0" + +# The port where the STH server will be listening. +# Default value: 8666. +export STH_PORT="8666" + +# The service to be used if not sent by the Orion Context Broker in the notifications. +# Default value: "testservice". +export DEFAULT_SERVICE="testservice" + +# The service path to be used if not sent by the Orion Context Broker in the notifications. +# Default value: "/testservicepath". +export DEFAULT_SERVICE_PATH="/testservicepath" + +# A flag indicating if the empty results should be removed from the response. +# Default value: "true". +export FILTER_OUT_EMPTY="true" + +# Array of resolutions the STH component should aggregate values for. +# Valid resolution values are: 'month', 'day', 'hour', 'minute' and 'second' +export AGGREGATION_BY="['day', 'hour', 'minute']" + +# The temporal directory to be used by the STH to temporarily store files (right now .csv files) before returning them. +# Default value: "temp". +export TEMPORAL_DIR="temp" + +# The STH component supports 3 alternative data models when storing the raw and aggregated data +# into the database: 1) one collection per attribute, 2) one collection per entity and +# 3) one collection per service path. The possible values are: "collection-per-attribute", +# "collection-per-entity" and "collection-per-service-path" respectively. Default value: +# "collection-per-entity". +export DATA_MODEL='collection-per-entity' + +# The username to use for the database connection. +# Default value: "". +export DB_USERNAME="" + +# The password to use for the database connection. +# Default value: "". +export DB_PASSWORD="" + +# The URI to use for the database connection. +# This does not include the 'mongo://' protocol part. +# Two or more host are comma separated. +# Default value: "localhost:27017". +export DB_URI="localhost:27017" + +# The name of the MongoDB replica set. +# Default value: "". +export REPLICA_SET="" + +# The prefix to be added to the service for the creation of the databases. +# More information below. +# Default value: "sth_". +export DB_PREFIX="sth_" + +# The prefix to be added to the collections in the databases. More information below. +# Default value: "sth_". +export COLLECTION_PREFIX="sth_" + +# The default MongoDB pool size of database connections. +# Default value: "5". +export POOL_SIZE="5" + +# The write concern policy to apply when writing data to the MongoDB database. +# Write concenr more info: http://docs.mongodb.org/manual/core/write-concern/ +# Default value: "1". +export WRITE_CONCERN="1" + +# Flag indicating if the raw and/or aggregated data should be persisted. +# Valid values are: "only-raw", "only-aggregated" and "both". +# Default value: "both". +export SHOULD_STORE="both" + +# Data from the raw and aggregated data collections will be removed if older than the value specified in seconds. +# In case of raw data the reference time is the one stored in the recvTime property whereas in the case of the +# aggregated data the reference of time is the one stored in the _id.origin property. +# Set the value to 0 not to apply this time-based truncation policy. Default value: "0". +export TRUNCATION_EXPIRE_AFTER_SECONDS="0" + +# The oldest raw data (according to insertion time) will be removed if the size of the raw data collection gets bigger +# than the value specified in bytes. +# Set the value to 0 not to apply this truncation policy. Take into consideration than the "size" configuration +# parameter is mandatory in case size collection truncation is desired as required by MongoDB. +# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not +# currently support updating documents in capped collections which increase the size of the documents. +# Notice also that in case of the raw data, the size-based truncation policy takes precedence over the TTL one. +# More concretely, if "size" is set, the value of "exporeAfterSeconds" is ignored for the raw data collections since +# currently MongoDB does not support TTL in capped collections. +# Default value: "0". +export TRUNCATION_SIZE="0" + +# The oldest raw data (according to insertion time) will be removed if the number of documents in the raw data +# collections goes beyond the specified value. Set the value to 0 not to apply this truncation policy. +# Notice that this configuration parameter does not affect the aggregated data collections since MongoDB does not +# currently support updating documents in capped collections which increase the size of the documents. +# Default value: "0". +export TRUNCATION_MAX="0" + +# Attribute values to one or more blank spaces should be ignored and not processed either as raw data or for the +# aggregated computations +# Default value: "true". +export IGNORE_BLANK_SPACES="true" + +# Database and collection names have to respect the limitations imposed by MongoDB (see +# https://docs.mongodb.com/manual/reference/limits/). To it, the STH provides 2 main mechanisms: mappings and +# encoding which can be configured using the next 2 configuration parameters. +# The mappings mechanism will substitute the original services, service paths, entity and attribute names and types +# by the ones defined in the configuration file. If enabled, the mappings mechanism will be the one applied. +export NAME_MAPPING ="{\"enabled\": \"false\", \"configFile\": \"./name-mapping.json\"}" + +# Database and collection names should be encoded to avoid the restrictions imposed by MongoDB and stated at +# https://docs.mongodb.com/manual/reference/limits/ +# The encoding criteria is the following one: +# 1. Encode the forbidden characters using an escaping character (x) and a numerical Unicode code for each character. +# For instance, the / character will be encoded as x002f. +# 2. Database and collection names already using the above encoding must be escaped prepending another x, +# for instance, the text x002a will be encoded as xx002a. +# 3. The uppercase characters included in database names will be encoded using the mechanism stated in 1. +# 4. Collection names starting with 'system.' will be encoded as 'xsystem.'. For instance, system.myData will be +# encoded as xsystem.myData. +# Default value: "true" (although we will set it to false until the Cygnus counterpart is ready and landed) +export NAME_ENCODING="false" + +# The log level to use. Possible values are: "DEBUG", "INFO", "WARN", "ERROR" and "FATAL". +# Default value: "INFO". +export LOGOPS_LEVEL="INFO" + +# The log format to use. Possible values are: "json" (writes logs as JSON), "dev" (for development, used when the +# NODE_ENV variable is set to 'development') and "pipe" (write logs separating the contents with pipes). +# Default value: "pipe". +export LOGOPS_FORMAT="pipe" + +# The time in seconds between proof of life logging messages informing that +# the server is up and running normally. +# Default value: "60". +export PROOF_OF_LIFE_INTERVAL="60" + +# The time in seconds between processed requests statistics appear in the logs +# Default value: "60". +export PROCESSED_REQUEST_LOG_STATISTICS_INTERVAL="60" From 589bad7b9136657f3fa0c000cb18f0c7618baac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 29 Nov 2023 12:41:09 +0100 Subject: [PATCH 4/4] Update Changelog --- Changelog | 1 - 1 file changed, 1 deletion(-) diff --git a/Changelog b/Changelog index a524ab9d..05664546 100644 --- a/Changelog +++ b/Changelog @@ -3,7 +3,6 @@ - Add: CORS_ENABLED env var (boolean) to enable the cors configuration (of config.js file) in your docker image (#608) - Set Nodejs 14 as minimum version in packages.json (effectively removing Nodev12 from supported versions) - Fix: the way in which list collection is done at MongoDB (ensureing nameOnly flag is boolean) so STH can support MongoDB 5.0+ versions - - Fix: TRUNCATION_EXPIRE_AFTER_SECONDS functionality (#606) 2.10.0 (November 2nd, 2022)