Skip to content

Commit

Permalink
feat(redis cache): add redis configuration
Browse files Browse the repository at this point in the history
* add configuration for RedisCache and use it in catalogDB
* add support for redis server password
* remove IsActive for cache config and replace by type Disabled
* separate cache config struct and functions to dedicated file
  • Loading branch information
jmkerloch authored and benoitdm-oslandia committed Feb 15, 2023
1 parent 541456a commit dbce243
Show file tree
Hide file tree
Showing 8 changed files with 189 additions and 92 deletions.
11 changes: 11 additions & 0 deletions config/pg_featureserv.toml.example
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,14 @@ LimitMax = 10000
[Website]
# URL for the map view basemap
BasemapUrl = "http://a.tile.openstreetmap.fr/hot/{z}/{x}/{y}.png"

[Cache]
# Type of cache, choose between Disabled / Naive / Redis
Type = "Naive"

[Cache.Naive]
MapSize = 400000

[Cache.Redis]
Url= "localhost:6379"
Password = "yourpassword"
1 change: 1 addition & 0 deletions internal/api/etag.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ func DecodeStrongEtag(encodedStrongEtag string) (*StrongEtagData, error) {
return MakeStrongEtag(collectionName, sridValue, format, weakEtag), nil
}

// Returns a WeakEtag string from a etag. If etag starts with W/ value is split to get WeakEtag otherwise we decode etag to strong format to get WeakEtag
func EtagToWeakEtag(strongEtag string) (string, error) {

weakEtagValue := ""
Expand Down
30 changes: 18 additions & 12 deletions internal/conf/appconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,26 @@ var setVersion string = "1.3"
// AppConfiguration is the set of global application configuration constants.
type AppConfiguration struct {
// AppName name of the software
Name string
Name string
EnvPrefix string
// AppVersion version number of the software
Version string
EnvDBURL string
EnvCache string
EnvCacheSize string
EnvPrefix string
Version string
// Database URL
EnvDBURL string
// Cache configuration
EnvCacheType string
EnvCacheNaiveSize string
EnvCacheRedisUrl string
EnvCacheRedisPassword string
}

var AppConfig = AppConfiguration{
Name: "pg_featureserv",
Version: setVersion,
EnvDBURL: "DATABASE_URL",
EnvCache: "PGFS_CACHE",
EnvCacheSize: "PGFS_CACHESIZE",
EnvPrefix: "PGFS",
Name: "pg_featureserv",
Version: setVersion,
EnvPrefix: "PGFS",
EnvDBURL: "DATABASE_URL",
EnvCacheType: "PGFS_CACHE_TYPE",
EnvCacheNaiveSize: "PGFS_CACHE_NAIVE_SIZE",
EnvCacheRedisUrl: "PGFS_CACHE_REDIS_URL",
EnvCacheRedisPassword: "PGFS_CACHE_REDIS_PASSWORD",
}
107 changes: 107 additions & 0 deletions internal/conf/cacheconfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package conf

/*
Copyright 2022 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Date : November 2022
Authors : Jean-Marie KERLOCH (jean-marie dot kerloch at oslandia dot com)
*/

import (
"fmt"
"os"
"strconv"

log "github.com/sirupsen/logrus"
)

// Cache config
type Cache struct {
Type string
Naive NaiveCacheConfig
Redis RedisCacheConfig
}

// Init Cache configuration from environnement variables
func (cache *Cache) InitFromEnvVariables() {
origin := "config file"
cacheTypeInput := os.Getenv(AppConfig.EnvCacheType)
if cacheTypeInput != "" {
origin = "environment variable"
cache.Type = cacheTypeInput
}
if cache.Type != "Disabled" && cache.Type != "Naive" && cache.Type != "Redis" {
log.Fatal(fmt.Errorf("Invalid cache type: Disabled, Naive and Redis are supported. %v defined", cache.Type))
}

log.Infof("Using cache type %s set from %s", Configuration.Cache.Type, origin)

if Configuration.Cache.Type == "Naive" {
cache.Naive.InitFromEnvVariables()
} else if Configuration.Cache.Type == "Redis" {
cache.Redis.InitFromEnvVariables()
}
}

func (cache *Cache) DumpConfig() {
log.Debug(" --- Cache ---")
log.Debugf(" Type = %v", Configuration.Cache.Type)
if Configuration.Cache.Type == "Naive" {
log.Debugf(" NaiveCache.MapSize = %v", Configuration.Cache.Naive.MapSize)
}
if Configuration.Cache.Type == "Redis" {
log.Debugf(" RedisCache.Url = %v", Configuration.Cache.Redis.Url)
}
}

// NaiveCache config
type NaiveCacheConfig struct {
MapSize int
}

// Init Naive cache configuration from environnement variables
func (cache *NaiveCacheConfig) InitFromEnvVariables() {
origin := "config file"
if cacheSizeInput, err := strconv.Atoi(os.Getenv(AppConfig.EnvCacheNaiveSize)); cacheSizeInput != 0 {
if err != nil {
log.Fatal(fmt.Errorf("fatal error reading env variable: %v", err))
}
cache.MapSize = cacheSizeInput
origin = "environment variable"
}
log.Infof("Using etag cache size set from %s (%d entries)", origin, cache.MapSize)
}

// RedisCache config
type RedisCacheConfig struct {
Url string
Password string
}

// Init Redis cache configuration from environnement variables
func (cache *RedisCacheConfig) InitFromEnvVariables() {
origin := "config file"
urlInput := os.Getenv(AppConfig.EnvCacheRedisUrl)
if urlInput != "" {
origin = "environment variable"
cache.Url = urlInput
}
log.Infof("Using Redis cache url %s set from %s", cache.Url, origin)

origin = "config file"
passwordInput := os.Getenv(AppConfig.EnvCacheRedisPassword)
if passwordInput != "" {
origin = "environment variable"
cache.Password = passwordInput
}
log.Infof("Using Redis cache password set from %s", origin)
}
53 changes: 9 additions & 44 deletions internal/conf/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package conf
import (
"fmt"
"os"
"strconv"
"strings"

log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -50,8 +49,11 @@ func setDefaultConfig() {
viper.SetDefault("Database.FunctionIncludes", []string{"postgisftw"})
viper.SetDefault("Database.AllowWrite", false)

viper.SetDefault("Cache.MapSize", 400000)
viper.SetDefault("Cache.IsActive", true)
viper.SetDefault("Cache.Type", "Naive")
viper.SetDefault("Cache.Naive.MapSize", 400000)
viper.SetDefault("Cache.Redis.Url", "localhost:6379")
viper.SetDefault("Cache.Redis.Password", "")

viper.SetDefault("Paging.LimitDefault", 10)
viper.SetDefault("Paging.LimitMax", 1000)

Expand Down Expand Up @@ -105,12 +107,6 @@ type Database struct {
AllowWrite bool
}

// Cache config
type Cache struct {
MapSize int
IsActive bool
}

// Metadata config
type Metadata struct {
Title string //`mapstructure:"METADATA_TITLE"`
Expand Down Expand Up @@ -178,41 +174,8 @@ func InitConfig(configFilename string, isDebug bool) {
}
log.Infof("Using database connection info from %v", dbconnSrc)

// Cache activation
cacheActivated := Configuration.Cache.IsActive
cacheActivationSrc := "config file"

value, present := os.LookupEnv(AppConfig.EnvCache)
if present {
cacheActivationEnvValue, err := strconv.ParseBool(value)
if err != nil {
log.Fatal(fmt.Errorf("fatal error reading env variable: %v", err))
}
cacheActivated = cacheActivationEnvValue
Configuration.Cache.IsActive = cacheActivationEnvValue
cacheActivationSrc = fmt.Sprintf("environment variable %s", AppConfig.EnvCache)
}

cacheActivationStatus := ""
if cacheActivated {
cacheActivationStatus = "active"
} else {
cacheActivationStatus = "disabled"
}
log.Infof("Etag cache mode set from %s (%s)", cacheActivationSrc, cacheActivationStatus)

// Cache size configuration
if cacheActivated {
cacheSizeSrc := "config file"
if cacheSizeInput, err := strconv.Atoi(os.Getenv(AppConfig.EnvCacheSize)); cacheSizeInput != 0 {
if err != nil {
log.Fatal(fmt.Errorf("fatal error reading env variable: %v", err))
}
Configuration.Cache.MapSize = cacheSizeInput
cacheSizeSrc = "environment variable"
}
log.Infof("Using etag cache size set from %s (%d entries)", cacheSizeSrc, Configuration.Cache.MapSize)
}
// Cache initialization
Configuration.Cache.InitFromEnvVariables()

// sanitize the configuration
Configuration.Server.BasePath = strings.TrimRight(Configuration.Server.BasePath, "/")
Expand All @@ -230,4 +193,6 @@ func DumpConfig() {
log.Debugf(" TableIncludes = %v", Configuration.Database.TableIncludes)
log.Debugf(" TableExcludes = %v", Configuration.Database.TableExcludes)
log.Debugf(" FunctionIncludes = %v", Configuration.Database.FunctionIncludes)

Configuration.Cache.DumpConfig()
}
10 changes: 4 additions & 6 deletions internal/data/cache_redis.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package data
*/

import (
"bytes"
"context"
"fmt"

Expand All @@ -30,9 +29,10 @@ type CacheRedis struct {
ctx context.Context
}

func (cache *CacheRedis) Init(addr string) error {
func (cache *CacheRedis) Init(addr string, password string) error {
cache.client = redis.NewClient(&redis.Options{
Addr: addr,
Addr: addr,
Password: password,
})
cache.ctx = context.Background()
_, err := cache.client.Ping(cache.ctx).Result()
Expand Down Expand Up @@ -69,9 +69,7 @@ func (cache CacheRedis) RemoveWeakEtag(weakEtag string) (bool, error) {
}

func (cache CacheRedis) String() string {
b := new(bytes.Buffer)
// TODO
return b.String()
return fmt.Sprintf("Redis Cache running on %s", cache.client.Options().Addr)
}

func (cache CacheRedis) Type() string {
Expand Down
Loading

0 comments on commit dbce243

Please sign in to comment.