Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: cache OPL when loading from HTTP(S) #1429

Merged
merged 1 commit into from
Sep 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions internal/driver/config/opl_cache_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0

package config

import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"

"github.com/ory/x/configx"
"github.com/ory/x/logrusx"
"github.com/stretchr/testify/require"
)

func TestNewOPLConfigWatcher(t *testing.T) {
hits := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits++
io.WriteString(w, testOPL)

Check failure on line 22 in internal/driver/config/opl_cache_test.go

View workflow job for this annotation

GitHub Actions / Run lints and checks

Error return value of `io.WriteString` is not checked (errcheck)
}))
t.Cleanup(ts.Close)
ctx := context.Background()
cfg, err := NewDefault(ctx, nil, logrusx.New("", ""), configx.SkipValidation())
require.NoError(t, err)
cw, err := newOPLConfigWatcher(ctx, cfg, ts.URL)
require.NoError(t, err)
require.Equal(t, 1, hits, "HTTP request made")
_, err = cw.GetNamespaceByName(ctx, "User")
require.NoError(t, err)
_, err = cw.GetNamespaceByName(ctx, "Document")
require.NoError(t, err)

cache.Wait()

cw, err = newOPLConfigWatcher(ctx, cfg, ts.URL)
require.NoError(t, err)
require.Equal(t, 1, hits, "content was cached")
_, err = cw.GetNamespaceByName(ctx, "User")
require.NoError(t, err)
_, err = cw.GetNamespaceByName(ctx, "Document")
require.NoError(t, err)
}

var testOPL = `
import { Namespace } from "@ory/keto-namespace-types"

class User implements Namespace {}
class Document implements Namespace {}
`
30 changes: 28 additions & 2 deletions internal/driver/config/opl_config_namespace_watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@
package config

import (
"bytes"
"context"
"fmt"
"io"
"sync"
"time"

"github.com/dgraph-io/ristretto"
"github.com/ory/x/logrusx"
"github.com/ory/x/urlx"
"github.com/ory/x/watcherx"
Expand All @@ -33,7 +36,14 @@ type (
}
)

var _ namespace.Manager = (*oplConfigWatcher)(nil)
var (
_ namespace.Manager = (*oplConfigWatcher)(nil)
cache, _ = ristretto.NewCache(&ristretto.Config{
MaxCost: 20_000_000, // 20 MB max size, each item ca. 10 KB => max 2000 items
NumCounters: 20_000, // max 2000 items => 20000 counters
BufferItems: 64,
})
aeneasr marked this conversation as resolved.
Show resolved Hide resolved
)

func newOPLConfigWatcher(ctx context.Context, c *Config, target string) (*oplConfigWatcher, error) {
nw := &oplConfigWatcher{
Expand All @@ -51,14 +61,30 @@ func newOPLConfigWatcher(ctx context.Context, c *Config, target string) (*oplCon
switch targetUrl.Scheme {
case "file", "":
return nw, watchTarget(ctx, target, nw, c.l)
case "http", "https", "base64":
case "base64":
file, err := c.Fetcher().FetchContext(ctx, target)
if err != nil {
return nil, err
}
nw.files.byPath[targetUrl.String()] = file
nw.parseFiles()
return nw, err
case "http", "https":
var file io.Reader
if item, ok := cache.Get(target); ok {
file = bytes.NewReader(item.([]byte))
} else {
buf, err := c.Fetcher().FetchContext(ctx, target)
if err != nil {
return nil, err
}
b := buf.Bytes()
cache.SetWithTTL(target, b, int64(cap(b)), 30*time.Minute)
file = bytes.NewReader(b)
}
nw.files.byPath[targetUrl.String()] = file
nw.parseFiles()
return nw, err
default:
return nil, fmt.Errorf("unexpected url scheme: %q", targetUrl.Scheme)
}
Expand Down
Loading