Skip to content

Commit

Permalink
feat: cache OPL when loading from HTTP(S) (#1429)
Browse files Browse the repository at this point in the history
  • Loading branch information
alnr authored Sep 19, 2023
1 parent b9670e3 commit b89ce02
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 2 deletions.
52 changes: 52 additions & 0 deletions internal/driver/config/opl_cache_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0

package config

import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"

"github.com/ory/x/configx"
"github.com/ory/x/logrusx"
"github.com/stretchr/testify/require"
)

func TestNewOPLConfigWatcher(t *testing.T) {
hits := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits++
io.WriteString(w, testOPL)

Check failure on line 22 in internal/driver/config/opl_cache_test.go

View workflow job for this annotation

GitHub Actions / Run lints and checks

Error return value of `io.WriteString` is not checked (errcheck)
}))
t.Cleanup(ts.Close)
ctx := context.Background()
cfg, err := NewDefault(ctx, nil, logrusx.New("", ""), configx.SkipValidation())
require.NoError(t, err)
cw, err := newOPLConfigWatcher(ctx, cfg, ts.URL)
require.NoError(t, err)
require.Equal(t, 1, hits, "HTTP request made")
_, err = cw.GetNamespaceByName(ctx, "User")
require.NoError(t, err)
_, err = cw.GetNamespaceByName(ctx, "Document")
require.NoError(t, err)

cache.Wait()

cw, err = newOPLConfigWatcher(ctx, cfg, ts.URL)
require.NoError(t, err)
require.Equal(t, 1, hits, "content was cached")
_, err = cw.GetNamespaceByName(ctx, "User")
require.NoError(t, err)
_, err = cw.GetNamespaceByName(ctx, "Document")
require.NoError(t, err)
}

var testOPL = `
import { Namespace } from "@ory/keto-namespace-types"
class User implements Namespace {}
class Document implements Namespace {}
`
30 changes: 28 additions & 2 deletions internal/driver/config/opl_config_namespace_watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@
package config

import (
"bytes"
"context"
"fmt"
"io"
"sync"
"time"

"github.com/dgraph-io/ristretto"
"github.com/ory/x/logrusx"
"github.com/ory/x/urlx"
"github.com/ory/x/watcherx"
Expand All @@ -33,7 +36,14 @@ type (
}
)

var _ namespace.Manager = (*oplConfigWatcher)(nil)
var (
_ namespace.Manager = (*oplConfigWatcher)(nil)
cache, _ = ristretto.NewCache(&ristretto.Config{
MaxCost: 20_000_000, // 20 MB max size, each item ca. 10 KB => max 2000 items
NumCounters: 20_000, // max 2000 items => 20000 counters
BufferItems: 64,
})
)

func newOPLConfigWatcher(ctx context.Context, c *Config, target string) (*oplConfigWatcher, error) {
nw := &oplConfigWatcher{
Expand All @@ -51,14 +61,30 @@ func newOPLConfigWatcher(ctx context.Context, c *Config, target string) (*oplCon
switch targetUrl.Scheme {
case "file", "":
return nw, watchTarget(ctx, target, nw, c.l)
case "http", "https", "base64":
case "base64":
file, err := c.Fetcher().FetchContext(ctx, target)
if err != nil {
return nil, err
}
nw.files.byPath[targetUrl.String()] = file
nw.parseFiles()
return nw, err
case "http", "https":
var file io.Reader
if item, ok := cache.Get(target); ok {
file = bytes.NewReader(item.([]byte))
} else {
buf, err := c.Fetcher().FetchContext(ctx, target)
if err != nil {
return nil, err
}
b := buf.Bytes()
cache.SetWithTTL(target, b, int64(cap(b)), 30*time.Minute)
file = bytes.NewReader(b)
}
nw.files.byPath[targetUrl.String()] = file
nw.parseFiles()
return nw, err
default:
return nil, fmt.Errorf("unexpected url scheme: %q", targetUrl.Scheme)
}
Expand Down

0 comments on commit b89ce02

Please sign in to comment.