diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go index 4dcb2c61842..f21770e4101 100644 --- a/internal/provider/file/file.go +++ b/internal/provider/file/file.go @@ -58,7 +58,14 @@ func (p *Provider) Start(ctx context.Context) error { }() // Start runnable servers. - go p.startHealthProbeServer(ctx) + readyz := false + var readyzChecker healthz.Checker = func(req *http.Request) error { + if !readyz { + return fmt.Errorf("file provider not ready yet") + } + return nil + } + go p.startHealthProbeServer(ctx, readyzChecker) initDirs, initFiles := path.ListDirsAndFiles(p.paths) // Initially load resources from paths on host. @@ -83,7 +90,9 @@ func (p *Provider) Start(ctx context.Context) error { }(ch) } + readyz = true curDirs, curFiles := initDirs.Clone(), initFiles.Clone() + initFilesParent := path.GetParentDirs(initFiles.UnsortedList()) for { select { case <-ctx.Done(): @@ -102,29 +111,35 @@ func (p *Provider) Start(ctx context.Context) error { // temporary file when file is saved. So the watcher will only receive: // - Create event, with name "filename~". // - Remove event, with name "filename", but the file actually exist. - if initFiles.Has(event.Name) { + if initFilesParent.Has(filepath.Dir(event.Name)) { p.logger.Info("file changed", "op", event.Op, "name", event.Name) // For Write event, the file definitely exist. - if event.Has(fsnotify.Write) { + if initFiles.Has(event.Name) && event.Has(fsnotify.Write) { goto handle } - _, err := os.Lstat(event.Name) - if err != nil && os.IsNotExist(err) { - curFiles.Delete(event.Name) - } else { - curFiles.Insert(event.Name) + // Iter over the watched files to see the different. + for f := range initFiles { + _, err := os.Lstat(f) + if err != nil { + if os.IsNotExist(err) { + curFiles.Delete(f) + } else { + p.logger.Error(err, "stat file error", "name", f) + } + } else { + curFiles.Insert(f) + } } goto handle } // Ignore the hidden or temporary file related change event under a directory. - if _, name := filepath.Split(event.Name); strings.HasPrefix(name, ".") || - strings.HasSuffix(name, "~") { + if _, name := filepath.Split(event.Name); strings.HasPrefix(name, ".") || strings.HasSuffix(name, "~") { continue } - p.logger.Info("file changed", "op", event.Op, "name", event.Name) + p.logger.Info("file changed", "op", event.Op, "name", event.Name, "dir", filepath.Dir(event.Name)) switch event.Op { case fsnotify.Create, fsnotify.Write, fsnotify.Remove: @@ -142,7 +157,7 @@ func (p *Provider) Start(ctx context.Context) error { } } -func (p *Provider) startHealthProbeServer(ctx context.Context) { +func (p *Provider) startHealthProbeServer(ctx context.Context, readyzChecker healthz.Checker) { const ( readyzEndpoint = "/readyz" healthzEndpoint = "/healthz" @@ -159,7 +174,7 @@ func (p *Provider) startHealthProbeServer(ctx context.Context) { readyzHandler := &healthz.Handler{ Checks: map[string]healthz.Checker{ - readyzEndpoint: healthz.Ping, + readyzEndpoint: readyzChecker, }, } mux.Handle(readyzEndpoint, http.StripPrefix(readyzEndpoint, readyzHandler)) diff --git a/internal/provider/file/file_test.go b/internal/provider/file/file_test.go new file mode 100644 index 00000000000..b514e3c7942 --- /dev/null +++ b/internal/provider/file/file_test.go @@ -0,0 +1,225 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "context" + "html/template" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" + "sigs.k8s.io/yaml" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi/resource" + "github.com/envoyproxy/gateway/internal/message" +) + +const ( + resourcesUpdateTimeout = 1 * time.Minute + resourcesUpdateTick = 1 * time.Second +) + +type resourcesParam struct { + GatewayClassName string + GatewayName string + GatewayListenerPort string + HTTPRouteName string + BackendName string +} + +func newDefaultResourcesParam() *resourcesParam { + return &resourcesParam{ + GatewayClassName: "eg", + GatewayName: "eg", + GatewayListenerPort: "8888", + HTTPRouteName: "backend", + BackendName: "backend", + } +} + +func newFileProviderConfig(paths []string) (*config.Server, error) { + cfg, err := config.New() + if err != nil { + return nil, err + } + + cfg.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: paths, + }, + }, + }, + } + return cfg, nil +} + +func TestFileProvider(t *testing.T) { + watchFileBase, _ := os.MkdirTemp(os.TempDir(), "test-files-*") + watchFilePath := filepath.Join(watchFileBase, "test.yaml") + watchDirPath, _ := os.MkdirTemp(os.TempDir(), "test-dir-*") + // Prepare the watched test file. + writeResourcesFile(t, "testdata/resources.tmpl", watchFilePath, newDefaultResourcesParam()) + require.FileExists(t, watchFilePath) + require.DirExists(t, watchDirPath) + + cfg, err := newFileProviderConfig([]string{watchFilePath, watchDirPath}) + require.NoError(t, err) + pResources := new(message.ProviderResources) + fp, err := New(cfg, pResources) + require.NoError(t, err) + // Start file provider. + go func() { + if err := fp.Start(context.Background()); err != nil { + t.Errorf("failed to start file provider: %v", err) + } + }() + + // Wait for file provider to be ready. + waitFileProviderReady(t) + + require.Equal(t, "gateway.envoyproxy.io/gatewayclass-controller", fp.resourcesStore.name) + + t.Run("initial resource load", func(t *testing.T) { + require.NotZero(t, pResources.GatewayAPIResources.Len()) + resources := pResources.GetResourcesByGatewayClass("eg") + require.NotNil(t, resources) + + want := &resource.Resources{} + mustUnmarshal(t, "testdata/resources.all.yaml", want) + + opts := []cmp.Option{ + cmpopts.IgnoreFields(resource.Resources{}, "serviceMap"), + cmpopts.EquateEmpty(), + } + require.Empty(t, cmp.Diff(want, resources, opts...)) + }) + + t.Run("rename the watched file then rename it back", func(t *testing.T) { + // Rename it + renameFilePath := filepath.Join(watchFileBase, "foobar.yaml") + err := os.Rename(watchFilePath, renameFilePath) + require.NoError(t, err) + require.Eventually(t, func() bool { + return pResources.GetResourcesByGatewayClass("eg") == nil + }, resourcesUpdateTimeout, resourcesUpdateTick) + + // Rename it back + err = os.Rename(renameFilePath, watchFilePath) + require.NoError(t, err) + require.Eventually(t, func() bool { + return pResources.GetResourcesByGatewayClass("eg") != nil + }, resourcesUpdateTimeout, resourcesUpdateTick) + + resources := pResources.GetResourcesByGatewayClass("eg") + want := &resource.Resources{} + mustUnmarshal(t, "testdata/resources.all.yaml", want) + + opts := []cmp.Option{ + cmpopts.IgnoreFields(resource.Resources{}, "serviceMap"), + cmpopts.EquateEmpty(), + } + require.Empty(t, cmp.Diff(want, resources, opts...)) + }) + + t.Run("remove the watched file", func(t *testing.T) { + err := os.Remove(watchFilePath) + require.NoError(t, err) + require.Eventually(t, func() bool { + return pResources.GetResourcesByGatewayClass("eg") == nil + }, resourcesUpdateTimeout, resourcesUpdateTick) + }) + + t.Run("add a file in watched dir", func(t *testing.T) { + // Write a new file under watched directory. + newFilePath := filepath.Join(watchDirPath, "test.yaml") + writeResourcesFile(t, "testdata/resources.tmpl", newFilePath, newDefaultResourcesParam()) + + require.Eventually(t, func() bool { + return pResources.GetResourcesByGatewayClass("eg") != nil + }, resourcesUpdateTimeout, resourcesUpdateTick) + + resources := pResources.GetResourcesByGatewayClass("eg") + want := &resource.Resources{} + mustUnmarshal(t, "testdata/resources.all.yaml", want) + + opts := []cmp.Option{ + cmpopts.IgnoreFields(resource.Resources{}, "serviceMap"), + cmpopts.EquateEmpty(), + } + require.Empty(t, cmp.Diff(want, resources, opts...)) + }) + + t.Run("remove a file in watched dir", func(t *testing.T) { + newFilePath := filepath.Join(watchDirPath, "test.yaml") + err := os.Remove(newFilePath) + require.NoError(t, err) + require.Eventually(t, func() bool { + return pResources.GetResourcesByGatewayClass("eg") == nil + }, resourcesUpdateTimeout, resourcesUpdateTick) + }) + + t.Cleanup(func() { + _ = os.RemoveAll(watchFileBase) + _ = os.RemoveAll(watchDirPath) + }) +} + +func writeResourcesFile(t *testing.T, tmpl, dst string, params *resourcesParam) { + dstFile, err := os.Create(dst) + require.NoError(t, err) + + // Write parametes into target file. + tmplFile, err := template.ParseFiles(tmpl) + require.NoError(t, err) + + err = tmplFile.Execute(dstFile, params) + require.NoError(t, err) + require.NoError(t, dstFile.Close()) +} + +func waitFileProviderReady(t *testing.T) { + require.Eventually(t, func() bool { + resp, err := http.Get("http://localhost:8081/readyz") + if err != nil { + t.Logf("failed to get from heathlz server") + return false + } + + body, err := io.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + t.Logf("failed to get body from response") + return false + } + + if string(body) != "ok" { + t.Logf("the file provider is not ready yet") + return false + } + return true + }, 3*resourcesUpdateTimeout, resourcesUpdateTick) +} + +func mustUnmarshal(t *testing.T, path string, out interface{}) { + t.Helper() + + content, err := os.ReadFile(path) + require.NoError(t, err) + require.NoError(t, yaml.UnmarshalStrict(content, out, yaml.DisallowUnknownFields)) +} diff --git a/internal/provider/file/testdata/resources.all.yaml b/internal/provider/file/testdata/resources.all.yaml new file mode 100644 index 00000000000..079647dc6c0 --- /dev/null +++ b/internal/provider/file/testdata/resources.all.yaml @@ -0,0 +1,62 @@ +backends: +- kind: Backend + metadata: + creationTimestamp: null + name: backend + namespace: envoy-gateway-system + spec: + endpoints: + - ip: + address: 0.0.0.0 + port: 3000 + status: {} +gatewayClass: + kind: GatewayClass + metadata: + creationTimestamp: null + name: eg + namespace: envoy-gateway-system + spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + status: {} +gateways: +- kind: Gateway + metadata: + creationTimestamp: null + name: eg + namespace: envoy-gateway-system + spec: + gatewayClassName: eg + listeners: + - name: http + port: 8888 + protocol: HTTP + status: {} +httpRoutes: +- kind: HTTPRoute + metadata: + creationTimestamp: null + name: backend + namespace: envoy-gateway-system + spec: + hostnames: + - www.example.com + parentRefs: + - name: eg + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend + matches: + - path: + type: PathPrefix + value: / + status: + parents: null +namespaces: +- metadata: + creationTimestamp: null + name: envoy-gateway-system + spec: {} + status: {} diff --git a/internal/provider/file/testdata/resources.tmpl b/internal/provider/file/testdata/resources.tmpl new file mode 100644 index 00000000000..f34bf1e0c3c --- /dev/null +++ b/internal/provider/file/testdata/resources.tmpl @@ -0,0 +1,46 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: {{.GatewayClassName}} +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: {{.GatewayName}} +spec: + gatewayClassName: {{.GatewayClassName}} + listeners: + - name: http + protocol: HTTP + port: {{.GatewayListenerPort}} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{.HTTPRouteName}} +spec: + parentRefs: + - name: {{.GatewayName}} + hostnames: + - "www.example.com" + rules: + - backendRefs: + - group: "gateway.envoyproxy.io" + kind: Backend + name: {{.BackendName}} + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: {{.BackendName}} +spec: + endpoints: + - ip: + address: 0.0.0.0 + port: 3000 diff --git a/internal/utils/path/path.go b/internal/utils/path/path.go index 4291dd58848..5a0793eff1e 100644 --- a/internal/utils/path/path.go +++ b/internal/utils/path/path.go @@ -56,3 +56,12 @@ func ListDirsAndFiles(paths []string) (dirs sets.Set[string], files sets.Set[str return } + +// GetParentDirs returns all the parent directories of given files. +func GetParentDirs(files []string) sets.Set[string] { + parents := sets.New[string]() + for _, f := range files { + parents.Insert(filepath.Dir(f)) + } + return parents +} diff --git a/internal/utils/path/path_test.go b/internal/utils/path/path_test.go index 8b3db14784d..8d1883ea336 100644 --- a/internal/utils/path/path_test.go +++ b/internal/utils/path/path_test.go @@ -64,3 +64,62 @@ func TestListDirsAndFiles(t *testing.T) { }) } } + +func TestGetParentDirs(t *testing.T) { + aPaths := path.Join("a") + bPaths := path.Join("a", "b") + cPaths := path.Join("a", "b", "c") + + testCases := []struct { + name string + paths []string + expectParentDirs []string + }{ + { + name: "all files", + paths: []string{ + path.Join(cPaths, "foo"), + path.Join(bPaths, "bar"), + }, + expectParentDirs: []string{ + cPaths, + bPaths, + }, + }, + { + name: "all dirs", + paths: []string{ + bPaths + "/", + cPaths + "/", + }, + expectParentDirs: []string{ + bPaths, + cPaths, + }, + }, + { + name: "mixed files and dirs", + paths: []string{ + path.Join(cPaths, "foo"), + path.Join(cPaths, "bar"), + path.Join(bPaths, "foo"), + path.Join(bPaths, "bar"), + aPaths + "/", + bPaths + "/", + cPaths + "/", + }, + expectParentDirs: []string{ + cPaths, + bPaths, + aPaths, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + parents := GetParentDirs(tc.paths) + require.ElementsMatch(t, parents.UnsortedList(), tc.expectParentDirs) + }) + } +}