Skip to content

Commit

Permalink
Changed the LoadPaths function.
Browse files Browse the repository at this point in the history 
Squashing all the commits for following issue:
Fix for the issue when OPA doesnot load tarball on cmd line as a bundle.

Fixes #5879

Signed-off-by: Yogesh Sinha <[email protected]>
  • Loading branch information
@yogisinha @ashutosh-narkar
yogisinha authored and ashutosh-narkar committed Aug 30, 2023
1 parent dea7c10 commit df0addf
Show file tree
Hide file tree
Showing 2 changed files with 264 additions and 3 deletions.
29 changes: 26 additions & 3 deletions internal/runtime/init/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,16 @@ func LoadPaths(paths []string,
caps = ast.CapabilitiesForThisVersion()
}

// tar.gz files are automatically loaded as bundles
var likelyBundles, nonBundlePaths []string
if !asBundle {
likelyBundles, nonBundlePaths = splitByTarGzExt(paths)
paths = likelyBundles
}

var result LoadPathsResult
var err error

if asBundle {
if asBundle || len(likelyBundles) > 0 {
result.Bundles = make(map[string]*bundle.Bundle, len(paths))
for _, path := range paths {
result.Bundles[path], err = loader.NewFileLoader().
Expand All @@ -145,14 +151,18 @@ func LoadPaths(paths []string,
return nil, err
}
}
}

if len(nonBundlePaths) == 0 {
return &result, nil
}

files, err := loader.NewFileLoader().
WithFS(fsys).
WithProcessAnnotation(processAnnotations).
WithCapabilities(caps).
Filtered(paths, filter)
Filtered(nonBundlePaths, filter)

if err != nil {
return nil, err
}
Expand All @@ -162,6 +172,19 @@ func LoadPaths(paths []string,
return &result, nil
}

// splitByTarGzExt splits the paths in 2 groups. Ones with .tar.gz and another with
// non .tar.gz extensions.
func splitByTarGzExt(paths []string) (targzs []string, nonTargzs []string) {
for _, path := range paths {
if strings.HasSuffix(path, ".tar.gz") {
targzs = append(targzs, path)
} else {
nonTargzs = append(nonTargzs, path)
}
}
return
}

// WalkPaths reads data and policy from the given paths and returns a set of bundle directory loaders
// or descriptors that contain information about files.
func WalkPaths(paths []string, filter loader.Filter, asBundle bool) (*WalkPathsResult, error) {
Expand Down
238 changes: 238 additions & 0 deletions internal/runtime/init/init_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
package init

import (
"bytes"
"context"
"encoding/json"
"io"
"io/fs"
"os"
Expand All @@ -14,6 +16,9 @@ import (
"strings"
"testing"

"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/bundle"
"github.com/open-policy-agent/opa/internal/file/archive"
"github.com/open-policy-agent/opa/loader"
"github.com/open-policy-agent/opa/storage"
inmem "github.com/open-policy-agent/opa/storage/inmem/test"
Expand Down Expand Up @@ -183,6 +188,239 @@ p = true { 1 = 2 }`
}
}

func TestLoadTarGzsInBundleAndNonBundleMode(t *testing.T) {

type bundleInfo struct {
fileName string
files [][2]string
expBundle bundle.Bundle
}

bundle1TarGz := bundleInfo{
fileName: "bundle1.tar.gz",
files: [][2]string{
{"/a/data.json", `{"foo": "bar1", "x": {"y": {"z": [1]}}}`},
{"/a/.manifest", `{"roots": ["a"]}`},
},
expBundle: bundle.Bundle{
Manifest: bundle.Manifest{
Roots: &[]string{"a"},
},
Data: map[string]interface{}{
"a": map[string]interface{}{
"foo": "bar1",
"x": map[string]interface{}{
"y": map[string]interface{}{
"z": []interface{}{json.Number("1")},
},
},
},
},
},
}

bundle2TarGz := bundleInfo{
fileName: "bundle2.tar.gz",
files: [][2]string{
{"/b/data.json", `{"foo": "bar2", "x": {"y": {"z": [1]}}}`},
{"/b/.manifest", `{"roots": ["b"]}`},
},
expBundle: bundle.Bundle{
Manifest: bundle.Manifest{
Roots: &[]string{"b"},
},
Data: map[string]interface{}{
"b": map[string]interface{}{
"foo": "bar2",
"x": map[string]interface{}{
"y": map[string]interface{}{
"z": []interface{}{json.Number("1")},
},
},
},
},
},
}

bundle1Folder := map[string]string{
"/bundle1/a/data.json": `{"foo1": "bar2", "x": {"y": {"z": [2]}}}`,
"/bundle1/a/.manifest": `{"roots": ["a"]}`,
"/bundle1/a/foo.rego": `package a.b.y`,
}

modulePath := "/bundle1/a/foo.rego"
module := `package a.b.y`

bundle1FolderInfo := bundleInfo{
fileName: "bundle1",
expBundle: bundle.Bundle{
Manifest: bundle.Manifest{
Roots: &[]string{"a"},
},
Data: map[string]interface{}{
"a": map[string]interface{}{
"foo1": "bar2",
"x": map[string]interface{}{
"y": map[string]interface{}{
"z": []interface{}{json.Number("2")},
},
},
},
},
Modules: []bundle.ModuleFile{
{
URL: modulePath,
Path: modulePath,
Parsed: ast.MustParseModule(module),
Raw: []byte(module),
},
},
},
}

tests := []struct {
note string
bundleInfoTC []bundleInfo
folderContent map[string]string
expectedBundles int
expectedModules int
asBundle bool
}{
{
note: "load multiple bundles. one tar.gz and one folder. Bundle mode is true",
folderContent: bundle1Folder,
bundleInfoTC: []bundleInfo{
bundle1TarGz,
bundle1FolderInfo,
},
expectedBundles: 2,
expectedModules: 0,
asBundle: true,
},
{
note: "load multiple bundles. one tar.gz and one folder. Bundle mode is false",
folderContent: bundle1Folder,
bundleInfoTC: []bundleInfo{
bundle1TarGz,
bundle1FolderInfo,
},
expectedBundles: 1,
expectedModules: 1,
asBundle: false,
},
{
note: "load multiple bundles. two tar.gz and one folder. Bundle mode is true",
folderContent: bundle1Folder,
bundleInfoTC: []bundleInfo{
bundle1TarGz,
bundle2TarGz,
bundle1FolderInfo,
},
expectedBundles: 3,
expectedModules: 0,
asBundle: true,
},
{
note: "load multiple bundles. two tar.gz and one folder. Bundle mode is false",
folderContent: bundle1Folder,
bundleInfoTC: []bundleInfo{
bundle1TarGz,
bundle2TarGz,
bundle1FolderInfo,
},
expectedBundles: 2,
expectedModules: 1,
asBundle: false,
},
{
note: "load just one folder. Bundle mode is true",
folderContent: bundle1Folder,
bundleInfoTC: []bundleInfo{
bundle1FolderInfo,
},
expectedBundles: 1,
expectedModules: 0,
asBundle: true,
},
{
note: "load just one folder. Bundle mode is false",
folderContent: bundle1Folder,
bundleInfoTC: []bundleInfo{
bundle1FolderInfo,
},
expectedBundles: 0,
expectedModules: 1,
asBundle: false,
},
}

for _, tc := range tests {
t.Run(tc.note, func(t *testing.T) {

test.WithTempFS(tc.folderContent, func(rootDir string) {
paths := []string{}
for _, bdlInfo := range tc.bundleInfoTC {
if strings.HasSuffix(bdlInfo.fileName, ".tar.gz") {

// Create the tar gz files temporarily
buf := archive.MustWriteTarGz(bdlInfo.files)
bundleFile := filepath.Join(rootDir, bdlInfo.fileName)
out, err := os.Create(bundleFile)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
_, err = out.Write(buf.Bytes())
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
}

paths = append(paths, filepath.Join(rootDir, bdlInfo.fileName))
}

loaded, err := LoadPaths(paths, nil, tc.asBundle, nil, true, false, nil, nil)
if err != nil {
t.Fatal("Failed LoadPaths ", err)
}
if tc.expectedBundles != len(loaded.Bundles) {
t.Fatalf("Expected %d bundles, got %d", tc.expectedBundles, len(loaded.Bundles))
}
if tc.expectedModules != len(loaded.Files.Modules) {
t.Fatalf("Expected %d modules, got %d", tc.expectedModules, len(loaded.Files.Modules))
}

// Testing the content
for path, actual := range loaded.Bundles {
for _, bdlInfo := range tc.bundleInfoTC {
if strings.HasSuffix(path, bdlInfo.fileName) {
var buf bytes.Buffer
if err := bundle.NewWriter(&buf).Write(bdlInfo.expBundle); err != nil {
t.Fatal(err)
}

expected, err := bundle.NewReader(&buf).Read()
if err != nil {
t.Fatal(err)
}

// adjusting the URL and Path due to /tmp/ path
if len(bdlInfo.expBundle.Modules) > 0 {
expected.Modules[0].URL = rootDir + expected.Modules[0].URL
expected.Modules[0].Path = rootDir + expected.Modules[0].Path
}

if !expected.Equal(*actual) {
t.Fatalf("\nExpected: %+v\nGot: %+v", expected, actual)
}
}
}
}
})
})
}

}

func TestWalkPaths(t *testing.T) {
files := map[string]string{
"/bundle1/a/data.json": `{"foo": "bar1", "x": {"y": {"z": [1]}}}`,
Expand Down

0 comments on commit df0addf

Please sign in to comment.