From 2ddba54bfe181f7793cd9507446944335ec5f6cd Mon Sep 17 00:00:00 2001 From: David F D Reis Date: Mon, 25 Mar 2024 18:45:34 -0300 Subject: [PATCH 1/7] Copy statedb/couchdb recursively discarding non .json files to create index for public and private data assets. Signed-off-by: David F D Reis --- internal/util/copy.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/util/copy.go b/internal/util/copy.go index 729df65..f5bfe6f 100644 --- a/internal/util/copy.go +++ b/internal/util/copy.go @@ -41,7 +41,7 @@ func CopyImageJSON(logger *log.CmdLogger, src, dest string) error { // CopyIndexFiles copies CouchDB index definitions from source to destination directories. func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { - indexDir := filepath.Join("statedb", "couchdb", "indexes") + indexDir := filepath.Join("statedb", "couchdb") indexSrcDir := filepath.Join(src, MetadataDir, indexDir) indexDestDir := filepath.Join(dest, indexDir) @@ -67,7 +67,22 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { opt := copy.Options{ Skip: func(_ os.FileInfo, src, _ string) (bool, error) { - return !strings.HasSuffix(src, ".json"), nil + logger.Debugf("Source folder to check and skip copy: %s", src) + fileInfo, err := os.Lstat(src) + if err != nil { + return true, fmt.Errorf( + "failed to create CouchDB index definitions from folder %s: %w", + src, + err, + ) + } + if fileInfo.IsDir() { // copy it recursively + logger.Debugf("This is a folder: %s", src) + return false, nil + } else { // any file that will not be a JSON will be skipped + logger.Debugf("This is a file: %s", src) + return !strings.HasSuffix(src, ".json"), nil + } }, } From 3494ea639b5fab0e030aef57183f78eb05c45edd Mon Sep 17 00:00:00 2001 From: David F D Reis Date: Mon, 25 Mar 2024 21:03:33 -0300 Subject: [PATCH 2/7] Typo errors and better debug messages. Signed-off-by: David F D Reis --- internal/util/copy.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/util/copy.go b/internal/util/copy.go index f5bfe6f..c09bfe1 100644 --- a/internal/util/copy.go +++ b/internal/util/copy.go @@ -67,7 +67,7 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { opt := copy.Options{ Skip: func(_ os.FileInfo, src, _ string) (bool, error) { - logger.Debugf("Source folder to check and skip copy: %s", src) + logger.Debugf("Checking source copy path: %s", src) fileInfo, err := os.Lstat(src) if err != nil { return true, fmt.Errorf( @@ -77,11 +77,16 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { ) } if fileInfo.IsDir() { // copy it recursively - logger.Debugf("This is a folder: %s", src) + logger.Debugf("This is a folder, copying: %s", src) return false, nil - } else { // any file that will not be a JSON will be skipped - logger.Debugf("This is a file: %s", src) - return !strings.HasSuffix(src, ".json"), nil + } else { // any non JSON will be skipped + isJsonFile := strings.HasSuffix(src, ".json") + if isJsonFile { + logger.Debugf("This is a JSON file, copying: %s", src) + } else { + logger.Debugf("This is NOT a JSON file, skipping copy: %s", src) + } + return !isJsonFile, nil } }, } From 24c0766eca8c08360e919f6cbde46e4ab5c553c8 Mon Sep 17 00:00:00 2001 From: David F D Reis Date: Tue, 26 Mar 2024 11:23:54 -0300 Subject: [PATCH 3/7] Linding rules and info.IsDir() PR comments Signed-off-by: David F D Reis --- internal/util/copy.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/internal/util/copy.go b/internal/util/copy.go index c09bfe1..b88b56a 100644 --- a/internal/util/copy.go +++ b/internal/util/copy.go @@ -66,9 +66,9 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { } opt := copy.Options{ - Skip: func(_ os.FileInfo, src, _ string) (bool, error) { + Skip: func(info os.FileInfo, src, _ string) (bool, error) { logger.Debugf("Checking source copy path: %s", src) - fileInfo, err := os.Lstat(src) + if err != nil { return true, fmt.Errorf( "failed to create CouchDB index definitions from folder %s: %w", @@ -76,18 +76,15 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { err, ) } - if fileInfo.IsDir() { // copy it recursively + + if info.IsDir() { logger.Debugf("This is a folder, copying: %s", src) + return false, nil - } else { // any non JSON will be skipped - isJsonFile := strings.HasSuffix(src, ".json") - if isJsonFile { - logger.Debugf("This is a JSON file, copying: %s", src) - } else { - logger.Debugf("This is NOT a JSON file, skipping copy: %s", src) - } - return !isJsonFile, nil } + logger.Debugf("Checking if it is a JSON file: %s", src) + + return !strings.HasSuffix(src, ".json"), nil }, } From cb946f3b741e7fa6d6c9d62c3edd49d49a77a9ae Mon Sep 17 00:00:00 2001 From: David F D Reis Date: Tue, 26 Mar 2024 16:43:11 -0300 Subject: [PATCH 4/7] Update tests for relese chaincode command Signed-off-by: David F D Reis --- cmd/release/main_test.go | 24 +++++++++- .../assetCollection/indexes/indexOwner.json | 11 +++++ .../fabCarCollection/indexes/indexOwner.json | 11 +++++ .../subdir/indexes/indexOwner.json | 1 + .../couchdb/subdir/indexes/indexOwner.json | 1 + .../subdir/indexes/subdir/indexOwner.json | 1 + .../statedb/couchdb/subdir/indexes/test.txt | 1 + internal/util/copy.go | 44 +++++++++++++++++-- 8 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/assetCollection/indexes/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/indexes/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/subdir/indexes/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/subdir/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/test.txt diff --git a/cmd/release/main_test.go b/cmd/release/main_test.go index 156a364..b76fded 100644 --- a/cmd/release/main_test.go +++ b/cmd/release/main_test.go @@ -48,8 +48,16 @@ var _ = Describe("Main", func() { indexPath := filepath.Join(tempDir, "statedb", "couchdb", "indexes", "indexOwner.json") Expect(indexPath).To(BeARegularFile()) + + assetPrivateDataCollectionIndexPath := filepath.Join(tempDir, "statedb", "couchdb", "collections", "assetCollection", "indexes", "indexOwner.json") + Expect(assetPrivateDataCollectionIndexPath).To(BeARegularFile(), "Private data index should be copied") + + fabCarPrivateDataCollectionIndexPath := filepath.Join(tempDir, "statedb", "couchdb", "collections", "fabCarCollection", "indexes", "indexOwner.json") + Expect(fabCarPrivateDataCollectionIndexPath).To(BeARegularFile(), "Private data index should be copied") + textPath := filepath.Join(tempDir, "statedb", "couchdb", "indexes", "test.txt") - Expect(textPath).NotTo(BeAnExistingFile()) + Expect(textPath).NotTo(BeAnExistingFile(), "Unexpected files should not be copied") + subdirPath := filepath.Join( tempDir, "statedb", @@ -58,6 +66,18 @@ var _ = Describe("Main", func() { "subdir", "indexOwner.json", ) - Expect(subdirPath).NotTo(BeAnExistingFile()) + Expect(subdirPath).NotTo(BeAnExistingFile(), "Files outside indexes directory should not be copied") + + privateDataCollectionSubdirPath := filepath.Join( + tempDir, + "statedb", + "couchdb", + "collections", + "fabCarCollection", + "subdir", + "indexes", + "indexOwner.json", + ) + Expect(privateDataCollectionSubdirPath).NotTo(BeAnExistingFile(), "Files outside indexes directory should not be copied") }) }) diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/assetCollection/indexes/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/assetCollection/indexes/indexOwner.json new file mode 100644 index 0000000..28ab876 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/assetCollection/indexes/indexOwner.json @@ -0,0 +1,11 @@ +{ + "index": { + "fields": [ + "objectType", + "owner" + ] + }, + "ddoc": "indexOwnerDoc", + "name": "indexOwner", + "type": "json" +} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/indexes/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/indexes/indexOwner.json new file mode 100644 index 0000000..c895391 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/indexes/indexOwner.json @@ -0,0 +1,11 @@ +{ + "index": { + "fields": [ + "model", + "owner" + ] + }, + "ddoc": "indexOwnerDoc", + "name": "indexOwner", + "type": "json" +} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/subdir/indexes/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/subdir/indexes/indexOwner.json new file mode 100644 index 0000000..305f090 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collections/fabCarCollection/subdir/indexes/indexOwner.json @@ -0,0 +1 @@ +{"index":{"fields":["docType","owner"]},"ddoc":"indexOwnerDoc", "name":"indexOwner","type":"json"} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/indexOwner.json new file mode 100644 index 0000000..305f090 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/indexOwner.json @@ -0,0 +1 @@ +{"index":{"fields":["docType","owner"]},"ddoc":"indexOwnerDoc", "name":"indexOwner","type":"json"} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/subdir/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/subdir/indexOwner.json new file mode 100644 index 0000000..305f090 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/subdir/indexOwner.json @@ -0,0 +1 @@ +{"index":{"fields":["docType","owner"]},"ddoc":"indexOwnerDoc", "name":"indexOwner","type":"json"} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/test.txt b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/test.txt new file mode 100644 index 0000000..73709ba --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/subdir/indexes/test.txt @@ -0,0 +1 @@ +Testing diff --git a/internal/util/copy.go b/internal/util/copy.go index b88b56a..1f7552b 100644 --- a/internal/util/copy.go +++ b/internal/util/copy.go @@ -78,10 +78,9 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { } if info.IsDir() { - logger.Debugf("This is a folder, copying: %s", src) - - return false, nil + return skipFolder(logger, indexSrcDir, src) } + logger.Debugf("Checking if it is a JSON file: %s", src) return !strings.HasSuffix(src, ".json"), nil @@ -100,6 +99,45 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { return nil } +// skipFolder checks if the folder will need to be skipped during indexes copy. +func skipFolder(logger *log.CmdLogger, indexSrcDir, src string) (bool, error) { + path, _ := filepath.Rel(indexSrcDir, src) + + matchContainsPublicIndexFolder, _ := filepath.Match("index*", path) + matchContainsPrivateDataCollectionFolder, _ := filepath.Match("collections*", path) + matchPrivateDataCollectionFolder, _ := filepath.Match("collections/*", path) + matchPrivateDataCollectionIndexFolder, _ := filepath.Match("collections/*/indexes", path) + relativeFoldersLength := len(strings.Split(path, "/")) + + logger.Debugf("relative path: %s, total relative folders: %d", path, relativeFoldersLength) + logger.Debugf("Match pattern - index*: %t", matchContainsPublicIndexFolder) + logger.Debugf("Match pattern - collections*: %t", matchContainsPrivateDataCollectionFolder) + logger.Debugf("Match pattern - collections/*: %t", matchPrivateDataCollectionFolder) + logger.Debugf("Match pattern - collections/*/indexes: %t", matchPrivateDataCollectionIndexFolder) + + switch { + case relativeFoldersLength == 1 && (!matchContainsPublicIndexFolder && !matchContainsPrivateDataCollectionFolder): + logger.Debugf("Should skip folder") + + return true, nil + + case relativeFoldersLength == 2 && (!matchPrivateDataCollectionFolder): + logger.Debugf("Should skip folder") + + return true, nil + + case relativeFoldersLength == 3 && (!matchPrivateDataCollectionIndexFolder): + logger.Debugf("Should skip folder") + + return true, nil + + default: + logger.Debugf("Should NOT skip folder") + + return false, nil + } +} + // CopyMetadataDir copies all chaincode metadata from source to destination directories. func CopyMetadataDir(logger *log.CmdLogger, src, dest string) error { metadataSrcDir := filepath.Join(src, MetadataDir) From 65041133dade87aa4d00697cc53f88226e6dd8ff Mon Sep 17 00:00:00 2001 From: David F D Reis Date: Tue, 26 Mar 2024 21:27:37 -0300 Subject: [PATCH 5/7] PR comments about error handling and folders matching Signed-off-by: David F D Reis --- .../assetCollection/indexes/indexOwner.json | 11 +++ .../fabCarCollection/indexes/indexOwner.json | 11 +++ .../subdir/indexes/indexOwner.json | 1 + .../statedb/couchdb/indexed/indexOwner.json | 1 + .../couchdb/indexed/subdir/indexOwner.json | 1 + .../META-INF/statedb/couchdb/indexed/test.txt | 1 + internal/util/copy.go | 99 ++++++++++++------- 7 files changed, 90 insertions(+), 35 deletions(-) create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/assetCollection/indexes/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/indexes/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/subdir/indexes/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/subdir/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/test.txt diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/assetCollection/indexes/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/assetCollection/indexes/indexOwner.json new file mode 100644 index 0000000..28ab876 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/assetCollection/indexes/indexOwner.json @@ -0,0 +1,11 @@ +{ + "index": { + "fields": [ + "objectType", + "owner" + ] + }, + "ddoc": "indexOwnerDoc", + "name": "indexOwner", + "type": "json" +} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/indexes/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/indexes/indexOwner.json new file mode 100644 index 0000000..c895391 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/indexes/indexOwner.json @@ -0,0 +1,11 @@ +{ + "index": { + "fields": [ + "model", + "owner" + ] + }, + "ddoc": "indexOwnerDoc", + "name": "indexOwner", + "type": "json" +} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/subdir/indexes/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/subdir/indexes/indexOwner.json new file mode 100644 index 0000000..305f090 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/collectionsd/fabCarCollection/subdir/indexes/indexOwner.json @@ -0,0 +1 @@ +{"index":{"fields":["docType","owner"]},"ddoc":"indexOwnerDoc", "name":"indexOwner","type":"json"} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/indexOwner.json new file mode 100644 index 0000000..305f090 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/indexOwner.json @@ -0,0 +1 @@ +{"index":{"fields":["docType","owner"]},"ddoc":"indexOwnerDoc", "name":"indexOwner","type":"json"} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/subdir/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/subdir/indexOwner.json new file mode 100644 index 0000000..305f090 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/subdir/indexOwner.json @@ -0,0 +1 @@ +{"index":{"fields":["docType","owner"]},"ddoc":"indexOwnerDoc", "name":"indexOwner","type":"json"} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/test.txt b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/test.txt new file mode 100644 index 0000000..73709ba --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexed/test.txt @@ -0,0 +1 @@ +Testing diff --git a/internal/util/copy.go b/internal/util/copy.go index 1f7552b..4e97737 100644 --- a/internal/util/copy.go +++ b/internal/util/copy.go @@ -47,38 +47,23 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { logger.Debugf("Copying CouchDB index definitions from %s to %s", indexSrcDir, indexDestDir) - fileInfo, err := os.Lstat(indexSrcDir) - if err != nil { - if os.IsNotExist(err) { - // indexes are optional - return nil - } - - return err - } - - if !fileInfo.IsDir() { - return fmt.Errorf( - "CouchDB index definitions path %s is not a directory: %w", - indexSrcDir, - err, - ) - } - opt := copy.Options{ Skip: func(info os.FileInfo, src, _ string) (bool, error) { logger.Debugf("Checking source copy path: %s", src) - if err != nil { - return true, fmt.Errorf( - "failed to create CouchDB index definitions from folder %s: %w", - src, - err, - ) - } - if info.IsDir() { - return skipFolder(logger, indexSrcDir, src) + skip, err := skipFolder(logger, indexSrcDir, src) + + if err != nil { + return skip, fmt.Errorf( + "error checking if the folder is eligible to have a couchdb index: %s, %s: %w", + indexSrcDir, + src, + err, + ) + } + + return skip, nil } logger.Debugf("Checking if it is a JSON file: %s", src) @@ -101,17 +86,61 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { // skipFolder checks if the folder will need to be skipped during indexes copy. func skipFolder(logger *log.CmdLogger, indexSrcDir, src string) (bool, error) { - path, _ := filepath.Rel(indexSrcDir, src) + path, err := filepath.Rel(indexSrcDir, src) + + if err != nil { + return true, fmt.Errorf( + "error resolving the relative path: %s, %s: %w", + indexSrcDir, + src, + err, + ) + } - matchContainsPublicIndexFolder, _ := filepath.Match("index*", path) - matchContainsPrivateDataCollectionFolder, _ := filepath.Match("collections*", path) - matchPrivateDataCollectionFolder, _ := filepath.Match("collections/*", path) - matchPrivateDataCollectionIndexFolder, _ := filepath.Match("collections/*/indexes", path) - relativeFoldersLength := len(strings.Split(path, "/")) + matchContainsPublicIndexFolder, err := filepath.Match("indexes", path) + + if err != nil { + return true, fmt.Errorf( + "error matching the path with public index couchdb folder: %s: %w", + path, + err, + ) + } + + matchContainsPrivateDataCollectionFolder, err := filepath.Match("collections", path) + + if err != nil { + return true, fmt.Errorf( + "error matching the path with the collection folder: %s: %w", + path, + err, + ) + } + + matchPrivateDataCollectionFolder, err := filepath.Match("collections/*", path) + + if err != nil { + return true, fmt.Errorf( + "error matching the path with the private data collection definition folder: %s: %w", + path, + err, + ) + } + + matchPrivateDataCollectionIndexFolder, err := filepath.Match("collections/*/indexes", path) + + if err != nil { + return true, fmt.Errorf( + "error matching the path with the private data collection index definition folder: %s: %w", + path, + err, + ) + } + relativeFoldersLength := len(strings.Split(path, string(filepath.Separator))) logger.Debugf("relative path: %s, total relative folders: %d", path, relativeFoldersLength) - logger.Debugf("Match pattern - index*: %t", matchContainsPublicIndexFolder) - logger.Debugf("Match pattern - collections*: %t", matchContainsPrivateDataCollectionFolder) + logger.Debugf("Match pattern - index: %t", matchContainsPublicIndexFolder) + logger.Debugf("Match pattern - collections: %t", matchContainsPrivateDataCollectionFolder) logger.Debugf("Match pattern - collections/*: %t", matchPrivateDataCollectionFolder) logger.Debugf("Match pattern - collections/*/indexes: %t", matchPrivateDataCollectionIndexFolder) From 0266c8750d6cf60cd10d3773956866b7eaf5a2ab Mon Sep 17 00:00:00 2001 From: David F D Reis Date: Wed, 27 Mar 2024 00:43:04 -0300 Subject: [PATCH 6/7] More test cases and new skip function for files. The JSON may be in the root folder Signed-off-by: David F D Reis --- cmd/release/main_test.go | 37 +++++ .../META-INF/statedb/couchdb/indexOwner.json | 1 + .../META-INF/statedb/couchdb/test.txt | 1 + internal/builder/release.go | 2 +- internal/util/copy.go | 147 +++++++++--------- 5 files changed, 111 insertions(+), 77 deletions(-) create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexOwner.json create mode 100644 cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/test.txt diff --git a/cmd/release/main_test.go b/cmd/release/main_test.go index b76fded..7c776f4 100644 --- a/cmd/release/main_test.go +++ b/cmd/release/main_test.go @@ -79,5 +79,42 @@ var _ = Describe("Main", func() { "indexOwner.json", ) Expect(privateDataCollectionSubdirPath).NotTo(BeAnExistingFile(), "Files outside indexes directory should not be copied") + + collectionsdCollectionPath := filepath.Join( + tempDir, + "statedb", + "couchdb", + "collectionsd", + "fabCarCollection", + "indexes", + "indexOwner.json", + ) + Expect(collectionsdCollectionPath).NotTo(BeAnExistingFile(), "Files outside indexes directory should not be copied") + + indexedCollectionSubdirPath := filepath.Join( + tempDir, + "statedb", + "couchdb", + "indexed", + "indexes", + "indexOwner.json", + ) + Expect(indexedCollectionSubdirPath).NotTo(BeAnExistingFile(), "Files outside indexes directory should not be copied") + + rootIndexOwnerJSONFile := filepath.Join( + tempDir, + "statedb", + "couchdb", + "indexOwner.json", + ) + Expect(rootIndexOwnerJSONFile).NotTo(BeAnExistingFile(), "Files outside indexes directory should not be copied") + + roottestTXTFile := filepath.Join( + tempDir, + "statedb", + "couchdb", + "test.txt", + ) + Expect(roottestTXTFile).NotTo(BeAnExistingFile(), "Files outside indexes directory should not be copied") }) }) diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexOwner.json b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexOwner.json new file mode 100644 index 0000000..305f090 --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/indexOwner.json @@ -0,0 +1 @@ +{"index":{"fields":["docType","owner"]},"ddoc":"indexOwnerDoc", "name":"indexOwner","type":"json"} diff --git a/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/test.txt b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/test.txt new file mode 100644 index 0000000..73709ba --- /dev/null +++ b/cmd/release/testdata/buildwithindexes/META-INF/statedb/couchdb/test.txt @@ -0,0 +1 @@ +Testing diff --git a/internal/builder/release.go b/internal/builder/release.go index 7fea281..e4b6f60 100644 --- a/internal/builder/release.go +++ b/internal/builder/release.go @@ -19,7 +19,7 @@ func (r *Release) Run(ctx context.Context) error { logger.Debugln("Releasing chaincode...") // If CouchDB index definitions are required for the chaincode, release is - // responsible for placing the indexes into the statedb/couchdb/indexes + // responsible for placing the indexes into the statedb/couchdb/ // directory under RELEASE_OUTPUT_DIR. The indexes must have a .json // extension. err := util.CopyIndexFiles(logger, r.BuildOutputDirectory, r.ReleaseOutputDirectory) diff --git a/internal/util/copy.go b/internal/util/copy.go index 4e97737..0335c91 100644 --- a/internal/util/copy.go +++ b/internal/util/copy.go @@ -41,19 +41,27 @@ func CopyImageJSON(logger *log.CmdLogger, src, dest string) error { // CopyIndexFiles copies CouchDB index definitions from source to destination directories. func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { + logger.Debugf("Copying couchdb index files from %s to %s", src, dest) + + _, err := os.Lstat(src) + if err != nil { + if os.IsNotExist(err) { + // indexes are optional + return nil + } + + return err + } + indexDir := filepath.Join("statedb", "couchdb") indexSrcDir := filepath.Join(src, MetadataDir, indexDir) indexDestDir := filepath.Join(dest, indexDir) - logger.Debugf("Copying CouchDB index definitions from %s to %s", indexSrcDir, indexDestDir) - opt := copy.Options{ Skip: func(info os.FileInfo, src, _ string) (bool, error) { logger.Debugf("Checking source copy path: %s", src) - if info.IsDir() { skip, err := skipFolder(logger, indexSrcDir, src) - if err != nil { return skip, fmt.Errorf( "error checking if the folder is eligible to have a couchdb index: %s, %s: %w", @@ -66,9 +74,17 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { return skip, nil } - logger.Debugf("Checking if it is a JSON file: %s", src) + skip, err := skipFile(indexSrcDir, src) + if err != nil { + return skip, fmt.Errorf( + "error checking if the file is eligible to have a couchdb index: %s, %s: %w", + indexSrcDir, + src, + err, + ) + } - return !strings.HasSuffix(src, ".json"), nil + return skip, nil }, } @@ -84,65 +100,77 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { return nil } -// skipFolder checks if the folder will need to be skipped during indexes copy. -func skipFolder(logger *log.CmdLogger, indexSrcDir, src string) (bool, error) { - path, err := filepath.Rel(indexSrcDir, src) +// CopyMetadataDir copies all chaincode metadata from source to destination directories. +func CopyMetadataDir(logger *log.CmdLogger, src, dest string) error { + metadataSrcDir := filepath.Join(src, MetadataDir) + metadataDestDir := filepath.Join(dest, MetadataDir) + + logger.Debugf("Copying chaincode metadata from %s to %s", metadataSrcDir, metadataDestDir) + fileInfo, err := os.Lstat(metadataSrcDir) if err != nil { - return true, fmt.Errorf( - "error resolving the relative path: %s, %s: %w", - indexSrcDir, - src, - err, - ) + if os.IsNotExist(err) { + // metadata is optional + return nil + } + + return err } - matchContainsPublicIndexFolder, err := filepath.Match("indexes", path) + if !fileInfo.IsDir() { + return fmt.Errorf("chaincode metadata path %s is not a directory: %w", metadataSrcDir, err) + } - if err != nil { - return true, fmt.Errorf( - "error matching the path with public index couchdb folder: %s: %w", - path, + if err := copy.Copy(metadataSrcDir, metadataDestDir); err != nil { + return fmt.Errorf( + "failed to copy chaincode metadata from %s to %s: %w", + metadataSrcDir, + metadataDestDir, err, ) } - matchContainsPrivateDataCollectionFolder, err := filepath.Match("collections", path) + return nil +} +func skipFile(indexSrcDir, src string) (bool, error) { + path, err := filepath.Rel(indexSrcDir, src) if err != nil { return true, fmt.Errorf( - "error matching the path with the collection folder: %s: %w", - path, + "error verifying relative path from %s to %s: %w", + indexSrcDir, + src, err, ) } - matchPrivateDataCollectionFolder, err := filepath.Match("collections/*", path) - - if err != nil { - return true, fmt.Errorf( - "error matching the path with the private data collection definition folder: %s: %w", - path, - err, - ) + if len(strings.Split(path, string(filepath.Separator))) == 1 { // JSON is in root couchdb folder + return true, nil } - matchPrivateDataCollectionIndexFolder, err := filepath.Match("collections/*/indexes", path) + return !strings.HasSuffix(src, ".json"), nil +} +// skipFolder checks if the folder will need to be skipped during indexes copy. +func skipFolder(logger *log.CmdLogger, indexSrcDir, src string) (bool, error) { + path, err := filepath.Rel(indexSrcDir, src) if err != nil { - return true, fmt.Errorf( - "error matching the path with the private data collection index definition folder: %s: %w", - path, - err, - ) + logger.Debugf("failed resolve relative path: %s, src: %s", indexSrcDir, src) + + return true, fmt.Errorf("failed resolve relative path %s to %s: %w", indexSrcDir, src, err) } + + matchContainsPublicIndexFolder, _ := filepath.Match("indexes", path) + matchContainsPrivateDataCollectionFolder, _ := filepath.Match("collections", path) + matchPrivateDataCollectionFolder, _ := filepath.Match("collections/*", path) + matchPrivateDataCollectionIndexFolder, _ := filepath.Match("collections/*/indexes", path) relativeFoldersLength := len(strings.Split(path, string(filepath.Separator))) - logger.Debugf("relative path: %s, total relative folders: %d", path, relativeFoldersLength) - logger.Debugf("Match pattern - index: %t", matchContainsPublicIndexFolder) - logger.Debugf("Match pattern - collections: %t", matchContainsPrivateDataCollectionFolder) - logger.Debugf("Match pattern - collections/*: %t", matchPrivateDataCollectionFolder) - logger.Debugf("Match pattern - collections/*/indexes: %t", matchPrivateDataCollectionIndexFolder) + logger.Debugf("Calculated relative path: %s. Total relative folders: %d", path, relativeFoldersLength) + logger.Debugf("Match pattern 'index': %t", matchContainsPublicIndexFolder) + logger.Debugf("Match pattern 'collections': %t", matchContainsPrivateDataCollectionFolder) + logger.Debugf("Match pattern 'collections/*': %t", matchPrivateDataCollectionFolder) + logger.Debugf("Match pattern 'collections/*/indexes': %t", matchPrivateDataCollectionIndexFolder) switch { case relativeFoldersLength == 1 && (!matchContainsPublicIndexFolder && !matchContainsPrivateDataCollectionFolder): @@ -161,41 +189,8 @@ func skipFolder(logger *log.CmdLogger, indexSrcDir, src string) (bool, error) { return true, nil default: - logger.Debugf("Should NOT skip folder") + logger.Debugf("Should not skip folder") return false, nil } } - -// CopyMetadataDir copies all chaincode metadata from source to destination directories. -func CopyMetadataDir(logger *log.CmdLogger, src, dest string) error { - metadataSrcDir := filepath.Join(src, MetadataDir) - metadataDestDir := filepath.Join(dest, MetadataDir) - - logger.Debugf("Copying chaincode metadata from %s to %s", metadataSrcDir, metadataDestDir) - - fileInfo, err := os.Lstat(metadataSrcDir) - if err != nil { - if os.IsNotExist(err) { - // metadata is optional - return nil - } - - return err - } - - if !fileInfo.IsDir() { - return fmt.Errorf("chaincode metadata path %s is not a directory: %w", metadataSrcDir, err) - } - - if err := copy.Copy(metadataSrcDir, metadataDestDir); err != nil { - return fmt.Errorf( - "failed to copy chaincode metadata from %s to %s: %w", - metadataSrcDir, - metadataDestDir, - err, - ) - } - - return nil -} From fbe076b052de7de4411d3184932614d333719371 Mon Sep 17 00:00:00 2001 From: David F D Reis Date: Wed, 27 Mar 2024 01:24:38 -0300 Subject: [PATCH 7/7] Errors due to cached tests and more debug logging. Signed-off-by: David F D Reis --- internal/util/copy.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/util/copy.go b/internal/util/copy.go index 0335c91..6eb1715 100644 --- a/internal/util/copy.go +++ b/internal/util/copy.go @@ -74,7 +74,7 @@ func CopyIndexFiles(logger *log.CmdLogger, src, dest string) error { return skip, nil } - skip, err := skipFile(indexSrcDir, src) + skip, err := skipFile(logger, indexSrcDir, src) if err != nil { return skip, fmt.Errorf( "error checking if the file is eligible to have a couchdb index: %s, %s: %w", @@ -133,9 +133,12 @@ func CopyMetadataDir(logger *log.CmdLogger, src, dest string) error { return nil } -func skipFile(indexSrcDir, src string) (bool, error) { +// skipFile checks if the file will need to be skipped during indexes copy. +func skipFile(logger *log.CmdLogger, indexSrcDir, src string) (bool, error) { path, err := filepath.Rel(indexSrcDir, src) if err != nil { + logger.Debugf("error verifying relative path from: %s, src: %s", indexSrcDir, src) + return true, fmt.Errorf( "error verifying relative path from %s to %s: %w", indexSrcDir, @@ -145,10 +148,20 @@ func skipFile(indexSrcDir, src string) (bool, error) { } if len(strings.Split(path, string(filepath.Separator))) == 1 { // JSON is in root couchdb folder + logger.Debugf("The JSON file in the root couchdb index folder, should skip: %s, src: %s", path, src) + return true, nil } - return !strings.HasSuffix(src, ".json"), nil + if strings.HasSuffix(src, ".json") { + logger.Debugf("The JSON file is valid, should copy: %s, src: %s", path, src) + + return false, nil + } + + logger.Debugf("The JSON file is invalid, should skip: %s, src: %s", path, src) + + return true, nil } // skipFolder checks if the folder will need to be skipped during indexes copy.