diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 3298053..23403b2 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -57,25 +57,24 @@ pipeline { // } // } //} - // There is nothing to test. Step must be enabled when we add code. - //stage('Test') { - // steps { - // withGithubNotify(context: "Test") { - // deleteDir() - // unstash 'source' - // dir("${BASE_DIR}"){ - // withGoEnv(){ - // goTestJUnit(options: '-v ./...', output: 'junit-report.xml') - // } - // } - // } - // } - // post { - // always { - // junit(allowEmptyResults: true, keepLongStdio: true, testResults: '**/junit-report.xml') - // } - // } - //} + stage('Test') { + steps { + withGithubNotify(context: "Test") { + deleteDir() + unstash 'source' + dir("${BASE_DIR}"){ + withGoEnv(){ + goTestJUnit(options: '-v ./...', output: 'junit-report.xml') + } + } + } + } + post { + always { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: '**/junit-report.xml') + } + } + } } post { cleanup { diff --git a/.golangci.yml b/.golangci.yml index ab835f0..0e15e8d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -85,7 +85,7 @@ linters-settings: # minimal length of string constant, 3 by default min-len: 3 # minimal occurrences count to trigger, 3 by default - min-occurrences: 2 + min-occurrences: 5 dupl: # tokens count to trigger issue, 150 by default diff --git a/CHANGELOG-developer.md b/CHANGELOG-developer.md index feee988..3f93a86 100644 --- a/CHANGELOG-developer.md +++ b/CHANGELOG-developer.md @@ -6,6 +6,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added +- Move initial version of input package from Filebeat. #17 + ### Changed ### Deprecated diff --git a/NOTICE.txt b/NOTICE.txt index 18906fa..8df1c6b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -221,12 +221,12 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l -------------------------------------------------------------------------------- -Dependency : github.com/magefile/mage -Version: v1.13.0 +Dependency : github.com/elastic/go-concert +Version: v0.2.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.2.0/LICENSE: Apache License Version 2.0, January 2004 @@ -408,7 +408,7 @@ Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.13.0/L APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -416,7 +416,7 @@ Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.13.0/L same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2017 the Mage authors + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -432,12 +432,12 @@ Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.13.0/L -------------------------------------------------------------------------------- -Dependency : go.elastic.co/go-licence-detector -Version: v0.5.0 +Dependency : github.com/elastic/go-licenser +Version: v0.4.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@v0.5.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0.4.0/LICENSE: Apache License @@ -628,7 +628,7 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2018 Elasticsearch B.V. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -643,176 +643,13 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@ limitations under the License. - - -================================================================================ -Indirect dependencies - - --------------------------------------------------------------------------------- -Dependency : github.com/Microsoft/go-winio -Version: v0.5.2 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/!microsoft/go-winio@v0.5.2/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 Microsoft - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - - --------------------------------------------------------------------------------- -Dependency : github.com/armon/go-radix -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/armon/go-radix@v1.0.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Armon Dadgar - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/benbjohnson/clock -Version: v1.1.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/benbjohnson/clock@v1.1.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Ben Johnson - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/cyphar/filepath-securejoin -Version: v0.2.2 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/cyphar/filepath-securejoin@v0.2.2/LICENSE: - -Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. -Copyright (C) 2017 SUSE LLC. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/davecgh/go-spew -Version: v1.1.1 -Licence type (autodetected): ISC --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/davecgh/go-spew@v1.1.1/LICENSE: - -ISC License - -Copyright (c) 2012-2016 Dave Collins - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -------------------------------------------------------------------------------- -Dependency : github.com/elastic/go-licenser -Version: v0.4.0 +Dependency : github.com/elastic/go-structform +Version: v0.0.9 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0.4.0/LICENSE: - +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.9/LICENSE: Apache License Version 2.0, January 2004 @@ -994,7 +831,7 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0. APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -1002,7 +839,7 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0. same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018 Elasticsearch B.V. + Copyright 2012–2018 Elastic Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1018,12 +855,79 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-licenser@v0. -------------------------------------------------------------------------------- -Dependency : github.com/elastic/go-structform -Version: v0.0.9 +Dependency : github.com/gofrs/uuid +Version: v3.3.0+incompatible +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid@v3.3.0+incompatible/LICENSE: + +Copyright (C) 2013-2018 by Maxim Bublis + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/google/go-cmp +Version: v0.5.6 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.5.6/LICENSE: + +Copyright (c) 2017 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/magefile/mage +Version: v1.13.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.13.0/LICENSE: Apache License Version 2.0, January 2004 @@ -1213,7 +1117,7 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2012–2018 Elastic + Copyright 2017 the Mage authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1229,224 +1133,43 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v -------------------------------------------------------------------------------- -Dependency : github.com/elastic/go-sysinfo -Version: v1.7.1 -Licence type (autodetected): Apache-2.0 +Dependency : github.com/stretchr/testify +Version: v1.7.0 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.7.1/LICENSE.txt: - +Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.7.0/LICENSE: - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +MIT License - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - http://www.apache.org/licenses/LICENSE-2.0 +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/elastic/go-ucfg -Version: v0.8.5 +Dependency : github.com/urso/sderr +Version: v0.0.0-20210525210834-52b04e8f5c71 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20210525210834-52b04e8f5c71/LICENSE: Apache License Version 2.0, January 2004 @@ -1628,7 +1351,7 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.5/ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -1636,7 +1359,7 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.5/ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1652,12 +1375,12 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.5/ -------------------------------------------------------------------------------- -Dependency : github.com/elastic/go-windows -Version: v1.0.1 +Dependency : go.elastic.co/go-licence-detector +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@v0.5.0/LICENSE: Apache License @@ -1864,46 +1587,119 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0 -------------------------------------------------------------------------------- -Dependency : github.com/fatih/color -Version: v1.13.0 +Dependency : go.uber.org/atomic +Version: v1.9.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/fatih/color@v1.13.0/LICENSE.md: - -The MIT License (MIT) +Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.9.0/LICENSE.txt: -Copyright (c) 2013 Fatih Arslan +Copyright (c) 2016 Uber Technologies, Inc. -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/gobuffalo/here -Version: v0.6.0 +Dependency : golang.org/x/sys +Version: v0.0.0-20220209214540-3681064d5158 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20220209214540-3681064d5158/LICENSE: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + +================================================================================ +Indirect dependencies + + +-------------------------------------------------------------------------------- +Dependency : github.com/BurntSushi/toml +Version: v0.3.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/gobuffalo/here@v0.6.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/toml@v0.3.1/COPYING: The MIT License (MIT) -Copyright (c) 2019 Mark Bates +Copyright (c) 2013 TOML authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/Microsoft/go-winio +Version: v0.5.2 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!microsoft/go-winio@v0.5.2/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 Microsoft Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -1924,15 +1720,78 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + -------------------------------------------------------------------------------- -Dependency : github.com/google/go-cmp -Version: v0.2.0 +Dependency : github.com/armon/go-radix +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/armon/go-radix@v1.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Armon Dadgar + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/benbjohnson/clock +Version: v1.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/benbjohnson/clock@v1.1.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Ben Johnson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/cyphar/filepath-securejoin +Version: v0.2.3 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.2.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/cyphar/filepath-securejoin@v0.2.3/LICENSE: -Copyright (c) 2017 The Go Authors. All rights reserved. +Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. +Copyright (C) 2017 SUSE LLC. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1962,12 +1821,37 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/google/licenseclassifier -Version: v0.0.0-20200402202327-879cb1424de0 +Dependency : github.com/davecgh/go-spew +Version: v1.1.1 +Licence type (autodetected): ISC +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/davecgh/go-spew@v1.1.1/LICENSE: + +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/go-sysinfo +Version: v1.7.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/google/licenseclassifier@v0.0.0-20200402202327-879cb1424de0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.7.1/LICENSE.txt: Apache License @@ -2174,47 +2058,1319 @@ Contents of probable licence file $GOMODCACHE/github.com/google/licenseclassifie -------------------------------------------------------------------------------- -Dependency : github.com/hashicorp/errwrap -Version: v1.0.0 -Licence type (autodetected): MPL-2.0 +Dependency : github.com/elastic/go-ucfg +Version: v0.8.5 +Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/hashicorp/errwrap@v1.0.0/LICENSE: - -Mozilla Public License, version 2.0 +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.5/LICENSE: -1. Definitions + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -1.1. “Contributor” + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. + 1. Definitions. -1.2. “Contributor Version” + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -1.3. “Contribution” + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. - means Covered Software of a particular Contributor. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. -1.4. “Covered Software” + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. -1.5. “Incompatible With Secondary Licenses” - means + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/go-windows +Version: v1.0.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0.1/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/fatih/color +Version: v1.13.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/fatih/color@v1.13.0/LICENSE.md: + +The MIT License (MIT) + +Copyright (c) 2013 Fatih Arslan + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/gobuffalo/here +Version: v0.6.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gobuffalo/here@v0.6.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2019 Mark Bates + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/google/licenseclassifier +Version: v0.0.0-20200402202327-879cb1424de0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/google/licenseclassifier@v0.0.0-20200402202327-879cb1424de0/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/google/renameio +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/google/renameio@v0.1.0/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/hashicorp/errwrap +Version: v1.0.0 +Licence type (autodetected): MPL-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/hashicorp/errwrap@v1.0.0/LICENSE: + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + + +-------------------------------------------------------------------------------- +Dependency : github.com/hashicorp/go-multierror +Version: v1.1.1 +Licence type (autodetected): MPL-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/hashicorp/go-multierror@v1.1.1/LICENSE: + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. 1.6. “Executable Form” @@ -2536,400 +3692,628 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice the Mozilla Public License, v. 2.0. +-------------------------------------------------------------------------------- +Dependency : github.com/inconshreveable/mousetrap +Version: v1.0.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/inconshreveable/mousetrap@v1.0.0/LICENSE: + +Copyright 2014 Alan Shreve + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + -------------------------------------------------------------------------------- -Dependency : github.com/hashicorp/go-multierror -Version: v1.1.1 -Licence type (autodetected): MPL-2.0 +Dependency : github.com/jcchavezs/porto +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/hashicorp/go-multierror@v1.1.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/jcchavezs/porto@v0.1.0/LICENSE: -Mozilla Public License, version 2.0 + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -1. Definitions + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -1.1. “Contributor” + 1. Definitions. - means each individual or legal entity that creates, contributes to the - creation of, or owns Covered Software. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -1.2. “Contributor Version” + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. - means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -1.3. “Contribution” + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. - means Covered Software of a particular Contributor. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -1.4. “Covered Software” + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. - means Source Code Form to which the initial Contributor has attached the - notice in Exhibit A, the Executable Form of such Source Code Form, and - Modifications of such Source Code Form, in each case including portions - thereof. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -1.5. “Incompatible With Secondary Licenses” - means + http://www.apache.org/licenses/LICENSE-2.0 - a. that the initial Contributor has attached the notice described in - Exhibit B to the Covered Software; or + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. -1.6. “Executable Form” +-------------------------------------------------------------------------------- +Dependency : github.com/joeshaw/multierror +Version: v0.0.0-20140124173710-69b34d4ec901 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - means any form of the work other than Source Code Form. +Contents of probable licence file $GOMODCACHE/github.com/joeshaw/multierror@v0.0.0-20140124173710-69b34d4ec901/LICENSE: -1.7. “Larger Work” +The MIT License (MIT) - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. +Copyright (c) 2014 Joe Shaw -1.8. “License” +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - means this document. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. -1.9. “Licensable” +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. -1.10. “Modifications” +-------------------------------------------------------------------------------- +Dependency : github.com/karrick/godirwalk +Version: v1.15.6 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- - means any of the following: +Contents of probable licence file $GOMODCACHE/github.com/karrick/godirwalk@v1.15.6/LICENSE: - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or +BSD 2-Clause License - b. any new file in Source Code Form that contains any Covered Software. +Copyright (c) 2017, Karrick McDermott +All rights reserved. -1.11. “Patent Claims” of a Contributor +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. -1.12. “Secondary License” +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. - means either the GNU General Public License, Version 2.0, the GNU Lesser - General Public License, Version 2.1, the GNU Affero General Public - License, Version 3.0, or any later versions of those licenses. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -1.13. “Source Code Form” - means the form of the work preferred for making modifications. +-------------------------------------------------------------------------------- +Dependency : github.com/kisielk/gotool +Version: v1.0.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- -1.14. “You” (or “Your”) +Contents of probable licence file $GOMODCACHE/github.com/kisielk/gotool@v1.0.0/LEGAL: - means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is - controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause - the direction or management of such entity, whether by contract or - otherwise, or (b) ownership of more than fifty percent (50%) of the - outstanding shares or beneficial ownership of such entity. +All the files in this distribution are covered under either the MIT +license (see the file LICENSE) except some files mentioned below. +match.go, match_test.go: -2. License Grants and Conditions + Copyright (c) 2009 The Go Authors. All rights reserved. -2.1. Grants + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: - Each Contributor hereby grants You a world-wide, royalty-free, - non-exclusive license: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. - a. under intellectual property rights (other than patent or trademark) - Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. -2.2. Effective Date +-------------------------------------------------------------------------------- +Dependency : github.com/kr/pretty +Version: v0.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. +Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.1.0/License: -2.3. Limitations on Grant Scope +The MIT License (MIT) - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: +Copyright 2012 Keith Rarick - a. for any code that a Contributor has removed from Covered Software; or +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - b. for infringements caused by: (i) Your and any other third party’s - modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor - Version); or +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). -2.4. Subsequent Licenses +-------------------------------------------------------------------------------- +Dependency : github.com/kr/pty +Version: v1.1.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). +Contents of probable licence file $GOMODCACHE/github.com/kr/pty@v1.1.1/License: -2.5. Representation +Copyright (c) 2011 Keith Rarick - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, +subject to the following conditions: -2.6. Fair Use +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the +Software. - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -2.7. Conditions - Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in - Section 2.1. +-------------------------------------------------------------------------------- +Dependency : github.com/kr/text +Version: v0.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- +Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.1.0/License: -3. Responsibilities +Copyright 2012 Keith Rarick -3.1. Distribution of Source Form +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. -3.2. Distribution of Executable Form +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - If You distribute Covered Software in Executable Form then: - a. such Covered Software must also be made available in Source Code Form, - as described in Section 3.1, and You must inform recipients of the - Executable Form how they can obtain a copy of such Source Code Form by - reasonable means in a timely manner, at a charge no more than the cost - of distribution to the recipient; and +-------------------------------------------------------------------------------- +Dependency : github.com/markbates/pkger +Version: v0.17.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. +Contents of probable licence file $GOMODCACHE/github.com/markbates/pkger@v0.17.0/LICENSE: -3.3. Distribution of a Larger Work +The MIT License (MIT) - You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). +Copyright (c) 2019 Mark Bates -3.4. Notices +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -3.5. Application of Additional Terms +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. - You may choose to offer, and to charge a fee for, warranty, support, - indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any - liability incurred by such Contributor as a result of warranty, support, - indemnity or liability terms You offer. You may include additional - disclaimers of warranty and limitations of liability specific to any - jurisdiction. -4. Inability to Comply Due to Statute or Regulation +-------------------------------------------------------------------------------- +Dependency : github.com/mattn/go-colorable +Version: v0.1.12 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. +Contents of probable licence file $GOMODCACHE/github.com/mattn/go-colorable@v0.1.12/LICENSE: -5. Termination +The MIT License (MIT) -5.1. The rights granted under this License will terminate automatically if You - fail to comply with any of its terms. However, if You become compliant, - then the rights granted under this License from a particular Contributor - are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. +Copyright (c) 2016 Yasuhiro Matsumoto -5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user - license agreements (excluding distributors and resellers) which have been - validly granted by You or Your distributors under this License prior to - termination shall survive termination. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -6. Disclaimer of Warranty +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. -7. Limitation of Liability +-------------------------------------------------------------------------------- +Dependency : github.com/mattn/go-isatty +Version: v0.0.14 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - Under no circumstances and under no legal theory, whether tort (including - negligence), contract, or otherwise, shall any Contributor, or anyone who - distributes Covered Software as permitted above, be liable to You for any - direct, indirect, special, incidental, or consequential damages of any - character including, without limitation, damages for lost profits, loss of - goodwill, work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses, even if such party shall have been - informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. +Contents of probable licence file $GOMODCACHE/github.com/mattn/go-isatty@v0.0.14/LICENSE: -8. Litigation +Copyright (c) Yasuhiro MATSUMOTO - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. +MIT License (Expat) -9. Miscellaneous +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -10. Versions of the License -10.1. New Versions +-------------------------------------------------------------------------------- +Dependency : github.com/niemeyer/pretty +Version: v0.0.0-20200227124842-a10e7caefd8e +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. +Contents of probable licence file $GOMODCACHE/github.com/niemeyer/pretty@v0.0.0-20200227124842-a10e7caefd8e/License: -10.2. Effect of New Versions +Copyright 2012 Keith Rarick - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -10.3. Modified Versions +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. -Exhibit A - Source Code Form License Notice +-------------------------------------------------------------------------------- +Dependency : github.com/pkg/errors +Version: v0.9.1 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. +Contents of probable licence file $GOMODCACHE/github.com/pkg/errors@v0.9.1/LICENSE: -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. +Copyright (c) 2015, Dave Cheney +All rights reserved. -You may add additional accurate notices of copyright ownership. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -Exhibit B - “Incompatible With Secondary Licenses” Notice +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/inconshreveable/mousetrap +Dependency : github.com/pmezard/go-difflib Version: v1.0.0 -Licence type (autodetected): Apache-2.0 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/inconshreveable/mousetrap@v1.0.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/pmezard/go-difflib@v1.0.0/LICENSE: -Copyright 2014 Alan Shreve +Copyright (c) 2013, Patrick Mezard +All rights reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - http://www.apache.org/licenses/LICENSE-2.0 + Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + The names of its contributors may not be used to endorse or promote +products derived from this software without specific prior written +permission. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/jcchavezs/porto -Version: v0.1.0 +Dependency : github.com/prometheus/procfs +Version: v0.7.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/jcchavezs/porto@v0.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.7.3/LICENSE: Apache License Version 2.0, January 2004 @@ -3125,363 +4509,166 @@ Contents of probable licence file $GOMODCACHE/github.com/jcchavezs/porto@v0.1.0/ you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/joeshaw/multierror -Version: v0.0.0-20140124173710-69b34d4ec901 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/joeshaw/multierror@v0.0.0-20140124173710-69b34d4ec901/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Joe Shaw - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/karrick/godirwalk -Version: v1.15.6 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/karrick/godirwalk@v1.15.6/LICENSE: - -BSD 2-Clause License - -Copyright (c) 2017, Karrick McDermott -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/kr/pretty -Version: v0.1.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.1.0/License: - -The MIT License (MIT) - -Copyright 2012 Keith Rarick - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/kr/pty -Version: v1.1.1 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/kr/pty@v1.1.1/License: - -Copyright (c) 2011 Keith Rarick - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, -sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall -be included in all copies or substantial portions of the -Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/kr/text -Version: v0.1.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.1.0/License: - -Copyright 2012 Keith Rarick - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/markbates/pkger -Version: v0.17.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/markbates/pkger@v0.17.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2019 Mark Bates - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/mattn/go-colorable -Version: v0.1.12 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mattn/go-colorable@v0.1.12/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2016 Yasuhiro Matsumoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/mattn/go-isatty -Version: v0.0.14 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mattn/go-isatty@v0.0.14/LICENSE: - -Copyright (c) Yasuhiro MATSUMOTO - -MIT License (Expat) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + http://www.apache.org/licenses/LICENSE-2.0 -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -------------------------------------------------------------------------------- -Dependency : github.com/niemeyer/pretty -Version: v0.0.0-20200227124842-a10e7caefd8e -Licence type (autodetected): MIT +Dependency : github.com/rcrowley/go-metrics +Version: v0.0.0-20201227073835-cf1acfcdf475 +Licence type (autodetected): BSD-2-Clause-FreeBSD -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/niemeyer/pretty@v0.0.0-20200227124842-a10e7caefd8e/License: +Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/LICENSE: -Copyright 2012 Keith Rarick +Copyright 2012 Richard Crowley. All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation +are those of the authors and should not be interpreted as representing +official policies, either expressed or implied, of Richard Crowley. -------------------------------------------------------------------------------- -Dependency : github.com/pkg/errors -Version: v0.9.1 -Licence type (autodetected): BSD-2-Clause +Dependency : github.com/rogpeppe/go-internal +Version: v1.3.0 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/pkg/errors@v0.9.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.3.0/LICENSE: -Copyright (c) 2015, Dave Cheney -All rights reserved. +Copyright (c) 2018 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. +modification, are permitted provided that the following conditions are +met: -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/pmezard/go-difflib -Version: v1.0.0 +Dependency : github.com/santhosh-tekuri/jsonschema +Version: v1.2.4 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/pmezard/go-difflib@v1.0.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/santhosh-tekuri/jsonschema@v1.2.4/LICENSE: -Copyright (c) 2013, Patrick Mezard -All rights reserved. +Copyright (c) 2017 Santhosh Kumar Tekuri. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - The names of its contributors may not be used to endorse or promote -products derived from this software without specific prior written -permission. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- +Dependency : github.com/sergi/go-diff +Version: v1.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/sergi/go-diff@v1.1.0/LICENSE: + +Copyright (c) 2012-2016 The go-diff Authors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/prometheus/procfs -Version: v0.7.3 +Dependency : github.com/spf13/cobra +Version: v1.3.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.7.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/spf13/cobra@v1.3.0/LICENSE.txt: - Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -3650,88 +4837,23 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.7. and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/rcrowley/go-metrics -Version: v0.0.0-20201227073835-cf1acfcdf475 -Licence type (autodetected): BSD-2-Clause-FreeBSD --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/LICENSE: - -Copyright 2012 Richard Crowley. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation -are those of the authors and should not be interpreted as representing -official policies, either expressed or implied, of Richard Crowley. + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. -------------------------------------------------------------------------------- -Dependency : github.com/santhosh-tekuri/jsonschema -Version: v1.2.4 +Dependency : github.com/spf13/pflag +Version: v1.0.5 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/santhosh-tekuri/jsonschema@v1.2.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/spf13/pflag@v1.0.5/LICENSE: -Copyright (c) 2017 Santhosh Kumar Tekuri. All rights reserved. +Copyright (c) 2012 Alex Ogier. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -3759,45 +4881,48 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + -------------------------------------------------------------------------------- -Dependency : github.com/sergi/go-diff -Version: v1.1.0 +Dependency : github.com/stretchr/objx +Version: v0.2.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/sergi/go-diff@v1.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.2.0/LICENSE: -Copyright (c) 2012-2016 The go-diff Authors. All rights reserved. +The MIT License -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: +Copyright (c) 2014 Stretchr, Inc. +Copyright (c) 2017-2018 objx contributors -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/spf13/cobra -Version: v1.3.0 +Dependency : github.com/urso/diag +Version: v0.0.0-20200210123136-21b3cc8eb797 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/spf13/cobra@v1.3.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/urso/diag@v0.0.0-20200210123136-21b3cc8eb797/LICENSE: - Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -3972,115 +5097,41 @@ Contents of probable licence file $GOMODCACHE/github.com/spf13/cobra@v1.3.0/LICE incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + END OF TERMS AND CONDITIONS --------------------------------------------------------------------------------- -Dependency : github.com/spf13/pflag -Version: v1.0.5 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/spf13/pflag@v1.0.5/LICENSE: - -Copyright (c) 2012 Alex Ogier. All rights reserved. -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/stretchr/objx -Version: v0.1.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.1.0/LICENSE: - -The MIT License - -Copyright (c) 2014 Stretchr, Inc. -Copyright (c) 2017-2018 objx contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/stretchr/testify -Version: v1.7.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.7.0/LICENSE: + APPENDIX: How to apply the Apache License to your work. -MIT License + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. -Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. + Copyright [yyyy] [name of copyright owner] -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + http://www.apache.org/licenses/LICENSE-2.0 -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -------------------------------------------------------------------------------- Dependency : github.com/yuin/goldmark -Version: v1.3.5 +Version: v1.4.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/yuin/goldmark@v1.3.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/yuin/goldmark@v1.4.0/LICENSE: MIT License @@ -4772,14 +5823,16 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- -Dependency : go.uber.org/atomic -Version: v1.7.0 +Dependency : go.uber.org/goleak +Version: v1.1.11 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/atomic@v1.7.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.1.11/LICENSE: -Copyright (c) 2016 Uber Technologies, Inc. +The MIT License (MIT) + +Copyright (c) 2018 Uber Technologies, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -4801,16 +5854,14 @@ THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : go.uber.org/goleak -Version: v1.1.11 +Dependency : go.uber.org/multierr +Version: v1.8.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.1.11/LICENSE: - -The MIT License (MIT) +Contents of probable licence file $GOMODCACHE/go.uber.org/multierr@v1.8.0/LICENSE.txt: -Copyright (c) 2018 Uber Technologies, Inc. +Copyright (c) 2017-2021 Uber Technologies, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -4832,12 +5883,12 @@ THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : go.uber.org/multierr -Version: v1.6.0 +Dependency : go.uber.org/tools +Version: v0.0.0-20190618225709-2cfd321de3ee Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/multierr@v1.6.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/tools@v0.0.0-20190618225709-2cfd321de3ee/LICENSE: Copyright (c) 2017 Uber Technologies, Inc. @@ -4928,11 +5979,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/lint -Version: v0.0.0-20190930215403-16217165b5de +Version: v0.0.0-20210508222113-6edffad5e616 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/lint@v0.0.0-20190930215403-16217165b5de/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/lint@v0.0.0-20210508222113-6edffad5e616/LICENSE: Copyright (c) 2013 The Go Authors. All rights reserved. @@ -5074,43 +6125,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : golang.org/x/sys -Version: v0.0.0-20220209214540-3681064d5158 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20220209214540-3681064d5158/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : golang.org/x/term Version: v0.0.0-20201126162022-7de9c90e9dd1 @@ -5294,6 +6308,42 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/errgo.v2 +Version: v2.1.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/errgo.v2@v2.1.0/LICENSE: + +Copyright © 2013, Roger Peppe +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of this project nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : gopkg.in/hjson/hjson-go.v3 Version: v3.0.1 @@ -5596,6 +6646,36 @@ See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Dependency : honnef.co/go/tools +Version: v0.0.1-2019.2.3 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/honnef.co/go/tools@v0.0.1-2019.2.3/LICENSE: + +Copyright (c) 2016 Dominik Honnef + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : howett.net/plist Version: v1.0.0 diff --git a/dev-tools/templates/.golangci.yml b/dev-tools/templates/.golangci.yml index 6ef975c..d959838 100644 --- a/dev-tools/templates/.golangci.yml +++ b/dev-tools/templates/.golangci.yml @@ -87,7 +87,7 @@ linters-settings: # minimal length of string constant, 3 by default min-len: 3 # minimal occurrences count to trigger, 3 by default - min-occurrences: 2 + min-occurrences: 5 dupl: # tokens count to trigger issue, 150 by default diff --git a/go.mod b/go.mod index 7b2606d..6c40f05 100644 --- a/go.mod +++ b/go.mod @@ -4,23 +4,39 @@ go 1.17 require ( github.com/elastic/elastic-agent-libs v0.2.4 + github.com/elastic/go-concert v0.2.0 + github.com/elastic/go-licenser v0.4.0 + github.com/elastic/go-structform v0.0.9 + github.com/gofrs/uuid v3.3.0+incompatible + github.com/google/go-cmp v0.5.6 github.com/magefile/mage v1.13.0 + github.com/stretchr/testify v1.7.0 + github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 go.elastic.co/go-licence-detector v0.5.0 + go.uber.org/atomic v1.9.0 + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 ) require ( - github.com/cyphar/filepath-securejoin v0.2.2 // indirect + github.com/cyphar/filepath-securejoin v0.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/elastic/go-ucfg v0.8.5 // indirect github.com/gobuffalo/here v0.6.0 // indirect github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/karrick/godirwalk v1.15.6 // indirect github.com/markbates/pkger v0.17.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect + github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect + go.elastic.co/ecszap v1.0.0 // indirect + go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 780b848..91a677f 100644 --- a/go.sum +++ b/go.sum @@ -49,7 +49,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -61,7 +60,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -93,23 +91,24 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elastic/elastic-agent-libs v0.2.4 h1:TOy+vild5MSkn/eTOwrnffAeAntq4GiLpkvWe+uNVms= github.com/elastic/elastic-agent-libs v0.2.4/go.mod h1:eUiaofWIVdxVOAR4ICGxn2wbFByhrnmR6/kppwYq3qI= +github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa20xg= +github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38= github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRsDqUI= github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx3HLD+o= github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= -github.com/elastic/go-sysinfo v1.7.1 h1:Wx4DSARcKLllpKT2TnFVdSUJOsybqMYCNQZq1/wO+s0= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-ucfg v0.8.5 h1:4GB/rMpuh7qTcSFaxJUk97a/JyvFzhi6t+kaskTTLdM= github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= -github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -124,7 +123,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -139,6 +137,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI= github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -175,7 +175,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -187,6 +186,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0 h1:OggOMmdI0JLwg1FkOKH9S7fVHF0oEm8PX6S8kAdpOps= @@ -254,10 +254,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jcchavezs/porto v0.1.0 h1:Xmxxn25zQMmgE7/yHYmh19KcItG81hIwfbEEFnd6w/Q= github.com/jcchavezs/porto v0.1.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -273,10 +271,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -292,14 +288,12 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -344,16 +338,13 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -372,9 +363,10 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -385,21 +377,21 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 h1:OHNw/6pXODJAB32NujjdQO/KIYQ3KAbHQfCzH81XdCs= +github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797/go.mod h1:pNWFTeQ+V1OYT/TzWpnWb6eQBdoXpdx+H+lrH97/Oyo= +github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 h1:CehQeKbysHV8J2V7AD0w8NL2x1h04kmmo/Ft5su4lU0= +github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71/go.mod h1:Wp40HwmjM59FkDIVFfcCb9LzBbnc0XAMp8++hJuWvSU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.elastic.co/apm/module/apmhttp/v2 v2.0.0 h1:GNfmK1LD4nE5fYqbLxROCpg1ucyjSFG5iwulxwAJ+3o= go.elastic.co/apm/module/apmhttp/v2 v2.0.0/go.mod h1:5KmxcNN7hkJh8sVW3Ggl/pYgnwiNenygE46bZoUb9RE= -go.elastic.co/apm/v2 v2.0.0 h1:5BeBh+oIrVbMwPrW3uO9Uxm4w7HpKy92lYl5Rfj69Kg= go.elastic.co/apm/v2 v2.0.0/go.mod h1:KGQn56LtRmkQjt2qw4+c1Jz8gv9rCBUU/m21uxrqcps= go.elastic.co/ecszap v1.0.0 h1:PdQkRUeraR3XHJ14T7JMa+ncU0XXrVrcEN/BoRa2nMI= go.elastic.co/ecszap v1.0.0/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ= -go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= go.elastic.co/go-licence-detector v0.5.0 h1:YXPCyt9faKMdJ8uMrkcI4patk8WZ0ME5oaIhYBUsRU4= go.elastic.co/go-licence-detector v0.5.0/go.mod h1:fSJQU8au4SAgDK+UQFbgUPsXKYNBDv4E/dwWevrMpXU= @@ -415,13 +407,16 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= @@ -436,7 +431,6 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -456,7 +450,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -520,7 +513,6 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -624,7 +616,6 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -634,7 +625,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -657,6 +647,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -670,6 +661,7 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= @@ -697,7 +689,6 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -854,7 +845,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/hjson/hjson-go.v3 v3.0.1 h1:sbWPfwXQa3FQOBM6DAqGxqIWgFeT9dVKbbHOXQhx+W0= gopkg.in/hjson/hjson-go.v3 v3.0.1/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= @@ -878,7 +868,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/pkg/feature/bundle.go b/pkg/feature/bundle.go new file mode 100644 index 0000000..2014dc6 --- /dev/null +++ b/pkg/feature/bundle.go @@ -0,0 +1,79 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feature + +// FilterFunc is the function use to filter elements from a bundle. +type FilterFunc = func(Featurable) bool + +// Bundleable merges featurable and bundle interface together. +type bundleable interface { + Features() []Featurable +} + +// Bundle defines a list of features available in the current beat. +type Bundle struct { + features []Featurable +} + +// NewBundle creates a new Bundle of feature to be registered. +func NewBundle(features ...Featurable) *Bundle { + return &Bundle{features: features} +} + +// FilterWith takes a predicate and return a list of filtered bundle matching the predicate. +func (b *Bundle) FilterWith(pred FilterFunc) *Bundle { + var filtered []Featurable + + for _, feature := range b.features { + if pred(feature) { + filtered = append(filtered, feature) + } + } + return NewBundle(filtered...) +} + +// Filter creates a new bundle with only the feature matching the requested stability. +func (b *Bundle) Filter(stabilities ...Stability) *Bundle { + return b.FilterWith(HasStabilityPred(stabilities...)) +} + +// Features returns the interface features slice so +func (b *Bundle) Features() []Featurable { + return b.features +} + +// MustBundle takes existing bundle or features and create a new Bundle with all the merged Features. +func MustBundle(bundle ...bundleable) *Bundle { + var merged []Featurable + for _, feature := range bundle { + merged = append(merged, feature.Features()...) + } + return NewBundle(merged...) +} + +// HasStabilityPred returns true if the feature match any of the provided stabilities. +func HasStabilityPred(stabilities ...Stability) FilterFunc { + return func(f Featurable) bool { + for _, s := range stabilities { + if s == f.Description().Stability { + return true + } + } + return false + } +} diff --git a/pkg/feature/bundle_test.go b/pkg/feature/bundle_test.go new file mode 100644 index 0000000..4855552 --- /dev/null +++ b/pkg/feature/bundle_test.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feature + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBundle(t *testing.T) { + factory := func() {} + features := []Featurable{ + New("libbeat.outputs", "elasticsearch", factory, Details{Stability: Stable}), + New("libbeat.outputs", "edge", factory, Details{Stability: Experimental}), + New("libbeat.input", "tcp", factory, Details{Stability: Beta}), + } + + t.Run("Creates a new Bundle", func(t *testing.T) { + b := NewBundle(features...) + assert.Equal(t, 3, len(b.Features())) + }) + + t.Run("Filters feature based on Stability", func(t *testing.T) { + b := NewBundle(features...) + new := b.Filter(Experimental) + assert.Equal(t, 1, len(new.Features())) + }) + + t.Run("Filters feature based on multiple different Stability", func(t *testing.T) { + b := NewBundle(features...) + new := b.Filter(Experimental, Stable) + assert.Equal(t, 2, len(new.Features())) + }) + + t.Run("Creates a new Bundle from specified feature", func(t *testing.T) { + f1 := New("libbeat.outputs", "elasticsearch", factory, Details{Stability: Stable}) + b := MustBundle(f1) + assert.Equal(t, 1, len(b.Features())) + }) + + t.Run("Creates a new Bundle with grouped features", func(t *testing.T) { + f1 := New("libbeat.outputs", "elasticsearch", factory, Details{Stability: Stable}) + f2 := New("libbeat.outputs", "edge", factory, Details{Stability: Experimental}) + f3 := New("libbeat.input", "tcp", factory, Details{Stability: Beta}) + f4 := New("libbeat.input", "udp", factory, Details{Stability: Beta}) + + b := MustBundle( + MustBundle(f1), + MustBundle(f2), + MustBundle(f3, f4), + ) + + assert.Equal(t, 4, len(b.Features())) + }) +} diff --git a/pkg/feature/details.go b/pkg/feature/details.go new file mode 100644 index 0000000..cb9d76c --- /dev/null +++ b/pkg/feature/details.go @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feature + +import "fmt" + +// Details minimal information that you must provide when creating a feature. +type Details struct { + Name string + Stability Stability + Deprecated bool + Info string // short info string + Doc string // long doc string +} + +func (d Details) String() string { + fmtStr := "name: %s, description: %s (%s)" + if d.Deprecated { + fmtStr = "name: %s, description: %s (deprecated, %s)" + } + return fmt.Sprintf(fmtStr, d.Name, d.Info, d.Stability) +} + +// MakeDetails return the minimal information a new feature must provide. +func MakeDetails(fullName string, doc string, stability Stability) Details { + return Details{Name: fullName, Info: doc, Stability: stability} +} diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go new file mode 100644 index 0000000..d3fd89d --- /dev/null +++ b/pkg/feature/feature.go @@ -0,0 +1,155 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feature + +import ( + "fmt" +) + +// Registry is the global plugin registry, this variable is meant to be temporary to move all the +// internal factory to receive a context that include the current beat registry. +var registry = NewRegistry() + +// Featurable implements the description of a feature. +type Featurable interface { + bundleable + + // Namespace is the kind of plugin or functionality we want to expose as a feature. + // Examples: Autodiscover's provider, processors, outputs. + Namespace() string + + // Name is the name of the feature, the name must unique by namespace and be a description of the + // actual functionality, it is usually the name of the package. + // Examples: dissect, elasticsearch, redis + Name() string + + // Factory returns the function used to create an instance of the Feature, the signature + // of the method is type checked by the 'FindFactory' of each namespace. + Factory() interface{} + + // Description return the available information for a specific feature. + Description() Details + + String() string +} + +// Feature contains the information for a specific feature +type Feature struct { + namespace string + name string + factory interface{} + description Details +} + +// Namespace return the namespace of the feature. +func (f *Feature) Namespace() string { + return f.namespace +} + +// Name returns the name of the feature. +func (f *Feature) Name() string { + return f.name +} + +// Factory returns the factory for the feature. +func (f *Feature) Factory() interface{} { + return f.factory +} + +// Description return the available information for a specific feature. +func (f *Feature) Description() Details { + return f.description +} + +// Features return the current feature as a slice to be compatible with Bundle merging and filtering. +func (f *Feature) Features() []Featurable { + return []Featurable{f} +} + +// String return the debug information +func (f *Feature) String() string { + return fmt.Sprintf("%s/%s (description: %s)", f.namespace, f.name, f.description) +} + +// New returns a new Feature. +func New(namespace, name string, factory interface{}, description Details) *Feature { + return &Feature{ + namespace: namespace, + name: name, + factory: factory, + description: description, + } +} + +// GlobalRegistry return the configured global registry. +func GlobalRegistry() *Registry { + return registry +} + +// RegisterBundle registers a bundle of features. +func RegisterBundle(bundle *Bundle) error { + for _, f := range bundle.Features() { + err := GlobalRegistry().Register(f) + if err != nil { + return err + } + } + return nil +} + +// MustRegisterBundle register a new bundle and panic on error. +func MustRegisterBundle(bundle *Bundle) { + err := RegisterBundle(bundle) + if err != nil { + panic(err) + } +} + +// OverwriteBundle register a bundle of feature and replace any existing feature with a new +// implementation. +func OverwriteBundle(bundle *Bundle) error { + for _, f := range bundle.Features() { + err := GlobalRegistry().Register(f) + if err != nil { + return err + } + } + return nil +} + +// MustOverwriteBundle register a bundle of feature, replace any existing feature with a new +// implementation and panic on error. +func MustOverwriteBundle(bundle *Bundle) { + err := OverwriteBundle(bundle) + if err != nil { + panic(err) + } +} + +// Register register a new feature on the global registry. +func Register(feature Featurable) error { + return GlobalRegistry().Register(feature) +} + +// MustRegister register a new Feature on the global registry and panic on error. +func MustRegister(feature Featurable) { + err := Register(feature) + if err != nil { + panic(err) + } +} diff --git a/pkg/feature/registry.go b/pkg/feature/registry.go new file mode 100644 index 0000000..2320ce7 --- /dev/null +++ b/pkg/feature/registry.go @@ -0,0 +1,202 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feature + +import ( + "fmt" + "reflect" + "strings" + "sync" + + "github.com/elastic/elastic-agent-libs/logp" +) + +type mapper map[string]map[string]Featurable + +// Registry implements a global FeatureRegistry for any kind of feature in beats. +// feature are grouped by namespace, a namespace is a kind of plugin like outputs, inputs, or queue. +// The feature name must be unique. +type Registry struct { + sync.RWMutex + namespaces mapper + log *logp.Logger +} + +// NewRegistry returns a new registry. +func NewRegistry() *Registry { + return &Registry{ + namespaces: make(mapper), + log: logp.NewLogger("registry"), + } +} + +// Register registers a new feature into a specific namespace, namespace are lazy created. +// Feature name must be unique. +func (r *Registry) Register(feature Featurable) error { + r.Lock() + defer r.Unlock() + + ns := normalize(feature.Namespace()) + name := normalize(feature.Name()) + + if feature.Factory() == nil { + return fmt.Errorf("feature '%s' cannot be registered with a nil factory", name) + } + + // Lazy create namespaces + _, found := r.namespaces[ns] + if !found { + r.namespaces[ns] = make(map[string]Featurable) + } + + f, found := r.namespaces[ns][name] + if found { + if featuresEqual(feature, f) { + // Allow both old style and new style of plugin to work together. + r.log.Debugw( + "ignoring, feature '%s' is already registered in the namespace '%s'", + name, + ns, + ) + return nil + } + + return fmt.Errorf( + "could not register new feature '%s' in namespace '%s', feature name must be unique", + name, + ns, + ) + } + + r.log.Debugw( + "registering new feature", + "namespace", + ns, + "name", + name, + ) + + r.namespaces[ns][name] = feature + + return nil +} + +// Unregister removes a feature from the registry. +func (r *Registry) Unregister(namespace, name string) error { + r.Lock() + defer r.Unlock() + ns := normalize(namespace) + + v, found := r.namespaces[ns] + if !found { + return fmt.Errorf("unknown namespace named '%s'", ns) + } + + _, found = v[name] + if !found { + return fmt.Errorf("unknown feature '%s' in namespace '%s'", name, ns) + } + + delete(r.namespaces[ns], name) + return nil +} + +// Lookup searches for a Feature by the namespace-name pair. +func (r *Registry) Lookup(namespace, name string) (Featurable, error) { + r.RLock() + defer r.RUnlock() + + ns := normalize(namespace) + n := normalize(name) + + v, found := r.namespaces[ns] + if !found { + return nil, fmt.Errorf("unknown namespace named '%s'", ns) + } + + m, found := v[n] + if !found { + return nil, fmt.Errorf("unknown feature '%s' in namespace '%s'", n, ns) + } + + return m, nil +} + +// LookupAll returns all the features for a specific namespace. +func (r *Registry) LookupAll(namespace string) ([]Featurable, error) { + r.RLock() + defer r.RUnlock() + + ns := normalize(namespace) + + v, found := r.namespaces[ns] + if !found { + return nil, fmt.Errorf("unknown namespace named '%s'", ns) + } + + list := make([]Featurable, len(v)) + c := 0 + for _, feature := range v { + list[c] = feature + c++ + } + + return list, nil +} + +// Overwrite allow to replace an existing feature with a new implementation. +func (r *Registry) Overwrite(feature Featurable) error { + _, err := r.Lookup(feature.Namespace(), feature.Name()) + if err == nil { + err := r.Unregister(feature.Namespace(), feature.Name()) + if err != nil { + return err + } + } + + return r.Register(feature) +} + +// Size returns the number of registered features in the registry. +func (r *Registry) Size() int { + r.RLock() + defer r.RUnlock() + + c := 0 + for _, namespace := range r.namespaces { + c += len(namespace) + } + + return c +} + +func featuresEqual(f1, f2 Featurable) bool { + // There is no safe way to compare function in go, + // but since the function pointers are global it should be stable. + if f1.Name() == f2.Name() && + f1.Namespace() == f2.Namespace() && + reflect.ValueOf(f1.Factory()).Pointer() == reflect.ValueOf(f2.Factory()).Pointer() { + return true + } + + return false +} + +func normalize(s string) string { + return strings.ToLower(s) +} diff --git a/pkg/feature/registry_test.go b/pkg/feature/registry_test.go new file mode 100644 index 0000000..4b51300 --- /dev/null +++ b/pkg/feature/registry_test.go @@ -0,0 +1,223 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feature + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var defaultDetails = Details{Stability: Stable} + +func TestRegister(t *testing.T) { + f := func() {} + + t.Run("when the factory is nil", func(t *testing.T) { + r := NewRegistry() + err := r.Register(New("outputs", "null", nil, defaultDetails)) + if !assert.Error(t, err) { + return + } + }) + + t.Run("namespace and feature doesn't exist", func(t *testing.T) { + r := NewRegistry() + err := r.Register(New("outputs", "null", f, defaultDetails)) + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, 1, r.Size()) + }) + + t.Run("namespace exists and feature doesn't exist", func(t *testing.T) { + r := NewRegistry() + mustRegister(r, New("processor", "bar", f, defaultDetails)) + err := r.Register(New("processor", "foo", f, defaultDetails)) + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, 2, r.Size()) + }) + + t.Run("namespace exists and feature exists and not the same factory", func(t *testing.T) { + r := NewRegistry() + mustRegister(r, New("processor", "foo", func() {}, defaultDetails)) + err := r.Register(New("processor", "foo", f, defaultDetails)) + if !assert.Error(t, err) { + return + } + assert.Equal(t, 1, r.Size()) + }) + + t.Run("when the exact feature is already registered", func(t *testing.T) { + feature := New("processor", "foo", f, defaultDetails) + r := NewRegistry() + mustRegister(r, feature) + err := r.Register(feature) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, 1, r.Size()) + }) +} + +func mustRegister(r *Registry, f Featurable) { + err := r.Register(f) + if err != nil { + panic(err) + } +} + +func TestFeature(t *testing.T) { + f := func() {} + + r := NewRegistry() + mustRegister(r, New("processor", "foo", f, defaultDetails)) + mustRegister(r, New("HOLA", "fOO", f, defaultDetails)) + + t.Run("when namespace and feature are present", func(t *testing.T) { + feature, err := r.Lookup("processor", "foo") + if !assert.NotNil(t, feature.Factory()) { + return + } + assert.NoError(t, err) + }) + + t.Run("when namespace doesn't exist", func(t *testing.T) { + _, err := r.Lookup("hello", "foo") + if !assert.Error(t, err) { + return + } + }) + + t.Run("when namespace and key are normalized", func(t *testing.T) { + _, err := r.Lookup("HOLA", "foo") + if !assert.NoError(t, err) { + return + } + }) +} + +func TestLookup(t *testing.T) { + f := func() {} + + r := NewRegistry() + mustRegister(r, New("processor", "foo", f, defaultDetails)) + mustRegister(r, New("processor", "foo2", f, defaultDetails)) + mustRegister(r, New("HELLO", "fOO", f, defaultDetails)) + + t.Run("when namespace and feature are present", func(t *testing.T) { + features, err := r.LookupAll("processor") + if !assert.NoError(t, err) { + return + } + assert.Equal(t, 2, len(features)) + }) + + t.Run("when namespace is not present", func(t *testing.T) { + _, err := r.LookupAll("foobar") + if !assert.Error(t, err) { + return + } + }) + + t.Run("when namespace and name are normalized", func(t *testing.T) { + features, err := r.LookupAll("hello") + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, 1, len(features)) + }) +} + +func TestUnregister(t *testing.T) { + f := func() {} + + t.Run("when the namespace and the feature exists", func(t *testing.T) { + r := NewRegistry() + err := r.Register(New("processor", "foo", f, defaultDetails)) + assert.NoError(t, err) + assert.Equal(t, 1, r.Size()) + err = r.Unregister("processor", "foo") + if !assert.NoError(t, err) { + return + } + assert.Equal(t, 0, r.Size()) + }) + + t.Run("when the namespace exist and the feature doesn't", func(t *testing.T) { + r := NewRegistry() + err := r.Register(New("processor", "foo", f, defaultDetails)) + assert.NoError(t, err) + assert.Equal(t, 1, r.Size()) + err = r.Unregister("processor", "bar") + if assert.Error(t, err) { + return + } + assert.Equal(t, 0, r.Size()) + }) + + t.Run("when the namespace doesn't exists", func(t *testing.T) { + r := NewRegistry() + err := r.Register(New("processor", "foo", f, defaultDetails)) + assert.NoError(t, err) + assert.Equal(t, 1, r.Size()) + err = r.Unregister("outputs", "bar") + if assert.Error(t, err) { + return + } + assert.Equal(t, 0, r.Size()) + }) +} + +func TestOverwrite(t *testing.T) { + t.Run("when the feature doesn't exist", func(t *testing.T) { + f := func() {} + r := NewRegistry() + assert.Equal(t, 0, r.Size()) + err := r.Overwrite(New("processor", "foo", f, defaultDetails)) + assert.NoError(t, err) + assert.Equal(t, 1, r.Size()) + }) + + t.Run("overwrite when the feature exists", func(t *testing.T) { + f := func() {} + r := NewRegistry() + err := r.Register(New("processor", "foo", f, defaultDetails)) + assert.NoError(t, err) + assert.Equal(t, 1, r.Size()) + + check := 42 + err = r.Overwrite(New("processor", "foo", check, defaultDetails)) + assert.NoError(t, err) + assert.Equal(t, 1, r.Size()) + + feature, err := r.Lookup("processor", "foo") + if !assert.NoError(t, err) { + return + } + + v, ok := feature.Factory().(int) + assert.True(t, ok) + assert.Equal(t, 42, v) + }) +} diff --git a/pkg/feature/stability.go b/pkg/feature/stability.go new file mode 100644 index 0000000..47a3832 --- /dev/null +++ b/pkg/feature/stability.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package feature + +//go:generate stringer -type=Stability + +// Stability defines the stability of the feature, this value can be used to filter a bundler. +type Stability int + +// List all the available stability for a feature. +const ( + Undefined Stability = iota + Stable + Beta + Experimental +) diff --git a/pkg/feature/stability_string.go b/pkg/feature/stability_string.go new file mode 100644 index 0000000..459700a --- /dev/null +++ b/pkg/feature/stability_string.go @@ -0,0 +1,33 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by "stringer -type=Stability"; DO NOT EDIT. + +package feature + +import "strconv" + +const _Stability_name = "UndefinedStableBetaExperimental" + +var _Stability_index = [...]uint8{0, 9, 15, 19, 31} + +func (i Stability) String() string { + if i < 0 || i >= Stability(len(_Stability_index)-1) { + return "Stability(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Stability_name[_Stability_index[i]:_Stability_index[i+1]] +} diff --git a/pkg/manager/input/error.go b/pkg/manager/input/error.go new file mode 100644 index 0000000..168386a --- /dev/null +++ b/pkg/manager/input/error.go @@ -0,0 +1,96 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + "errors" + "fmt" + "strings" +) + +// LoadError is returned by Loaders in case of failures. +type LoadError struct { + // Name of input/module that failed to load (if applicable) + Name string + + // Reason why the loader failed. Can either be the cause reported by the + // Plugin or some other indicator like ErrUnknown + Reason error + + // (optional) Message to report in additon. + Message string +} + +// SetupError indicates that the loader initialization has detected +// errors in individual plugin configurations or duplicates. +type SetupError struct { + Fails []error +} + +// ErrUnknownInput indicates that the plugin type does not exist. Either +// because the 'type' setting name does not match the loaders expectations, +// or because the type is unknown. +var ErrUnknownInput = errors.New("unknown input type") + +// ErrNoInputConfigured indicates that the 'type' setting is missing. +var ErrNoInputConfigured = errors.New("no input type configured") + +// ErrPluginWithoutName reports that the operation failed because +// the plugin is required to have a Name. +var ErrPluginWithoutName = errors.New("the plugin has no name") + +// IsUnknownInputError checks if an error value indicates an input load +// error because there is no existing plugin that can create the input. +func IsUnknownInputError(err error) bool { return errors.Is(err, ErrUnknownInput) } + +// Unwrap returns the reason if present +func (e *LoadError) Unwrap() error { return e.Reason } + +// Error returns the errors string repesentation +func (e *LoadError) Error() string { + var buf strings.Builder + + if e.Message != "" { + buf.WriteString(e.Message) + } else if e.Name != "" { + buf.WriteString("failed to load ") + buf.WriteString(e.Name) + } + + if e.Reason != nil { + if buf.Len() > 0 { + buf.WriteString(": ") + } + fmt.Fprintf(&buf, "%v", e.Reason) + } + + if buf.Len() == 0 { + return "" + } + return buf.String() +} + +// Error returns the errors string repesentation +func (e *SetupError) Error() string { + var buf strings.Builder + buf.WriteString("invalid plugin setup found:") + for _, err := range e.Fails { + fmt.Fprintf(&buf, "\n\t%v", err) + } + return buf.String() +} diff --git a/pkg/manager/input/input-cursor/clean.go b/pkg/manager/input/input-cursor/clean.go new file mode 100644 index 0000000..7113dec --- /dev/null +++ b/pkg/manager/input/input-cursor/clean.go @@ -0,0 +1,124 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "time" + + "github.com/elastic/go-concert/timed" + "github.com/elastic/go-concert/unison" + + "github.com/elastic/elastic-agent-libs/logp" +) + +// cleaner removes finished entries from the registry file. +type cleaner struct { + log *logp.Logger +} + +// run starts a loop that tries to clean entries from the registry. +// The cleaner locks the store, such that no new states can be created +// during the cleanup phase. Only resources that are finished and whos TTL +// (clean_timeout setting) has expired will be removed. +// +// Resources are considered "Finished" if they do not have a current owner (active input), and +// if they have no pending updates that still need to be written to the registry file after associated +// events have been ACKed by the outputs. +// The event acquisition timestamp is used as reference to clean resources. If a resources was blocked +// for a long time, and the life time has been exhausted, then the resource will be removed immediately +// once the last event has been ACKed. +func (c *cleaner) run(canceler unison.Canceler, store *store, interval time.Duration) { + started := time.Now() + _ = timed.Periodic(canceler, interval, func() error { + gcStore(c.log, started, store) + return nil + }) +} + +// gcStore looks for resources to remove and deletes these. `gcStore` receives +// the start timestamp of the cleaner as reference. If we have entries without +// updates in the registry, that are older than `started`, we will use `started +// + ttl` to decide if an entry will be removed. This way old entries are not +// removed immediately on startup if the Beat is down for a longer period of +// time. +func gcStore(log *logp.Logger, started time.Time, store *store) { + log.Debugf("Start store cleanup") + defer log.Debugf("Done store cleanup") + + states := store.ephemeralStore + states.mu.Lock() + defer states.mu.Unlock() + + keys := gcFind(states.table, started, time.Now()) + if len(keys) == 0 { + log.Debug("No entries to remove were found") + return + } + + if err := gcClean(store, keys); err != nil { + log.Errorf("Failed to remove all entries from the registry: %+v", err) + } +} + +// gcFind searches the store of resources that can be removed. A set of keys to delete is returned. +func gcFind(table map[string]*resource, started, now time.Time) map[string]struct{} { + keys := map[string]struct{}{} + for key, resource := range table { + clean := checkCleanResource(started, now, resource) + if !clean { + // do not clean the resource if it is still live or not serialized to the persistent store yet. + continue + } + keys[key] = struct{}{} + } + + return keys +} + +// gcClean removes key value pairs in the removeSet from the store. +// If deletion in the persistent store fails the entry is kept in memory and +// eventually cleaned up later. +func gcClean(store *store, removeSet map[string]struct{}) error { + for key := range removeSet { + if err := store.persistentStore.Remove(key); err != nil { + return err + } + delete(store.ephemeralStore.table, key) + } + return nil +} + +// checkCleanResource returns true for a key-value pair is assumed to be old, +// if is not in use and there are no more pending updates that still need to be +// written to the persistent store anymore. +func checkCleanResource(started, now time.Time, resource *resource) bool { + if !resource.Finished() { + return false + } + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + ttl := resource.internalState.TTL + reference := resource.internalState.Updated + if started.After(reference) { + reference = started + } + + return reference.Add(ttl).Before(now) && resource.stored +} diff --git a/pkg/manager/input/input-cursor/clean_test.go b/pkg/manager/input/input-cursor/clean_test.go new file mode 100644 index 0000000..b8f60bd --- /dev/null +++ b/pkg/manager/input/input-cursor/clean_test.go @@ -0,0 +1,162 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestGCStore(t *testing.T) { + t.Run("empty store", func(t *testing.T) { + started := time.Now() + + backend := createSampleStore(t, nil) + store := testOpenStore(t, backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("state is still alive", func(t *testing.T) { + started := time.Now() + const ttl = 60 * time.Second + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl / 2), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + checkEqualStoreState(t, initState, backend.snapshot()) + }) + + t.Run("old state can be removed", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("old state is not removed if cleanup is not active long enough", func(t *testing.T) { + const ttl = 60 * time.Minute + started := time.Now() + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-2 * ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + checkEqualStoreState(t, initState, backend.snapshot()) + }) + + t.Run("old state but resource is accessed", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, backend) + defer store.Release() + + // access resource and check it is not gc'ed + res := store.Get("test::key") + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, initState, backend.snapshot()) + + // release resource and check it gets gc'ed + res.Release() + want := map[string]state{} + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("old state but resource has pending updates", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, backend) + defer store.Release() + + // create pending update operation + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + res.Release() + + // cleanup fails + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, initState, backend.snapshot()) + + // cancel operation (no more pending operations) and try to gc again + op.done(1) + gcStore(logp.NewLogger("test"), started, store) + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) +} diff --git a/pkg/manager/input/input-cursor/cursor.go b/pkg/manager/input/input-cursor/cursor.go new file mode 100644 index 0000000..2d30d38 --- /dev/null +++ b/pkg/manager/input/input-cursor/cursor.go @@ -0,0 +1,43 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +// Cursor allows the input to check if cursor status has been stored +// in the past and unpack the status into a custom structure. +type Cursor struct { + store *store + resource *resource +} + +func makeCursor(store *store, res *resource) Cursor { + return Cursor{store: store, resource: res} +} + +// IsNew returns true if no cursor information has been stored +// for the current Source. +func (c Cursor) IsNew() bool { return c.resource.IsNew() } + +// Unpack deserialized the cursor state into to. Unpack fails if no pointer is +// given, or if the structure to points to is not compatible with the document +// stored. +func (c Cursor) Unpack(to interface{}) error { + if c.IsNew() { + return nil + } + return c.resource.UnpackCursor(to) +} diff --git a/pkg/manager/input/input-cursor/cursor_test.go b/pkg/manager/input/input-cursor/cursor_test.go new file mode 100644 index 0000000..238bb3a --- /dev/null +++ b/pkg/manager/input/input-cursor/cursor_test.go @@ -0,0 +1,124 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCursor_IsNew(t *testing.T) { + t.Run("true if key is not in store", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.True(t, cursor.IsNew()) + }) + + t.Run("true if key is in store but without cursor value", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key": {Cursor: nil}, + })) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.True(t, cursor.IsNew()) + }) + + t.Run("false if key with cursor value is in persistent store", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.False(t, cursor.IsNew()) + }) + + t.Run("false if key with cursor value is in memory store only", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key": {Cursor: nil}, + })) + defer store.Release() + + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + cursor := makeCursor(store, res) + require.False(t, cursor.IsNew()) + }) +} + +func TestCursor_Unpack(t *testing.T) { + t.Run("nothing to unpack if key is new", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "", st) + }) + + t.Run("unpack fails if types are not compatible", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + var st struct{ A uint } + cursor := makeCursor(store, store.Get("test::key")) + require.Error(t, cursor.Unpack(&st)) + }) + + t.Run("unpack from state in persistent store", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "test", st) + }) + + t.Run("unpack from in memory state if updates are pending", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "test-state-update", st) + }) +} diff --git a/pkg/manager/input/input-cursor/doc.go b/pkg/manager/input/input-cursor/doc.go new file mode 100644 index 0000000..1c6f399 --- /dev/null +++ b/pkg/manager/input/input-cursor/doc.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package cursor provides an InputManager for use with the v2 API, that is +// capable of storing an internal cursor state between restarts. +// +// The InputManager requires authors to Implement a configuration function and +// the cursor.Input interface. The configuration function returns a slice of +// sources ([]Source) that it has read from the configuration object, and the +// actual Input that will be used to collect events from each configured +// source. +// When Run a go-routine will be started per configured source. If two inputs have +// configured the same source, only one will be active, while the other waits +// for the resource to become free. +// The manager keeps track of the state per source. When publishing an event a +// new cursor value can be passed as well. Future instance of the input can +// read the last published cursor state. +// +// For each source an in-memory and a persitent state are tracked. Internal +// meta updates by the input manager can not be read by Inputs, and will be +// written to the persistent store immediately. Cursor state updates are read +// and update by the input. Cursor updates are written to the persistent store +// only after the events have been ACKed by the output. Internally the input +// manager keeps track of already ACKed updates and pending ACKs. +// In order to guarantee progress even if the pbulishing is slow or blocked, all cursor +// updates are written to the in-memory state immediately. Source without any +// pending updates are in-sync (in-memory state == persistet state). All +// updates are ordered, but we allow the in-memory state to be ahead of the +// persistent state. +// When an input is started, the cursor state is read from the in-memory state. +// This way a new input instance can continue where other inputs have been +// stopped, even if we still have in-flight events from older input instances. +// The coordination between inputs guarantees that all updates are always in +// order. +// +// When a shutdown signal is received, the publisher is directly disconnected +// from the outputs. As all coordination is directly handled by the +// InputManager, shutdown will be immediate (once the input itself has +// returned), and can not be blocked by the outputs. +// +// An input that is about to collect a source that is already collected by +// another input will wait until the other input has returned or the current +// input did receive a shutdown signal. +package cursor diff --git a/pkg/manager/input/input-cursor/input.go b/pkg/manager/input/input-cursor/input.go new file mode 100644 index 0000000..0d81626 --- /dev/null +++ b/pkg/manager/input/input-cursor/input.go @@ -0,0 +1,200 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "fmt" + "runtime/debug" + "time" + + "github.com/urso/sderr" + + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/unison" + + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + "github.com/elastic/elastic-agent-inputs/pkg/publisher/acker" +) + +// Input interface for cursor based inputs. This interface must be implemented +// by inputs that with to use the InputManager in order to implement a stateful +// input that can store state between restarts. +type Input interface { + Name() string + + // Test checks the configuaration and runs additional checks if the Input can + // actually collect data for the given configuration (e.g. check if host/port or files are + // accessible). + // The input manager will call Test per configured source. + Test(Source, input.TestContext) error + + // Run starts the data collection. Run must return an error only if the + // error is fatal making it impossible for the input to recover. + // The input run a go-routine can call Run per configured Source. + Run(input.Context, Source, Cursor, Publisher) error +} + +// managedInput implements the v2.Input interface, integrating cursor Inputs +// with the v2 input API. +// The managedInput starts go-routines per configured source. +// If a Run returns the error is 'remembered', but active data collecting +// continues. Only after all Run calls have returned will the managedInput be +// done. +type managedInput struct { + manager *InputManager + userID string + sources []Source + input Input + cleanTimeout time.Duration +} + +// Name is required to implement the v2.Input interface +func (inp *managedInput) Name() string { return inp.input.Name() } + +// Test runs the Test method for each configured source. +func (inp *managedInput) Test(ctx input.TestContext) error { + var grp unison.MultiErrGroup + for _, source := range inp.sources { + source := source + grp.Go(func() (err error) { + return inp.testSource(ctx, source) + }) + } + + errs := grp.Wait() + if len(errs) > 0 { + return sderr.WrapAll(errs, "input tests failed") + } + return nil +} + +func (inp *managedInput) testSource(ctx input.TestContext, source Source) (err error) { + defer func() { + if v := recover(); v != nil { + err = fmt.Errorf("input panic with: %+v\n%s", v, debug.Stack()) + ctx.Logger.Errorf("Input crashed with: %+v", err) + } + }() + return inp.input.Test(source, ctx) +} + +// Run creates a go-routine per source, waiting until all go-routines have +// returned, either by error, or by shutdown signal. +// If an input panics, we create an error value with stack trace to report the +// issue, but not crash the whole process. +func (inp *managedInput) Run( + ctx input.Context, + pipeline publisher.PipelineConnector, +) (err error) { + // Setup cancellation using a custom cancel context. All workers will be + // stopped if one failed badly by returning an error. + cancelCtx, cancel := context.WithCancel(ctxtool.FromCanceller(ctx.Cancelation)) + defer cancel() + ctx.Cancelation = cancelCtx + + var grp unison.MultiErrGroup + for _, source := range inp.sources { + source := source + grp.Go(func() (err error) { + // refine per worker context + inpCtx := ctx + inpCtx.ID = ctx.ID + "::" + source.Name() + inpCtx.Logger = ctx.Logger.With("input_source", source.Name()) + + if err = inp.runSource(inpCtx, inp.manager.store, source, pipeline); err != nil { + cancel() + } + return err + }) + } + + if errs := grp.Wait(); len(errs) > 0 { + return sderr.WrapAll(errs, "input %{id} failed", ctx.ID) + } + return nil +} + +func (inp *managedInput) runSource( + ctx input.Context, + store *store, + source Source, + pipeline publisher.PipelineConnector, +) (err error) { + defer func() { + if v := recover(); v != nil { + err = fmt.Errorf("input panic with: %+v\n%s", v, debug.Stack()) + ctx.Logger.Errorf("Input crashed with: %+v", err) + } + }() + + client, err := pipeline.ConnectWith(publisher.ClientConfig{ + CloseRef: ctx.Cancelation, + ACKHandler: newInputACKHandler(), + }) + if err != nil { + return err + } + defer client.Close() + + resourceKey := inp.createSourceID(source) + resource, err := inp.manager.lock(ctx, resourceKey) + if err != nil { + return err + } + defer releaseResource(resource) + + store.UpdateTTL(resource, inp.cleanTimeout) + + cursor := makeCursor(store, resource) + p := &cursorPublisher{canceler: ctx.Cancelation, client: client, cursor: &cursor} + return inp.input.Run(ctx, source, cursor, p) +} + +func (inp *managedInput) createSourceID(s Source) string { + if inp.userID != "" { + return fmt.Sprintf("%v::%v::%v", inp.manager.Type, inp.userID, s.Name()) + } + return fmt.Sprintf("%v::%v", inp.manager.Type, s.Name()) +} + +func newInputACKHandler() publisher.ACKer { + return acker.EventPrivateReporter(func(acked int, private []interface{}) { + var n uint + var last int + for i := 0; i < len(private); i++ { + current := private[i] + if current == nil { + continue + } + + if _, ok := current.(*updateOp); !ok { + continue + } + + n++ + last = i + } + + if n == 0 { + return + } + private[last].(*updateOp).Execute(n) + }) +} diff --git a/pkg/manager/input/input-cursor/manager.go b/pkg/manager/input/input-cursor/manager.go new file mode 100644 index 0000000..04d9ca6 --- /dev/null +++ b/pkg/manager/input/input-cursor/manager.go @@ -0,0 +1,210 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/urso/sderr" + + "github.com/elastic/go-concert/unison" + + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + "github.com/elastic/elastic-agent-inputs/pkg/statestore" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +// InputManager is used to create, manage, and coordinate stateful inputs and +// their persistent state. +// The InputManager ensures that only one input can be active for a unique source. +// If two inputs have overlapping sources, both can still collect data, but +// only one input will collect from the common source. +// +// The InputManager automatically cleans up old entries without an active +// input, and without any pending update operations for the persistent store. +// +// The Type field is used to create the key name in the persistent store. Users +// are allowed to add a custome per input configuration ID using the `id` +// setting, to collect the same source multiple times, but with different +// state. The key name in the persistent store becomes -[]- +type InputManager struct { + Logger *logp.Logger + + // StateStore gives the InputManager access to the persitent key value store. + StateStore StateStore + + // Type must contain the name of the input type. It is used to create the key name + // for all sources the inputs collect from. + Type string + + // DefaultCleanTimeout configures the key/value garbage collection interval. + // The InputManager will only collect keys for the configured 'Type' + DefaultCleanTimeout time.Duration + + // Configure returns an array of Sources, and a configured Input instances + // that will be used to collect events from each source. + Configure func(cfg *conf.C) ([]Source, Input, error) + + initOnce sync.Once + initErr error + store *store +} + +// Source describe a source the input can collect data from. +// The `Name` method must return an unique name, that will be used to identify +// the source in the persistent state store. +type Source interface { + Name() string +} + +var ( + errNoSourceConfigured = errors.New("no source has been configured") + errNoInputRunner = errors.New("no input runner available") +) + +// StateStore interface and configurations used to give the Manager access to the persistent store. +type StateStore interface { + Access() (*statestore.Store, error) + CleanupInterval() time.Duration +} + +func (cim *InputManager) init() error { + cim.initOnce.Do(func() { + if cim.DefaultCleanTimeout <= 0 { + cim.DefaultCleanTimeout = 30 * time.Minute + } + + log := cim.Logger.With("input_type", cim.Type) + var store *store + store, cim.initErr = openStore(log, cim.StateStore, cim.Type) + if cim.initErr != nil { + return + } + + cim.store = store + }) + + return cim.initErr +} + +// Init starts background processes for deleting old entries from the +// persistent store if mode is ModeRun. +func (cim *InputManager) Init(group unison.Group, mode input.Mode) error { + if mode != input.ModeRun { + return nil + } + + if err := cim.init(); err != nil { + return err + } + + log := cim.Logger.With("input_type", cim.Type) + + store := cim.store + cleaner := &cleaner{log: log} + store.Retain() + err := group.Go(func(canceler context.Context) error { + defer cim.shutdown() + defer store.Release() + interval := cim.StateStore.CleanupInterval() + if interval <= 0 { + interval = 5 * time.Minute + } + cleaner.run(canceler, store, interval) + return nil + }) + if err != nil { + store.Release() + cim.shutdown() + return sderr.Wrap(err, "Can not start registry cleanup process") + } + + return nil +} + +func (cim *InputManager) shutdown() { + cim.store.Release() +} + +// Create builds a new input.Input using the provided Configure function. +// The Input will run a go-routine per source that has been configured. +func (cim *InputManager) Create(config *conf.C) (input.Input, error) { + if err := cim.init(); err != nil { + return nil, err + } + + settings := struct { + ID string `config:"id"` + CleanTimeout time.Duration `config:"clean_timeout"` + }{ID: "", CleanTimeout: cim.DefaultCleanTimeout} + if err := config.Unpack(&settings); err != nil { + return nil, err + } + + sources, inp, err := cim.Configure(config) + if err != nil { + return nil, err + } + if len(sources) == 0 { + return nil, errNoSourceConfigured + } + if inp == nil { + return nil, errNoInputRunner + } + + return &managedInput{ + manager: cim, + userID: settings.ID, + sources: sources, + input: inp, + cleanTimeout: settings.CleanTimeout, + }, nil +} + +// Lock locks a key for exclusive access and returns an resource that can be used to modify +// the cursor state and unlock the key. +func (cim *InputManager) lock(ctx input.Context, key string) (*resource, error) { + resource := cim.store.Get(key) + err := lockResource(ctx.Logger, resource, ctx.Cancelation) + if err != nil { + resource.Release() + return nil, err + } + return resource, nil +} + +func lockResource(log *logp.Logger, resource *resource, canceler input.Canceler) error { + if !resource.lock.TryLock() { + log.Infof("Resource '%v' currently in use, waiting...", resource.key) + err := resource.lock.LockContext(canceler) + if err != nil { + log.Infof("Input for resource '%v' has been stopped while waiting", resource.key) + return err + } + } + return nil +} + +func releaseResource(resource *resource) { + resource.lock.Unlock() + resource.Release() +} diff --git a/pkg/manager/input/input-cursor/manager_test.go b/pkg/manager/input/input-cursor/manager_test.go new file mode 100644 index 0000000..7388ba6 --- /dev/null +++ b/pkg/manager/input/input-cursor/manager_test.go @@ -0,0 +1,613 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "errors" + "fmt" + "runtime" + "sort" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + "github.com/elastic/elastic-agent-inputs/pkg/manager/internal/resources" + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + pubtest "github.com/elastic/elastic-agent-inputs/pkg/publisher/testing" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/go-concert/unison" +) + +type fakeTestInput struct { + OnTest func(Source, input.TestContext) error + OnRun func(input.Context, Source, Cursor, Publisher) error +} + +type stringSource string + +func TestManager_Init(t *testing.T) { + // Integration style tests for the InputManager and the state garbage collector + + t.Run("stopping the taskgroup kills internal go-routines", func(t *testing.T) { + numRoutines := runtime.NumGoroutine() + + var grp unison.TaskGroup + store := createSampleStore(t, nil) + manager := &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: store, + Type: "test", + DefaultCleanTimeout: 10 * time.Millisecond, + } + + err := manager.Init(&grp, input.ModeRun) + require.NoError(t, err) + + time.Sleep(200 * time.Millisecond) + _ = grp.Stop() + + // wait for all go-routines to be gone + + for numRoutines < runtime.NumGoroutine() { + time.Sleep(1 * time.Millisecond) + } + }) + + t.Run("collect old entries after startup", func(t *testing.T) { + store := createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Millisecond, + Updated: time.Now().Add(-24 * time.Hour), + }, + }) + store.GCPeriod = 10 * time.Millisecond + + var grp unison.TaskGroup + defer func() { + _ = grp.Stop() + }() + manager := &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: store, + Type: "test", + DefaultCleanTimeout: 10 * time.Millisecond, + } + + err := manager.Init(&grp, input.ModeRun) + require.NoError(t, err) + + for len(store.snapshot()) > 0 { + time.Sleep(1 * time.Millisecond) + } + }) +} + +func TestManager_Create(t *testing.T) { + t.Run("fail if no source is configured", func(t *testing.T) { + manager := constInput(t, nil, &fakeTestInput{}) + _, err := manager.Create(conf.NewConfig()) + require.Error(t, err) + }) + + t.Run("fail if config error", func(t *testing.T) { + manager := failingManager(t, errors.New("oops")) + _, err := manager.Create(conf.NewConfig()) + require.Error(t, err) + }) + + t.Run("fail if no input runner is returned", func(t *testing.T) { + manager := constInput(t, sourceList("test"), nil) + _, err := manager.Create(conf.NewConfig()) + require.Error(t, err) + }) + + t.Run("configure ok", func(t *testing.T) { + manager := constInput(t, sourceList("test"), &fakeTestInput{}) + _, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + }) + + t.Run("configuring inputs with overlapping sources is allowed", func(t *testing.T) { + manager := simpleManagerWithConfigure(t, func(cfg *conf.C) ([]Source, Input, error) { + config := struct{ Sources []string }{} + err := cfg.Unpack(&config) + return sourceList(config.Sources...), &fakeTestInput{}, err + }) + + _, err := manager.Create(conf.MustNewConfigFrom(map[string]interface{}{ + "sources": []string{"a"}, + })) + require.NoError(t, err) + + _, err = manager.Create(conf.MustNewConfigFrom(map[string]interface{}{ + "sources": []string{"a"}, + })) + require.NoError(t, err) + }) +} + +func TestManager_InputsTest(t *testing.T) { + var mu sync.Mutex + var seen []string + + sources := sourceList("source1", "source2") + + t.Run("test is run for each source", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ input.TestContext) error { + mu.Lock() + defer mu.Unlock() + seen = append(seen, source.Name()) + return nil + }, + }) + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + err = inp.Test(input.TestContext{}) + require.NoError(t, err) + + sort.Strings(seen) + require.Equal(t, []string{"source1", "source2"}, seen) + }) + + t.Run("cancel gets distributed to all source tests", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(_ Source, ctx input.TestContext) error { + <-ctx.Cancelation.Done() + return nil + }, + }) + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.TODO()) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{Cancelation: ctx}) + }() + + cancel() + wg.Wait() + require.NoError(t, err) + }) + + t.Run("fail if test for one source fails", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + failing := Source(stringSource("source1")) + sources := []Source{failing, stringSource("source2")} + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ input.TestContext) error { + if source == failing { + t.Log("return error") + return errors.New("oops") + } + t.Log("return ok") + return nil + }, + }) + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{}) + t.Logf("Test returned: %v", err) + }() + + wg.Wait() + require.Error(t, err) + }) + + t.Run("panic is captured", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ input.TestContext) error { + panic("oops") + }, + }) + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{Logger: logp.NewLogger("test")}) + t.Logf("Test returned: %v", err) + }() + + wg.Wait() + require.Error(t, err) + }) +} + +func TestManager_InputsRun(t *testing.T) { + // Integration style tests for the InputManager and Input.Run + + t.Run("input returned with error", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, _ Cursor, _ Publisher) error { + return errors.New("oops") + }, + }) + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + err = inp.Run(input.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + require.Error(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("panic is captured", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, _ Cursor, _ Publisher) error { + panic("oops") + }, + }) + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + err = inp.Run(input.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + require.Error(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("shutdown on signal", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(ctx input.Context, _ Source, _ Cursor, _ Publisher) error { + <-ctx.Cancelation.Done() + return nil + }, + }) + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(input.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + }() + + cancel() + wg.Wait() + require.NoError(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("continue sending from last known position", func(t *testing.T) { + log := logp.NewLogger("test") + + type runConfig struct{ Max int } + + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + manager := simpleManagerWithConfigure(t, func(cfg *conf.C) ([]Source, Input, error) { + config := runConfig{} + if err := cfg.Unpack(&config); err != nil { + return nil, nil, err + } + + inp := &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, cursor Cursor, pub Publisher) error { + state := struct{ N int }{} + if !cursor.IsNew() { + if err := cursor.Unpack(&state); err != nil { + return fmt.Errorf("failed to unpack cursor: %w", err) + } + } + + for i := 0; i < config.Max; i++ { + event := publisher.Event{Fields: mapstr.M{"n": state.N}} + state.N++ + mustPublish(pub, event, state) + } + return nil + }, + } + + return sourceList("test"), inp, nil + }) + + var ids []int + pipeline := pubtest.ConstClient(&pubtest.FakeClient{ + PublishFunc: func(event publisher.Event) { + id, ok := event.Fields["n"].(int) + if !ok { + panic(fmt.Errorf("cannot convert id to int")) + } + ids = append(ids, id) + }, + }) + + // create and run first instance + inp, err := manager.Create(conf.MustNewConfigFrom(runConfig{Max: 3})) + require.NoError(t, err) + require.NoError(t, inp.Run(input.Context{ + Logger: log, + Cancelation: context.Background(), + }, pipeline)) + + // create and run second instance instance + inp, err = manager.Create(conf.MustNewConfigFrom(runConfig{Max: 3})) + require.NoError(t, err) + err = inp.Run(input.Context{ + Logger: log, + Cancelation: context.Background(), + }, pipeline) + assert.NoError(t, err) + + // verify + assert.Equal(t, []int{0, 1, 2, 3, 4, 5}, ids) + }) + + t.Run("event ACK triggers execution of update operations", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + store := createSampleStore(t, nil) + var wgSend sync.WaitGroup + wgSend.Add(1) + manager := constInput(t, sourceList("key"), &fakeTestInput{ + OnRun: func(ctx input.Context, _ Source, _ Cursor, pub Publisher) error { + defer wgSend.Done() + fields := mapstr.M{"hello": "world"} + mustPublish(pub, publisher.Event{Fields: fields}, "test-cursor-state1") + mustPublish(pub, publisher.Event{Fields: fields}, "test-cursor-state2") + mustPublish(pub, publisher.Event{Fields: fields}, "test-cursor-state3") + mustPublish(pub, publisher.Event{Fields: fields}, nil) + mustPublish(pub, publisher.Event{Fields: fields}, "test-cursor-state4") + mustPublish(pub, publisher.Event{Fields: fields}, "test-cursor-state5") + mustPublish(pub, publisher.Event{Fields: fields}, "test-cursor-state6") + return nil + }, + }) + manager.StateStore = store + + inp, err := manager.Create(conf.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // setup publishing pipeline and capture ACKer, so we can simulate progress in the Output + var acker publisher.ACKer + var wgACKer sync.WaitGroup + wgACKer.Add(1) + pipeline := &pubtest.FakeConnector{ + ConnectFunc: func(cfg publisher.ClientConfig) (publisher.Client, error) { + defer wgACKer.Done() + acker = cfg.ACKHandler + return &pubtest.FakeClient{ + PublishFunc: func(event publisher.Event) { + acker.AddEvent(event, true) + }, + }, nil + }, + } + + // start the input + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(input.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, pipeline) + }() + // wait for test setup to shutdown + defer wg.Wait() + + // wait for setup complete and events being send (pending operations in the pipeline) + wgACKer.Wait() + wgSend.Wait() + + // 1. No cursor state in store yet, all operations are still pending + require.Equal(t, nil, store.snapshot()["test::key"].Cursor) + + // ACK first 2 events and check snapshot state + acker.ACKEvents(2) + require.Equal(t, "test-cursor-state2", store.snapshot()["test::key"].Cursor) + + // ACK 1 events and check snapshot state (3 events published) + acker.ACKEvents(1) + require.Equal(t, "test-cursor-state3", store.snapshot()["test::key"].Cursor) + + // ACK event without cursor update and check snapshot state not modified + acker.ACKEvents(1) + require.Equal(t, "test-cursor-state3", store.snapshot()["test::key"].Cursor) + + // ACK rest + acker.ACKEvents(3) + require.Equal(t, "test-cursor-state6", store.snapshot()["test::key"].Cursor) + }) +} + +func mustPublish(p Publisher, e publisher.Event, cursor interface{}) { + err := p.Publish(e, cursor) + if err != nil { + panic(err) + } +} + +func TestLockResource(t *testing.T) { + t.Run("can lock unused resource", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + err := lockResource(logp.NewLogger("test"), res, context.TODO()) + require.NoError(t, err) + }) + + t.Run("fail to lock resource in use when context is cancelled", func(t *testing.T) { + log := logp.NewLogger("test") + + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + resUsed := store.Get("test::key") + err := lockResource(log, resUsed, context.TODO()) + require.NoError(t, err) + + // fail to lock resource in use + ctx, cancel := context.WithCancel(context.TODO()) + cancel() + resFail := store.Get("test::key") + err = lockResource(log, resFail, ctx) + require.Error(t, err) + resFail.Release() + + // unlock and release resource in use -> it should be marked finished now + releaseResource(resUsed) + require.True(t, resUsed.Finished()) + }) + + t.Run("succeed to lock resource after it has been released", func(t *testing.T) { + log := logp.NewLogger("test") + + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + resUsed := store.Get("test::key") + err := lockResource(log, resUsed, context.TODO()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + resOther := store.Get("test::key") + err := lockResource(log, resOther, context.TODO()) + if err == nil { + releaseResource(resOther) + } + }() + + go func() { + time.Sleep(100 * time.Millisecond) + releaseResource(resUsed) + }() + + wg.Wait() // <- block forever if waiting go-routine can not acquire lock + }) +} + +func (s stringSource) Name() string { return string(s) } + +func simpleManagerWithConfigure(t *testing.T, configure func(*conf.C) ([]Source, Input, error)) *InputManager { + return &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: createSampleStore(t, nil), + Type: "test", + Configure: configure, + } +} + +func constConfigureResult(t *testing.T, sources []Source, inp Input, err error) *InputManager { + return simpleManagerWithConfigure(t, func(cfg *conf.C) ([]Source, Input, error) { + return sources, inp, err + }) +} + +func failingManager(t *testing.T, err error) *InputManager { + return constConfigureResult(t, nil, nil, err) +} + +func constInput(t *testing.T, sources []Source, inp Input) *InputManager { + return constConfigureResult(t, sources, inp, nil) +} + +func (f *fakeTestInput) Name() string { return "test" } + +func (f *fakeTestInput) Test(source Source, ctx input.TestContext) error { + if f.OnTest != nil { + return f.OnTest(source, ctx) + } + return nil +} + +func (f *fakeTestInput) Run(ctx input.Context, source Source, cursor Cursor, pub Publisher) error { + if f.OnRun != nil { + return f.OnRun(ctx, source, cursor, pub) + } + return nil +} + +func sourceList(names ...string) []Source { + tmp := make([]Source, len(names)) + for i, name := range names { + tmp[i] = stringSource(name) + } + return tmp +} diff --git a/pkg/manager/input/input-cursor/publish.go b/pkg/manager/input/input-cursor/publish.go new file mode 100644 index 0000000..c20ac1b --- /dev/null +++ b/pkg/manager/input/input-cursor/publish.go @@ -0,0 +1,152 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "time" + + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + "github.com/elastic/elastic-agent-inputs/pkg/statestore" + "github.com/elastic/elastic-agent-libs/transform/typeconv" +) + +// Publisher is used to publish an event and update the cursor in a single call to Publish. +// Inputs are allowed to pass `nil` as cursor state. In this case the state is not updated, but the +// event will still be published as is. +type Publisher interface { + Publish(event publisher.Event, cursor interface{}) error +} + +// cursorPublisher implements the Publisher interface and used internally by the managedInput. +// When publishing an event with cursor state updates, the cursorPublisher +// updates the in memory state and create an updateOp that is used to schedule +// an update for the persistent store. The updateOp is run by the inputs ACK +// handler, persisting the pending update. +type cursorPublisher struct { + canceler input.Canceler + client publisher.Client + cursor *Cursor +} + +// updateOp keeps track of pending updates that are not written to the persistent store yet. +// Update operations are ordered. The input manager guarantees that only one +// input can create update operation for a source, such that new input +// instances can add update operations to be executed after already pending +// update operations from older inputs instances that have been shutdown. +type updateOp struct { + store *store + resource *resource + + // state updates to persist + timestamp time.Time + delta interface{} +} + +// Publish publishes an event. Publish returns false if the inputs cancellation context has been marked as done. +// If cursorUpdate is not nil, Publish updates the in memory state and create and updateOp for the pending update. +// It overwrite event.Private with the update operation, before finally sending the event. +// The ACK ordering in the publisher pipeline guarantees that update operations +// will be ACKed and executed in the correct order. +func (c *cursorPublisher) Publish(event publisher.Event, cursorUpdate interface{}) error { + if cursorUpdate == nil { + return c.forward(event) + } + + op, err := createUpdateOp(c.cursor.store, c.cursor.resource, cursorUpdate) + if err != nil { + return err + } + + event.Private = op + return c.forward(event) +} + +func (c *cursorPublisher) forward(event publisher.Event) error { + c.client.Publish(event) + if c.canceler == nil { + return nil + } + return c.canceler.Err() +} + +func createUpdateOp(store *store, resource *resource, updates interface{}) (*updateOp, error) { + ts := time.Now() + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + cursor := resource.pendingCursor + if resource.activeCursorOperations == 0 { + var tmp interface{} + _ = typeconv.Convert(&tmp, cursor) + resource.pendingCursor = tmp + cursor = tmp + } + if err := typeconv.Convert(&cursor, updates); err != nil { + return nil, err + } + resource.pendingCursor = cursor + + resource.Retain() + resource.activeCursorOperations++ + return &updateOp{ + resource: resource, + store: store, + timestamp: ts, + delta: updates, + }, nil +} + +// done releases resources held by the last N updateOps. +func (op *updateOp) done(n uint) { + op.resource.UpdatesReleaseN(n) + op.resource = nil + *op = updateOp{} +} + +// Execute updates the persistent store with the scheduled changes and releases the resource. +func (op *updateOp) Execute(n uint) { + resource := op.resource + defer op.done(n) + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + resource.activeCursorOperations -= n + if resource.activeCursorOperations == 0 { + resource.cursor = resource.pendingCursor + resource.pendingCursor = nil + } else { + _ = typeconv.Convert(&resource.cursor, op.delta) + } + + if resource.internalState.Updated.Before(op.timestamp) { + resource.internalState.Updated = op.timestamp + } + + err := op.store.persistentStore.Set(resource.key, resource.inSyncStateSnapshot()) + if err != nil { + if !statestore.IsClosed(err) { + op.store.log.Errorf("Failed to update state in the registry for '%v'", resource.key) + } + } else { + resource.internalInSync = true + resource.stored = true + } +} diff --git a/pkg/manager/input/input-cursor/publish_test.go b/pkg/manager/input/input-cursor/publish_test.go new file mode 100644 index 0000000..c685aa9 --- /dev/null +++ b/pkg/manager/input/input-cursor/publish_test.go @@ -0,0 +1,160 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + pubtest "github.com/elastic/elastic-agent-inputs/pkg/publisher/testing" +) + +func TestPublish(t *testing.T) { + t.Run("event with cursor state creates update operation", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + var actual publisher.Event + client := &pubtest.FakeClient{ + PublishFunc: func(event publisher.Event) { actual = event }, + } + p := cursorPublisher{nil, client, &cursor} + err := p.Publish(publisher.Event{}, "test") + require.NoError(t, err) + + require.NotNil(t, actual.Private) + }) + + t.Run("event without cursor creates no update operation", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + var actual publisher.Event + client := &pubtest.FakeClient{ + PublishFunc: func(event publisher.Event) { actual = event }, + } + p := cursorPublisher{nil, client, &cursor} + err := p.Publish(publisher.Event{}, nil) + require.NoError(t, err) + require.Nil(t, actual.Private) + }) + + t.Run("publish returns error if context has been cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + cancel() + + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + p := cursorPublisher{ctx, &pubtest.FakeClient{}, &cursor} + err := p.Publish(publisher.Event{}, nil) + require.Equal(t, context.Canceled, err) + }) +} + +func TestOp_Execute(t *testing.T) { + t.Run("applying final op marks the key as finished", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create op and release resource. The 'resource' must still be active + op := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state") + res.Release() + require.False(t, res.Finished()) + + // this was the last op, the resource should become inactive + op.Execute(1) + require.True(t, res.Finished()) + + // validate state: + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + want := "test-updated-cursor-state" + assert.Equal(t, want, inSyncCursor) + assert.Equal(t, want, inMemCursor) + }) + + t.Run("acking multiple ops applies the latest update and marks key as finished", func(t *testing.T) { + // when acking N events, intermediate updates are dropped in favor of the latest update operation. + // This test checks that the resource is correctly marked as finished. + + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create update operations and release resource. The 'resource' must still be active + mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-dropped") + op := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-final") + res.Release() + require.False(t, res.Finished()) + + // this was the last op, the resource should become inactive + op.Execute(2) + require.True(t, res.Finished()) + + // validate state: + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + want := "test-updated-cursor-state-final" + assert.Equal(t, want, inSyncCursor) + assert.Equal(t, want, inMemCursor) + }) + + t.Run("ACK only subset of pending ops will only update up to ACKed state", func(t *testing.T) { + // when acking N events, intermediate updates are dropped in favor of the latest update operation. + // This test checks that the resource is correctly marked as finished. + + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create update operations and release resource. The 'resource' must still be active + op1 := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-intermediate") + op2 := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-final") + res.Release() + require.False(t, res.Finished()) + + defer op2.done(1) // cleanup after test + + // this was the intermediate op, the resource should still be active + op1.Execute(1) + require.False(t, res.Finished()) + + // validate state (in memory state is always up to data to most recent update): + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + assert.Equal(t, "test-updated-cursor-state-intermediate", inSyncCursor) + assert.Equal(t, "test-updated-cursor-state-final", inMemCursor) + }) +} + +func mustCreateUpdateOp(t *testing.T, store *store, resource *resource, updates interface{}) *updateOp { + op, err := createUpdateOp(store, resource, updates) + if err != nil { + t.Fatalf("Failed to create update op: %v", err) + } + return op +} diff --git a/pkg/manager/input/input-cursor/store.go b/pkg/manager/input/input-cursor/store.go new file mode 100644 index 0000000..c3436f8 --- /dev/null +++ b/pkg/manager/input/input-cursor/store.go @@ -0,0 +1,323 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "strings" + "sync" + "time" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore" + "github.com/elastic/elastic-agent-inputs/pkg/statestore/cleanup" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/transform/typeconv" + "github.com/elastic/go-concert" + "github.com/elastic/go-concert/unison" + "go.uber.org/atomic" +) + +// store encapsulates the persistent store and the in memory state store, that +// can be ahead of the the persistent store. +// The store lifetime is managed by a reference counter. Once all owners (the +// session, and the resource cleaner) have dropped ownership, backing resources +// will be released and closed. +type store struct { + log *logp.Logger + refCount concert.RefCount + persistentStore *statestore.Store + ephemeralStore *states +} + +// states stores resource states in memory. When a cursor for an input is updated, +// it's state is updated first. The entry in the persistent store 'follows' the internal state. +// As long as a resources stored in states is not 'Finished', the in memory +// store is assumed to be ahead (in memory and persistent state are out of +// sync). +type states struct { + mu sync.Mutex + table map[string]*resource +} + +// resource holds the in memory state and keeps track of pending updates and inputs collecting +// event for the resource its key. +// A resource is assumed active for as long as at least one input has (or tries +// to) acuired the lock, and as long as there are pending updateOp instances in +// the pipeline not ACKed yet. The key can not gc'ed by the cleaner, as long as the resource is active. +// +// State chagnes and writes to the persistent store are protected using the +// stateMutex, to ensure full consistency between direct writes and updates +// after ACK. +type resource struct { + // pending counts the number of Inputs and outstanding registry updates. + // as long as pending is > 0 the resource is in used and must not be garbage collected. + pending atomic.Uint64 + + // lock guarantees only one input create updates for this entry + lock unison.Mutex + + // key of the resource as used for the registry. + key string + + // stateMutex is used to lock the resource when it is update/read from + // multiple go-routines like the ACK handler or the input publishing an + // event. + // stateMutex is used to access the fields 'stored', 'state' and 'internalInSync' + stateMutex sync.Mutex + + // stored indicates that the state is available in the registry file. It is false for new entries. + stored bool + + // internalInSync is true if all 'Internal' metadata like TTL or update timestamp are in sync. + // Normally resources are added when being created. But if operations failed we will retry inserting + // them on each update operation until we eventually succeeded + internalInSync bool + + activeCursorOperations uint + internalState stateInternal + + // cursor states. The cursor holds the state as it is currently known to the + // persistent store, while pendingCursor contains the most recent update + // (in-memory state), that still needs to be synced to the persistent store. + // The pendingCursor is nil if there are no pending updates. + // When processing update operations on ACKs, the state is applied to cursor + // first, which is finally written to the persistent store. This ensures that + // we always write the complete state of the key/value pair. + cursor interface{} + pendingCursor interface{} +} + +type ( + // state represents the full document as it is stored in the registry. + // + // The TTL and Update fields are for internal use only. + // + // The `Cursor` namespace is used to store the cursor information that are + // required to continue processing from the last known position. Cursor + // updates in the registry file are only executed after events have been + // ACKed by the outputs. Therefore the cursor MUST NOT include any + // information that are require to identify/track the source we are + // collecting from. + state struct { + TTL time.Duration + Updated time.Time + Cursor interface{} + } + + stateInternal struct { + TTL time.Duration + Updated time.Time + } +) + +// hook into store close for testing purposes +var closeStore = (*store).close + +func openStore(log *logp.Logger, statestore StateStore, prefix string) (*store, error) { + ok := false + + persistentStore, err := statestore.Access() + if err != nil { + return nil, err + } + defer cleanup.IfNot(&ok, func() { persistentStore.Close() }) + + states, err := readStates(log, persistentStore, prefix) + if err != nil { + return nil, err + } + + ok = true + return &store{ + log: log, + persistentStore: persistentStore, + ephemeralStore: states, + }, nil +} + +func (s *store) Retain() { s.refCount.Retain() } +func (s *store) Release() { + if s.refCount.Release() { + closeStore(s) + } +} + +func (s *store) close() { + if err := s.persistentStore.Close(); err != nil { + s.log.Errorf("Closing registry store did report an error: %+v", err) + } +} + +// Get returns the resource for the key. +// A new shared resource is generated if the key is not known. The generated +// resource is not synced to disk yet. +func (s *store) Get(key string) *resource { + return s.ephemeralStore.Find(key, true) +} + +// UpdateTTL updates the time-to-live of a resource. Inactive resources with expired TTL are subject to removal. +// The TTL value is part of the internal state, and will be written immediately to the persistent store. +// On update the resource its `cursor` state is used, to keep the cursor state in sync with the current known +// on disk store state. +func (s *store) UpdateTTL(resource *resource, ttl time.Duration) { + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + if resource.stored && resource.internalState.TTL == ttl { + return + } + + resource.internalState.TTL = ttl + if resource.internalState.Updated.IsZero() { + resource.internalState.Updated = time.Now() + } + + err := s.persistentStore.Set(resource.key, state{ + TTL: resource.internalState.TTL, + Updated: resource.internalState.Updated, + Cursor: resource.cursor, + }) + if err != nil { + s.log.Errorf("Failed to update resource management fields for '%v'", resource.key) + resource.internalInSync = false + } else { + resource.stored = true + resource.internalInSync = true + } +} + +// Find returns the resource for a given key. If the key is unknown and create is set to false nil will be returned. +// The resource returned by Find is marked as active. (*resource).Release must be called to mark the resource as inactive again. +func (s *states) Find(key string, create bool) *resource { + s.mu.Lock() + defer s.mu.Unlock() + + if resource := s.table[key]; resource != nil { + resource.Retain() + return resource + } + + if !create { + return nil + } + + // resource is owned by table(session) and input that uses the resource. + resource := &resource{ + stored: false, + key: key, + lock: unison.MakeMutex(), + } + s.table[key] = resource + resource.Retain() + return resource +} + +// IsNew returns true if we have no state recorded for the current resource. +func (r *resource) IsNew() bool { + r.stateMutex.Lock() + defer r.stateMutex.Unlock() + return r.pendingCursor == nil && r.cursor == nil +} + +// Retain is used to indicate that 'resource' gets an additional 'owner'. +// Owners of an resource can be active inputs or pending update operations +// not yet written to disk. +func (r *resource) Retain() { r.pending.Inc() } + +// Release reduced the owner ship counter of the resource. +func (r *resource) Release() { r.pending.Dec() } + +// UpdatesReleaseN is used to release ownership of N pending update operations. +func (r *resource) UpdatesReleaseN(n uint) { + r.pending.Sub(uint64(n)) +} + +// Finished returns true if the resource is not in use and if there are no pending updates +// that still need to be written to the registry. +func (r *resource) Finished() bool { return r.pending.Load() == 0 } + +// UnpackCursor deserializes the in memory state. +func (r *resource) UnpackCursor(to interface{}) error { + r.stateMutex.Lock() + defer r.stateMutex.Unlock() + if r.activeCursorOperations == 0 { + return typeconv.Convert(to, r.cursor) + } + return typeconv.Convert(to, r.pendingCursor) +} + +// syncStateSnapshot returns the current insync state based on already ACKed update operations. +func (r *resource) inSyncStateSnapshot() state { + return state{ + TTL: r.internalState.TTL, + Updated: r.internalState.Updated, + Cursor: r.cursor, + } +} + +// stateSnapshot returns the current in memory state, that already contains state updates +// not yet ACKed. +func (r *resource) stateSnapshot() state { + cursor := r.pendingCursor + if r.activeCursorOperations == 0 { + cursor = r.cursor + } + + return state{ + TTL: r.internalState.TTL, + Updated: r.internalState.Updated, + Cursor: cursor, + } +} + +func readStates(log *logp.Logger, store *statestore.Store, prefix string) (*states, error) { + keyPrefix := prefix + "::" + states := &states{ + table: map[string]*resource{}, + } + + err := store.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + if !strings.HasPrefix(key, keyPrefix) { + return true, nil + } + + var st state + if err := dec.Decode(&st); err != nil { + log.Errorf("Failed to read regisry state for '%v', cursor state will be ignored. Error was: %+v", + key, err) + return true, nil + } + + resource := &resource{ + key: key, + stored: true, + lock: unison.MakeMutex(), + internalInSync: true, + internalState: stateInternal{ + TTL: st.TTL, + Updated: st.Updated, + }, + cursor: st.Cursor, + } + states.table[resource.key] = resource + + return true, nil + }) + if err != nil { + return nil, err + } + return states, nil +} diff --git a/pkg/manager/input/input-cursor/store_test.go b/pkg/manager/input/input-cursor/store_test.go new file mode 100644 index 0000000..8065411 --- /dev/null +++ b/pkg/manager/input/input-cursor/store_test.go @@ -0,0 +1,336 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore" + "github.com/elastic/elastic-agent-inputs/pkg/statestore/storetest" + "github.com/elastic/elastic-agent-libs/logp" +) + +type testStateStore struct { + Store *statestore.Store + GCPeriod time.Duration +} + +func TestStore_OpenClose(t *testing.T) { + t.Run("releasing store closes", func(t *testing.T) { + var closed bool + cleanup := closeStoreWith(func(s *store) { + closed = true + s.close() + }) + defer cleanup() + + store := testOpenStore(t, nil) + store.Release() + + require.True(t, closed) + }) + + t.Run("fail if persistent store can not be accessed", func(t *testing.T) { + _, err := openStore(logp.NewLogger("test"), testStateStore{}, "test") + require.Error(t, err) + }) + + t.Run("load from empty", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + require.Equal(t, 0, len(storeMemorySnapshot(store))) + require.Equal(t, 0, len(storeInSyncSnapshot(store))) + }) + + t.Run("already available state is loaded", func(t *testing.T) { + states := map[string]state{ + "test::key0": {Cursor: "1"}, + "test::key1": {Cursor: "2"}, + } + + store := testOpenStore(t, createSampleStore(t, states)) + defer store.Release() + + checkEqualStoreState(t, states, storeMemorySnapshot(store)) + checkEqualStoreState(t, states, storeInSyncSnapshot(store)) + }) + + t.Run("ignore entries with wrong index on open", func(t *testing.T) { + states := map[string]state{ + "test::key0": {Cursor: "1"}, + "other::key": {Cursor: "2"}, + } + + store := testOpenStore(t, createSampleStore(t, states)) + defer store.Release() + + want := map[string]state{ + "test::key0": {Cursor: "1"}, + } + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) +} + +func TestStore_Get(t *testing.T) { + t.Run("find existing resource", func(t *testing.T) { + cursorState := state{Cursor: "1"} + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key0": cursorState, + })) + defer store.Release() + + res := store.Get("test::key0") + require.NotNil(t, res) + defer res.Release() + + // check in memory state matches matches original persistent state + require.Equal(t, cursorState, res.stateSnapshot()) + // check assumed in-sync state matches matches original persistent state + require.Equal(t, cursorState, res.inSyncStateSnapshot()) + }) + + t.Run("access unknown resource", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + require.NotNil(t, res) + defer res.Release() + + // new resource has empty state + require.Equal(t, state{}, res.stateSnapshot()) + }) + + t.Run("same resource is returned", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + res1 := store.Get("test::key") + require.NotNil(t, res1) + defer res1.Release() + + res2 := store.Get("test::key") + require.NotNil(t, res2) + defer res2.Release() + + assert.Equal(t, res1, res2) + }) +} + +func TestStore_UpdateTTL(t *testing.T) { + t.Run("add TTL for new entry to store", func(t *testing.T) { + // when creating a resource we set the TTL and insert a new key value pair without cursor value into the store: + store := testOpenStore(t, createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + store.UpdateTTL(res, 60*time.Second) + + want := map[string]state{ + "test::key": { + TTL: 60 * time.Second, + Updated: res.internalState.Updated, + Cursor: nil, + }, + } + + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) + + t.Run("update TTL for in-sync resource does not overwrite state", func(t *testing.T) { + store := testOpenStore(t, createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Second, + Cursor: "test", + }, + })) + defer store.Release() + + res := store.Get("test::key") + store.UpdateTTL(res, 60*time.Second) + want := map[string]state{ + "test::key": { + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test", + }, + } + + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) + + t.Run("update TTL for resource with pending updates", func(t *testing.T) { + // This test updates the resource TTL while update operations are still + // pending, but not synced to the persistent store yet. + // UpdateTTL changes the state in the persistent store immediately, and must therefore + // serialize the old in-sync state with update meta-data. + + // create store + backend := createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Second, + Cursor: "test", + }, + }) + store := testOpenStore(t, backend) + defer store.Release() + + // create pending update operation + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + // Update key/value pair TTL. This will update the internal state in the + // persistent store only, not modifying the old cursor state yet. + store.UpdateTTL(res, 60*time.Second) + + // validate + wantMemoryState := state{ + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test-state-update", + } + wantInSyncState := state{ + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test", + } + + checkEqualStoreState(t, map[string]state{"test::key": wantMemoryState}, storeMemorySnapshot(store)) + checkEqualStoreState(t, map[string]state{"test::key": wantInSyncState}, storeInSyncSnapshot(store)) + checkEqualStoreState(t, map[string]state{"test::key": wantInSyncState}, backend.snapshot()) + }) +} + +func closeStoreWith(fn func(s *store)) func() { + old := closeStore + closeStore = fn + return func() { + closeStore = old + } +} + +func testOpenStore(t *testing.T, persistentStore StateStore) *store { + if persistentStore == nil { + persistentStore = createSampleStore(t, nil) + } + + store, err := openStore(logp.NewLogger("test"), persistentStore, "test") + if err != nil { + t.Fatalf("failed to open the store") + } + return store +} + +func createSampleStore(t *testing.T, data map[string]state) testStateStore { + storeReg := statestore.NewRegistry(storetest.NewMemoryStoreBackend()) + store, err := storeReg.Get("test") + if err != nil { + t.Fatalf("Failed to access store: %v", err) + } + + for k, v := range data { + if err := store.Set(k, v); err != nil { + t.Fatalf("Error when populating the sample store: %v", err) + } + } + + return testStateStore{ + Store: store, + } +} + +func (ts testStateStore) WithGCPeriod(d time.Duration) testStateStore { ts.GCPeriod = d; return ts } +func (ts testStateStore) CleanupInterval() time.Duration { return ts.GCPeriod } +func (ts testStateStore) Access() (*statestore.Store, error) { + if ts.Store == nil { + return nil, errors.New("no store configured") + } + return ts.Store, nil +} + +// snapshot copies all key/value pairs from the persistent store into a table for inspection. +func (ts testStateStore) snapshot() map[string]state { + states := map[string]state{} + err := ts.Store.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + var st state + if err := dec.Decode(&st); err != nil { + return false, err + } + states[key] = st + return true, nil + }) + if err != nil { + panic("unexpected decode error from persistent test store") + } + return states +} + +// storeMemorySnapshot copies all key/value pairs into a table for inspection. +// The state returned reflects the in memory state, which can be ahead of the +// persistent state. +// +// Note: The state returned by storeMemorySnapshot is always ahead of the state returned by storeInSyncSnapshot. +// All key value pairs are fully in-sync, if both snapshot functions return the same state. +func storeMemorySnapshot(store *store) map[string]state { + store.ephemeralStore.mu.Lock() + defer store.ephemeralStore.mu.Unlock() + + states := map[string]state{} + for k, res := range store.ephemeralStore.table { + states[k] = res.stateSnapshot() + } + return states +} + +// storeInSyncSnapshot copies all key/value pairs into the table for inspection. +// The state returned reflects the current state that the in-memory tables assumed to be +// written to the persistent store already. + +// Note: The state returned by storeMemorySnapshot is always ahead of the state returned by storeInSyncSnapshot. +// All key value pairs are fully in-sync, if both snapshot functions return the same state. +func storeInSyncSnapshot(store *store) map[string]state { + store.ephemeralStore.mu.Lock() + defer store.ephemeralStore.mu.Unlock() + + states := map[string]state{} + for k, res := range store.ephemeralStore.table { + states[k] = res.inSyncStateSnapshot() + } + return states +} + +// checkEqualStoreState compares 2 store snapshot tables for equality. The test +// fails with Errorf if the state differ. +// +// Note: testify is too strict when comparing timestamp, better use checkEqualStoreState. +func checkEqualStoreState(t *testing.T, want, got map[string]state) { + if d := cmp.Diff(want, got); d != "" { + t.Errorf("store state mismatch (-want +got):\n%s", d) + } +} diff --git a/pkg/manager/input/input-stateless/stateless.go b/pkg/manager/input/input-stateless/stateless.go new file mode 100644 index 0000000..d7335bb --- /dev/null +++ b/pkg/manager/input/input-stateless/stateless.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stateless + +import ( + "fmt" + "runtime/debug" + + "github.com/elastic/go-concert/unison" + + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + conf "github.com/elastic/elastic-agent-libs/config" +) + +// InputManager provides an InputManager for transient inputs, that do not store +// state in the registry or require end-to-end event acknowledgement. +type InputManager struct { + Configure func(*conf.C) (Input, error) +} + +// Input is the interface transient inputs are required to implemented. +type Input interface { + Name() string + Test(input.TestContext) error + Run(ctx input.Context, publish Publisher) error +} + +// Publisher is used by the Input to emit events. +type Publisher interface { + Publish(publisher.Event) +} + +type configuredInput struct { + input Input +} + +var _ input.InputManager = InputManager{} + +// NewInputManager wraps the given configure function to create a new stateless input manager. +func NewInputManager(configure func(*conf.C) (Input, error)) InputManager { + return InputManager{Configure: configure} +} + +// Init does nothing. Init is required to fullfil the input.InputManager interface. +func (m InputManager) Init(_ unison.Group, _ input.Mode) error { return nil } + +// Create configures a transient input and ensures that the final input can be used with +// with the filebeat input architecture. +func (m InputManager) Create(cfg *conf.C) (input.Input, error) { + inp, err := m.Configure(cfg) + if err != nil { + return nil, err + } + return configuredInput{inp}, nil +} + +func (si configuredInput) Name() string { return si.input.Name() } + +func (si configuredInput) Run(ctx input.Context, pipeline publisher.PipelineConnector) (err error) { + defer func() { + if v := recover(); v != nil { + if e, ok := v.(error); ok { + err = e + } else { + err = fmt.Errorf("input panic with: %+v\n%s", v, debug.Stack()) + } + } + }() + + client, err := pipeline.ConnectWith(publisher.ClientConfig{ + PublishMode: publisher.DefaultGuarantees, + + // configure pipeline to disconnect input on stop signal. + CloseRef: ctx.Cancelation, + }) + if err != nil { + return err + } + + defer client.Close() + return si.input.Run(ctx, client) +} + +func (si configuredInput) Test(ctx input.TestContext) error { + return si.input.Test(ctx) +} diff --git a/pkg/manager/input/input-stateless/stateless_test.go b/pkg/manager/input/input-stateless/stateless_test.go new file mode 100644 index 0000000..4460664 --- /dev/null +++ b/pkg/manager/input/input-stateless/stateless_test.go @@ -0,0 +1,195 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stateless_test + +import ( + "context" + "errors" + "runtime" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + stateless "github.com/elastic/elastic-agent-inputs/pkg/manager/input/input-stateless" + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + pubtest "github.com/elastic/elastic-agent-inputs/pkg/publisher/testing" + "github.com/elastic/elastic-agent-libs/atomic" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type fakeStatelessInput struct { + OnTest func(input.TestContext) error + OnRun func(input.Context, stateless.Publisher) error +} + +func TestStateless_Run(t *testing.T) { + t.Run("events are published", func(t *testing.T) { + const numEvents = 5 + + ch := make(chan publisher.Event) + connector := pubtest.ConstClient(pubtest.ChClient(ch)) + + inp := createConfiguredInput(t, constInputManager(&fakeStatelessInput{ + OnRun: func(ctx input.Context, p stateless.Publisher) error { + defer close(ch) + for i := 0; i < numEvents; i++ { + p.Publish(publisher.Event{Fields: map[string]interface{}{"id": i}}) + } + return nil + }, + }), nil) + + var err error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(input.Context{}, connector) + }() + + var receivedEvents int + for range ch { + receivedEvents++ + } + wg.Wait() + + require.NoError(t, err) + require.Equal(t, numEvents, receivedEvents) + }) + + t.Run("capture panic and return error", func(t *testing.T) { + inp := createConfiguredInput(t, constInputManager(&fakeStatelessInput{ + OnRun: func(_ input.Context, _ stateless.Publisher) error { + panic("oops") + }, + }), nil) + + var clientCounters pubtest.ClientCounter + err := inp.Run(input.Context{}, clientCounters.BuildConnector()) + + require.Error(t, err) + require.Equal(t, 1, clientCounters.Total()) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("publisher unblocks if shutdown signal is send", func(t *testing.T) { + // the input blocks in the publisher. We loop until the shutdown signal is received + var started atomic.Bool + inp := createConfiguredInput(t, constInputManager(&fakeStatelessInput{ + OnRun: func(ctx input.Context, p stateless.Publisher) error { + for ctx.Cancelation.Err() == nil { + started.Store(true) + p.Publish(publisher.Event{ + Fields: mapstr.M{ + "hello": "world", + }, + }) + } + return ctx.Cancelation.Err() + }, + }), nil) + + // connector creates a client the blocks forever until the shutdown signal is received + var publishCalls atomic.Int + connector := pubtest.FakeConnector{ + ConnectFunc: func(config publisher.ClientConfig) (publisher.Client, error) { + return &pubtest.FakeClient{ + PublishFunc: func(event publisher.Event) { + publishCalls.Inc() + <-config.CloseRef.Done() + }, + }, nil + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + var err error + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(input.Context{Cancelation: ctx}, connector) + }() + + // signal and wait for shutdown + for !started.Load() { + runtime.Gosched() + } + cancel() + wg.Wait() + + // validate + require.Equal(t, context.Canceled, err) + require.Equal(t, 1, publishCalls.Load()) + }) + + t.Run("do not start input of pipeline connection fails", func(t *testing.T) { + errOpps := errors.New("oops") + connector := pubtest.FailingConnector(errOpps) + + var run atomic.Int + i := createConfiguredInput(t, constInputManager(&fakeStatelessInput{ + OnRun: func(_ input.Context, publisher stateless.Publisher) error { + run.Inc() + return nil + }, + }), nil) + + err := i.Run(input.Context{}, connector) + require.True(t, errors.Is(err, errOpps)) + require.Equal(t, 0, run.Load()) + }) +} + +func (f *fakeStatelessInput) Name() string { return "test" } + +func (f *fakeStatelessInput) Test(ctx input.TestContext) error { + if f.OnTest != nil { + return f.OnTest(ctx) + } + return nil +} + +func (f *fakeStatelessInput) Run(ctx input.Context, publish stateless.Publisher) error { + if f.OnRun != nil { + return f.OnRun(ctx, publish) + } + return errors.New("oops, run not implemented") +} + +//nolint:unparam // when we add more tests it will get a config +func createConfiguredInput(t *testing.T, manager stateless.InputManager, config map[string]interface{}) input.Input { + input, err := manager.Create(conf.MustNewConfigFrom(config)) + require.NoError(t, err) + return input +} + +func constInputManager(input stateless.Input) stateless.InputManager { + return stateless.NewInputManager(constInput(input)) +} + +func constInput(input stateless.Input) func(*conf.C) (stateless.Input, error) { + return func(_ *conf.C) (stateless.Input, error) { + return input, nil + } +} diff --git a/pkg/manager/input/input.go b/pkg/manager/input/input.go new file mode 100644 index 0000000..bda0484 --- /dev/null +++ b/pkg/manager/input/input.go @@ -0,0 +1,134 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + "time" + + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/gofrs/uuid" + + "github.com/elastic/go-concert/unison" +) + +// InputManager creates and maintains actions and background processes for an +// input type. +// The InputManager is used to create inputs. The InputManager can provide +// additional functionality like coordination between input of the same type, +// custom functionality for querying or caching shared information, application +// of common settings not unique to a particular input type, or require a more +// specific Input interface to be implemented by the actual input. +type InputManager interface { + // Init signals to InputManager to initialize internal resources. + // The mode tells the input manager if the Beat is actually running the inputs or + // if inputs are only configured for testing/validation purposes. + Init(grp unison.Group, mode Mode) error + + // Creates builds a new Input instance from the given configuation, or returns + // an error if the configuation is invalid. + // The input must establish any connection for data collection yet. The Beat + // will use the Test/Run methods of the input. + Create(*conf.C) (Input, error) +} + +// Mode tells the InputManager in which mode it is initialized. +type Mode uint8 + +//go:generate stringer -type Mode -trimprefix Mode +const ( + ModeRun Mode = iota + ModeTest + ModeOther +) + +// Input is a configured input object that can be used to test or start +// the actual data collection. +type Input interface { + // Name reports the input name. + // + // XXX: check if/how we can remove this method. Currently it is required for + // compatibility reasons with existing interfaces in libbeat, autodiscovery + // and filebeat. + Name() string + + // Test checks the configuaration and runs additional checks if the Input can + // actually collect data for the given configuration (e.g. check if host/port or files are + // accessible). + Test(TestContext) error + + // Run starts the data collection. Run must return an error only if the + // error is fatal making it impossible for the input to recover. + Run(Context, publisher.PipelineConnector) error +} + +// Info stores a input instance meta data. +type Info struct { + Input string // The actual beat's name + IndexPrefix string // The beat's index prefix in Elasticsearch. + Version string // The beat version. Defaults to the libbeat version when an implementation does not set a version + Name string // configured beat name + Hostname string // hostname + ID uuid.UUID // ID assigned to beat machine + EphemeralID uuid.UUID // ID assigned to beat process invocation (PID) + FirstStart time.Time // The time of the first start of the Beat. + StartTime time.Time // The time of last start of the Beat. Updated when the Beat is started or restarted. + + // Monitoring-related fields + Monitoring struct { + DefaultUsername string // The default username to be used to connect to Elasticsearch Monitoring + } +} + +// Context provides the Input Run function with common environmental +// information and services. +type Context struct { + // Logger provides a structured logger to inputs. The logger is initialized + // with labels that will identify logs for the input. + Logger *logp.Logger + + // The input ID. + ID string + + // Agent provides additional Beat info like instance ID or beat name. + Agent Info + + // Cancelation is used by Beats to signal the input to shutdown. + Cancelation Canceler +} + +// TestContext provides the Input Test function with common environmental +// information and services. +type TestContext struct { + // Logger provides a structured logger to inputs. The logger is initialized + // with labels that will identify logs for the input. + Logger *logp.Logger + + // Agent provides additional info like instance ID or beat name. + Agent Info + + // Cancelation is used by the binary to signal the input to shutdown. + Cancelation Canceler +} + +// Canceler is used to provide shutdown handling to the Context. +type Canceler interface { + Done() <-chan struct{} + Err() error +} diff --git a/pkg/manager/input/internal/inputest/inputest.go b/pkg/manager/input/internal/inputest/inputest.go new file mode 100644 index 0000000..2256fad --- /dev/null +++ b/pkg/manager/input/internal/inputest/inputest.go @@ -0,0 +1,106 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package inputest + +import ( + "errors" + + "github.com/elastic/elastic-agent-inputs/pkg/feature" + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/go-concert/unison" +) + +// MockInputManager can be used as InputManager replacement in tests that require a new Input Manager. +// The OnInit and OnConfigure functions are executed if the corresponding methods get called. +type MockInputManager struct { + OnInit func(input.Mode) error + OnConfigure InputConfigurer +} + +// InputConfigurer describes the interface for user supplied functions, that is +// used to create a new input from a configuration object. +type InputConfigurer func(*conf.C) (input.Input, error) + +// MockInput can be used as an Input instance in tests that require a new Input with definable behavior. +// The OnTest and OnRun functions are executed if the corresponding methods get called. +type MockInput struct { + Type string + OnTest func(input.TestContext) error + OnRun func(input.Context, publisher.PipelineConnector) error +} + +// Init returns nil if OnInit is not set. Otherwise the return value of OnInit is returned. +func (m *MockInputManager) Init(_ unison.Group, mode input.Mode) error { + if m.OnInit != nil { + return m.OnInit(mode) + } + return nil +} + +// Create fails with an error if OnConfigure is not set. Otherwise the return +// values of OnConfigure are returned. +func (m *MockInputManager) Create(cfg *conf.C) (input.Input, error) { + if m.OnConfigure != nil { + return m.OnConfigure(cfg) + } + return nil, errors.New("oops, OnConfigure not implemented ") +} + +// Name return the `Type` field of MockInput. It is required to satisfy the input.Input interface. +func (f *MockInput) Name() string { return f.Type } + +// Test return nil if OnTest is not set. Otherwise OnTest will be called. +func (f *MockInput) Test(ctx input.TestContext) error { + if f.OnTest != nil { + return f.OnTest(ctx) + } + return nil +} + +// Run returns nil if OnRun is not set. +func (f *MockInput) Run(ctx input.Context, pipeline publisher.PipelineConnector) error { + if f.OnRun != nil { + return f.OnRun(ctx, pipeline) + } + return nil +} + +// ConstInputManager create a MockInputManager that always returns input when +// Configure is called. Use ConstInputManager for tests that require an +// InputManager, but create only one Input instance. +func ConstInputManager(input input.Input) *MockInputManager { + return &MockInputManager{OnConfigure: ConfigureConstInput(input)} +} + +// ConfigureConstInput return an InputConfigurer that returns always input when called. +func ConfigureConstInput(i input.Input) InputConfigurer { + return func(_ *conf.C) (input.Input, error) { + return i, nil + } +} + +// SinglePlugin wraps an InputManager into a slice of input.Plugin, that can be used directly with input.NewLoader. +func SinglePlugin(name string, manager input.InputManager) []input.Plugin { + return []input.Plugin{{ + Name: name, + Stability: feature.Stable, + Manager: manager, + }} +} diff --git a/pkg/manager/input/internal/inputest/loader.go b/pkg/manager/input/internal/inputest/loader.go new file mode 100644 index 0000000..5f2e370 --- /dev/null +++ b/pkg/manager/input/internal/inputest/loader.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package inputest + +import ( + "testing" + + "github.com/elastic/elastic-agent-inputs/pkg/manager/input" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +// Loader wraps the input Loader in order to provide additional methods for reuse in tests. +type Loader struct { + t testing.TB + *input.Loader +} + +// MustNewTestLoader creates a new Loader. The test fails with fatal if the +// NewLoader constructor function returns an error. +func MustNewTestLoader(t testing.TB, plugins []input.Plugin, typeField, defaultType string) *Loader { + l, err := input.NewLoader(logp.NewLogger("test"), plugins, typeField, defaultType) + if err != nil { + t.Fatalf("Failed to create loader: %v", err) + } + return &Loader{t: t, Loader: l} +} + +// MustConfigure confiures a new input. The test fails with t.Fatal if the +// operation failed. +func (l *Loader) MustConfigure(cfg *conf.C) input.Input { + i, err := l.Configure(cfg) + if err != nil { + l.t.Fatalf("Failed to create the input: %v", err) + } + return i +} diff --git a/pkg/manager/input/loader.go b/pkg/manager/input/loader.go new file mode 100644 index 0000000..9fdf8f4 --- /dev/null +++ b/pkg/manager/input/loader.go @@ -0,0 +1,132 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + "fmt" + + "github.com/elastic/elastic-agent-inputs/pkg/feature" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-concert/unison" +) + +// Loader can be used to create Inputs from configurations. +// The loader is initialized with a list of plugins, and finds the correct plugin +// when a configuration is passed to Configure. +type Loader struct { + log *logp.Logger + registry map[string]Plugin + typeField string + defaultType string +} + +// NewLoader creates a new Loader for configuring inputs from a slice if plugins. +// NewLoader returns a SetupError if invalid plugin configurations or duplicates in the slice are detected. +// The Loader will read the plugin name from the configuration object as is +// configured by typeField. If typeField is empty, it defaults to "type". +func NewLoader(log *logp.Logger, plugins []Plugin, typeField, defaultType string) (*Loader, error) { + if typeField == "" { + typeField = "type" + } + + if errs := validatePlugins(plugins); len(errs) > 0 { + return nil, &SetupError{errs} + } + + registry := make(map[string]Plugin, len(plugins)) + for _, p := range plugins { + registry[p.Name] = p + } + + return &Loader{ + log: log, + registry: registry, + typeField: typeField, + defaultType: defaultType, + }, nil +} + +// Init runs Init on all InputManagers for all plugins known to the loader. +func (l *Loader) Init(group unison.Group, mode Mode) error { + for _, p := range l.registry { + if err := p.Manager.Init(group, mode); err != nil { + return err + } + } + return nil +} + +// Configure creates a new input from a Config object. +// The loader reads the input type name from the cfg object and tries to find a +// matching plugin. If a plugin is found, the plugin it's InputManager is used to create +// the input. +// Returns a LoadError if the input name can not be read from the config or if +// the type does not exist. Error values for Ccnfiguration errors do depend on +// the InputManager. +func (l *Loader) Configure(cfg *conf.C) (Input, error) { + name, err := cfg.String(l.typeField, -1) + if err != nil { + if l.defaultType == "" { + return nil, &LoadError{ + Reason: ErrNoInputConfigured, + Message: fmt.Sprintf("%v setting is missing", l.typeField), + } + } + name = l.defaultType + } + + p, exists := l.registry[name] + if !exists { + return nil, &LoadError{Name: name, Reason: ErrUnknownInput} + } + + log := l.log.With("input", name, "stability", p.Stability, "deprecated", p.Deprecated) + switch p.Stability { + case feature.Experimental: + log.Warnf("EXPERIMENTAL: The %v input is experimental", name) + case feature.Beta: + log.Warnf("BETA: The %v input is beta", name) + } + if p.Deprecated { + log.Warnf("DEPRECATED: The %v input is deprecated", name) + } + + return p.Manager.Create(cfg) +} + +// validatePlugins checks if there are multiple plugins with the same name in +// the registry. +func validatePlugins(plugins []Plugin) []error { + var errs []error + + counts := map[string]int{} + for _, p := range plugins { + counts[p.Name]++ + if err := p.validate(); err != nil { + errs = append(errs, err) + } + } + + for name, count := range counts { + if count > 1 { + errs = append(errs, fmt.Errorf("plugin '%v' found %v times", name, count)) + } + } + return errs +} diff --git a/pkg/manager/input/loader_test.go b/pkg/manager/input/loader_test.go new file mode 100644 index 0000000..6f8b470 --- /dev/null +++ b/pkg/manager/input/loader_test.go @@ -0,0 +1,204 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + "errors" + "testing" + + "github.com/elastic/elastic-agent-inputs/pkg/feature" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +type loaderConfig struct { + Plugins []Plugin + TypeField string + DefaultType string +} + +type inputCheck func(t *testing.T, input Input, err error) + +func TestLoader_New(t *testing.T) { + cases := map[string]struct { + setup loaderConfig + check func(*testing.T, error) + }{ + "ok": { + setup: loaderConfig{ + Plugins: []Plugin{ + {Name: "a", Stability: feature.Stable, Manager: ConfigureWith(nil)}, + {Name: "b", Stability: feature.Stable, Manager: ConfigureWith(nil)}, + {Name: "c", Stability: feature.Stable, Manager: ConfigureWith(nil)}, + }, + }, + check: expectNoError, + }, + "duplicate": { + setup: loaderConfig{ + Plugins: []Plugin{ + {Name: "a", Stability: feature.Stable, Manager: ConfigureWith(nil)}, + {Name: "a", Stability: feature.Stable, Manager: ConfigureWith(nil)}, + }, + }, + check: expectError, + }, + "fail with invalid plugin": { + setup: loaderConfig{ + Plugins: []Plugin{{Name: "", Manager: nil}}, + }, + check: expectError, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + _, err := test.setup.NewLoader() + test.check(t, err) + }) + } +} + +func TestLoader_Init(t *testing.T) { + pluginWithInit := func(name string, fn func(Mode) error) Plugin { + return Plugin{ + Name: name, + Stability: feature.Stable, + Manager: &fakeInputManager{OnInit: fn}, + } + } + + t.Run("calls all input managers", func(t *testing.T) { + count := 0 + incCountOnInit := func(_ Mode) error { count++; return nil } + + setup := loaderConfig{ + Plugins: []Plugin{ + pluginWithInit("a", incCountOnInit), + pluginWithInit("b", incCountOnInit), + }, + } + loader := setup.MustNewLoader() + err := loader.Init(nil, ModeRun) + expectNoError(t, err) + if count != 2 { + t.Errorf("expected init count 2, but got %v", count) + } + }) + + t.Run("stop init on error", func(t *testing.T) { + count := 0 + incCountOnInit := func(_ Mode) error { count++; return errors.New("oops") } + setup := loaderConfig{ + Plugins: []Plugin{ + pluginWithInit("a", incCountOnInit), + pluginWithInit("b", incCountOnInit), + }, + } + loader := setup.MustNewLoader() + err := loader.Init(nil, ModeRun) + expectError(t, err) + if count != 1 { + t.Errorf("expected init count 1, but got %v", count) + } + }) +} + +func TestLoader_Configure(t *testing.T) { + createManager := func(name string) InputManager { + return ConfigureWith(makeConfigFakeInput(fakeInput{Type: name})) + } + createPlugin := func(name string) Plugin { + return Plugin{Name: name, Stability: feature.Stable, Manager: createManager(name)} + } + plugins := []Plugin{ + createPlugin("a"), + createPlugin("b"), + createPlugin("c"), + } + defaultSetup := loaderConfig{Plugins: plugins, TypeField: "type"} + + cases := map[string]struct { + setup loaderConfig + config map[string]interface{} + check inputCheck + }{ + "success": { + setup: defaultSetup, + config: map[string]interface{}{"type": "a"}, + check: okSetup, + }, + "load default": { + setup: defaultSetup.WithDefaultType("a"), + config: map[string]interface{}{}, + check: okSetup, + }, + "type is missing": { + setup: defaultSetup, + config: map[string]interface{}{}, + check: failSetup, + }, + "unknown type": { + setup: defaultSetup, + config: map[string]interface{}{"type": "unknown"}, + check: failSetup, + }, + "input config fails": { + setup: defaultSetup.WithPlugins(Plugin{ + Name: "a", + Stability: feature.Beta, + Manager: ConfigureWith(func(_ *conf.C) (Input, error) { + return nil, errors.New("oops") + }), + }), + config: map[string]interface{}{"type": "a"}, + check: failSetup, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + loader := test.setup.MustNewLoader() + input, err := loader.Configure(conf.MustNewConfigFrom(test.config)) + test.check(t, input, err) + }) + } +} + +func (b loaderConfig) MustNewLoader() *Loader { + l, err := b.NewLoader() + if err != nil { + panic(err) + } + return l +} + +func (b loaderConfig) NewLoader() (*Loader, error) { + return NewLoader(logp.NewLogger("test"), b.Plugins, b.TypeField, b.DefaultType) +} +func (b loaderConfig) WithPlugins(p ...Plugin) loaderConfig { b.Plugins = p; return b } +func (b loaderConfig) WithTypeField(name string) loaderConfig { b.TypeField = name; return b } +func (b loaderConfig) WithDefaultType(name string) loaderConfig { b.DefaultType = name; return b } + +func failSetup(t *testing.T, _ Input, err error) { + expectError(t, err) +} + +func okSetup(t *testing.T, _ Input, err error) { + expectNoError(t, err) +} diff --git a/pkg/manager/input/mode_string.go b/pkg/manager/input/mode_string.go new file mode 100644 index 0000000..3c418cb --- /dev/null +++ b/pkg/manager/input/mode_string.go @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by "stringer -type Mode -trimprefix Mode"; DO NOT EDIT. + +package input + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[ModeRun-0] + _ = x[ModeTest-1] + _ = x[ModeOther-2] +} + +const _Mode_name = "RunTestOther" + +var _Mode_index = [...]uint8{0, 3, 7, 12} + +func (i Mode) String() string { + if i >= Mode(len(_Mode_index)-1) { + return "Mode(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Mode_name[_Mode_index[i]:_Mode_index[i+1]] +} diff --git a/pkg/manager/input/plugin.go b/pkg/manager/input/plugin.go new file mode 100644 index 0000000..61704b3 --- /dev/null +++ b/pkg/manager/input/plugin.go @@ -0,0 +1,92 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + "fmt" + + "github.com/elastic/elastic-agent-inputs/pkg/feature" +) + +// Plugin describes an input type. Input types should provide a constructor +// function that requires dependencies to be passed and fills out the Plugin structure. +// The Manager is used to finally create and manage inputs of the same type. +// The input-stateless and input-cursor packages, as well as the ConfigureWith function provide +// sample input managers. +// +// Example (stateless input): +// +// func Plugin() input.Plugin { +// return input.Plugin{ +// Name: "myservice", +// Stability: feature.Stable, +// Deprecated: false, +// Info: "collect data from myservice", +// Manager: stateless.NewInputManager(configure), +// } +// } +// +type Plugin struct { + // Name of the input type. + Name string + + // Configure the input stability. If the stability is not 'Stable' a message + // is logged when the input type is configured. + Stability feature.Stability + + // Deprecated marks the plugin as deprecated. If set a deprecation message is logged if + // an input is configured. + Deprecated bool + + // Info contains a short description of the input type. + Info string + + // Doc contains an optional longer description. + Doc string + + // Manager MUST be configured. The manager is used to create the inputs. + Manager InputManager +} + +// Details returns a generic feature description that is compatible with the +// feature package. +func (p Plugin) Details() feature.Details { + return feature.Details{ + Name: p.Name, + Stability: p.Stability, + Deprecated: p.Deprecated, + Info: p.Info, + Doc: p.Doc, + } +} + +func (p Plugin) validate() error { + if p.Name == "" { + return fmt.Errorf("input plugin without name found") + } + switch p.Stability { + case feature.Beta, feature.Experimental, feature.Stable: + break + default: + return fmt.Errorf("plugin '%v' has stability not set", p.Name) + } + if p.Manager == nil { + return fmt.Errorf("invalid plugin (%v) structure detected", p.Name) + } + return nil +} diff --git a/pkg/manager/input/plugin_test.go b/pkg/manager/input/plugin_test.go new file mode 100644 index 0000000..0595bbb --- /dev/null +++ b/pkg/manager/input/plugin_test.go @@ -0,0 +1,84 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + "testing" + + "github.com/elastic/elastic-agent-inputs/pkg/feature" +) + +func TestPlugin_Validate(t *testing.T) { + cases := map[string]struct { + valid bool + plugin Plugin + }{ + "valid": { + valid: true, + plugin: Plugin{ + Name: "test", + Stability: feature.Stable, + Deprecated: false, + Info: "test", + Doc: "doc string", + Manager: ConfigureWith(nil), + }, + }, + "missing name": { + valid: false, + plugin: Plugin{ + Stability: feature.Stable, + Deprecated: false, + Info: "test", + Doc: "doc string", + Manager: ConfigureWith(nil), + }, + }, + "invalid stability": { + valid: false, + plugin: Plugin{ + Name: "test", + Deprecated: false, + Info: "test", + Doc: "doc string", + Manager: ConfigureWith(nil), + }, + }, + "missing manager": { + valid: false, + plugin: Plugin{ + Name: "test", + Stability: feature.Stable, + Deprecated: false, + Info: "test", + Doc: "doc string", + }, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + err := test.plugin.validate() + if test.valid { + expectNoError(t, err) + } else { + expectError(t, err) + } + }) + } +} diff --git a/pkg/manager/input/simplemanager.go b/pkg/manager/input/simplemanager.go new file mode 100644 index 0000000..5fa2484 --- /dev/null +++ b/pkg/manager/input/simplemanager.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/go-concert/unison" +) + +type simpleInputManager struct { + configure func(*conf.C) (Input, error) +} + +// ConfigureWith creates an InputManager that provides no extra logic and +// allows each input to fully control event collection and publishing in +// isolation. The function fn will be called for every input to be configured. +func ConfigureWith(fn func(*conf.C) (Input, error)) InputManager { + return &simpleInputManager{configure: fn} +} + +// Init is required to fulfil the input.InputManager interface. +// For the kafka input no special initialization is required. +func (*simpleInputManager) Init(grp unison.Group, m Mode) error { return nil } + +// Create builds a new Input instance from the given configuration, or returns +// an error if the configuration is invalid. +func (manager *simpleInputManager) Create(cfg *conf.C) (Input, error) { + return manager.configure(cfg) +} diff --git a/pkg/manager/input/util_test.go b/pkg/manager/input/util_test.go new file mode 100644 index 0000000..fa20527 --- /dev/null +++ b/pkg/manager/input/util_test.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package input + +import ( + "errors" + "testing" + + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/go-concert/unison" +) + +type fakeInputManager struct { + OnInit func(Mode) error + OnConfigure func(*conf.C) (Input, error) +} + +type fakeInput struct { + Type string + OnTest func(TestContext) error + OnRun func(Context, publisher.PipelineConnector) error +} + +func makeConfigFakeInput(prototype fakeInput) func(*conf.C) (Input, error) { + return func(cfg *conf.C) (Input, error) { + tmp := prototype + return &tmp, nil + } +} + +func (m *fakeInputManager) Init(_ unison.Group, mode Mode) error { + if m.OnInit != nil { + return m.OnInit(mode) + } + return nil +} + +func (m *fakeInputManager) Create(cfg *conf.C) (Input, error) { + if m.OnConfigure != nil { + return m.OnConfigure(cfg) + } + return nil, errors.New("oops") +} + +func (f *fakeInput) Name() string { return f.Type } +func (f *fakeInput) Test(ctx TestContext) error { + if f.OnTest != nil { + return f.OnTest(ctx) + } + return nil +} + +func (f *fakeInput) Run(ctx Context, pipeline publisher.PipelineConnector) error { + if f.OnRun != nil { + return f.OnRun(ctx, pipeline) + } + return nil +} + +func expectError(t *testing.T, err error) { + if err == nil { + t.Errorf("expected error") + } +} + +func expectNoError(t *testing.T, err error) { + if err != nil { + t.Errorf("unexpected error: %v", err) + } +} diff --git a/pkg/manager/internal/resources/goroutines.go b/pkg/manager/internal/resources/goroutines.go new file mode 100644 index 0000000..780070f --- /dev/null +++ b/pkg/manager/internal/resources/goroutines.go @@ -0,0 +1,112 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package resources + +import ( + "errors" + "fmt" + "os" + "runtime" + "runtime/pprof" + "testing" + "time" +) + +// This is the maximum waiting time for goroutine shutdown. +// If the shutdown happens earlier the waiting time will be lower. +// High maximum waiting time was due to flaky tests on CI workers +const defaultFinalizationTimeout = 35 * time.Second + +// GoroutinesChecker keeps the count of goroutines when it was created +// so later it can check if this number has increased +type GoroutinesChecker struct { + before int + + // FinalizationTimeout is the time to wait till goroutines have finished + FinalizationTimeout time.Duration +} + +// NewGoroutinesChecker creates a new GoroutinesChecker +func NewGoroutinesChecker() GoroutinesChecker { + return GoroutinesChecker{ + before: runtime.NumGoroutine(), + FinalizationTimeout: defaultFinalizationTimeout, + } +} + +// Check if the number of goroutines has increased since the checker +// was created +func (c GoroutinesChecker) Check(t testing.TB) { + t.Helper() + err := c.check() + if err != nil { + dumpGoroutines() + t.Error(err) + } +} + +func dumpGoroutines() { + profile := pprof.Lookup("goroutine") + _ = profile.WriteTo(os.Stdout, 2) +} + +func (c GoroutinesChecker) check() error { + after, err := c.WaitUntilOriginalCount() + if errors.Is(err, ErrTimeout) { + return fmt.Errorf("possible goroutines leak, before: %d, after: %d", c.before, after) + } + return err +} + +// CallAndCheckGoroutines calls a function and checks if it has increased +// the number of goroutines +func CallAndCheckGoroutines(t testing.TB, f func()) { + t.Helper() + c := NewGoroutinesChecker() + f() + c.Check(t) +} + +// ErrTimeout is the error returned when WaitUntilOriginalCount timeouts. +var ErrTimeout = fmt.Errorf("timeout waiting for finalization of goroutines") + +// WaitUntilOriginalCount waits until the original number of goroutines are +// present before we created the resource checker. +// It returns the number of goroutines after the check and a timeout error +// in case the wait has expired. +func (c GoroutinesChecker) WaitUntilOriginalCount() (int, error) { + timeout := time.Now().Add(c.FinalizationTimeout) + + var after int + for time.Now().Before(timeout) { + after = runtime.NumGoroutine() + if after <= c.before { + return after, nil + } + time.Sleep(10 * time.Millisecond) + } + return after, ErrTimeout +} + +// WaitUntilIncreased waits till the number of goroutines is n plus the number +// before creating the checker. +func (c *GoroutinesChecker) WaitUntilIncreased(n int) { + for runtime.NumGoroutine() < c.before+n { + time.Sleep(10 * time.Millisecond) + } +} diff --git a/pkg/manager/internal/resources/goroutines_test.go b/pkg/manager/internal/resources/goroutines_test.go new file mode 100644 index 0000000..369ba0c --- /dev/null +++ b/pkg/manager/internal/resources/goroutines_test.go @@ -0,0 +1,110 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package resources + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGoroutinesChecker(t *testing.T) { + cases := []struct { + title string + test func(ctl *goroutineTesterControl) + timeout time.Duration + fail bool + }{ + { + title: "no goroutines", + test: func(ctl *goroutineTesterControl) {}, + }, + { + title: "fast goroutine", + test: func(ctl *goroutineTesterControl) { + ctl.startGoroutine(func() {}) + }, + }, + { + title: "blocked goroutine", + test: func(ctl *goroutineTesterControl) { + ctl.startGoroutine(func() { + ctl.block() + }) + }, + timeout: 10 * time.Millisecond, + fail: true, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + ctl := newControl() + defer ctl.cleanup(t) + + goroutines := NewGoroutinesChecker() + if c.timeout > 0 { + goroutines.FinalizationTimeout = c.timeout + } + c.test(ctl) + err := goroutines.check() + if c.fail { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +// goroutineTesterControl helps keeping track of goroutines started for each test case. +type goroutineTesterControl struct { + checker GoroutinesChecker + blocker chan struct{} +} + +func newControl() *goroutineTesterControl { + return &goroutineTesterControl{ + checker: NewGoroutinesChecker(), + blocker: make(chan struct{}), + } +} + +// startGoroutine ensures that a goroutine is started before continuing. +func (c *goroutineTesterControl) startGoroutine(f func()) { + started := make(chan struct{}) + go func() { + started <- struct{}{} + f() + }() + <-started +} + +// block blocks forever (being "ever" the life of the test). +func (c *goroutineTesterControl) block() { + <-c.blocker +} + +// cleanup ensures that all started goroutines are finished. +func (c *goroutineTesterControl) cleanup(t *testing.T) { + close(c.blocker) + if _, err := c.checker.WaitUntilOriginalCount(); err != nil { + t.Fatal("goroutines in test cases should be started using startGoroutine") + } +} diff --git a/pkg/publisher/acker/acker.go b/pkg/publisher/acker/acker.go new file mode 100644 index 0000000..9ad3c41 --- /dev/null +++ b/pkg/publisher/acker/acker.go @@ -0,0 +1,341 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package acker + +import ( + "sync" + + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + "go.uber.org/atomic" +) + +// Nil creates an ACKer that does nothing. +func Nil() publisher.ACKer { + return nilACKer{} +} + +type nilACKer struct{} + +func (nilACKer) AddEvent(event publisher.Event, published bool) {} +func (nilACKer) ACKEvents(n int) {} +func (nilACKer) Close() {} + +// RawCounting reports the number of ACKed events as has been reported by the outputs or queue. +// The ACKer does not keep track of dropped events. Events after the client has +// been closed will still be reported. +func RawCounting(fn func(int)) publisher.ACKer { + return countACKer(fn) +} + +type countACKer func(int) + +func (countACKer) AddEvent(_ publisher.Event, _ bool) {} +func (fn countACKer) ACKEvents(n int) { fn(n) } +func (countACKer) Close() {} + +// TrackingCounter keeps track of published and dropped events. It reports +// the number of acked events from the queue in the 'acked' argument and the +// total number of events published via the Client in the 'total' argument. +// The TrackingCountACKer keeps track of the order of events being send and events being acked. +// If N events have been acked by the output, then `total` will include all events dropped in between +// the last forwarded N events and the 'tail' of dropped events. For example (X = send, D = dropped): +// +// index: 0 1 2 3 4 5 6 7 8 9 10 11 +// event: X X D D X D D X D X X X +// +// If the output ACKs 3 events, then all events from index 0 to 6 will be reported because: +// - the drop sequence for events 2 and 3 is in between the number of forwarded and ACKed events +// - events 5-6 have been dropped as well, but event 7 is not ACKed yet +// +// If there is no event currently tracked by this ACKer and the next event is dropped by the processors, +// then `fn` will be called immediately with acked=0 and total=1. +func TrackingCounter(fn func(acked, total int)) publisher.ACKer { + a := &trackingACKer{fn: fn} + init := &gapInfo{} + a.lst.head = init + a.lst.tail = init + return a +} + +// Counting returns an ACK count for all events a client has tried to publish. +// The ACKer keeps track of dropped events as well, and adjusts the ACK from the outputs accordingly. +func Counting(fn func(n int)) publisher.ACKer { + return TrackingCounter(func(_ int, total int) { + fn(total) + }) +} + +type trackingACKer struct { + fn func(acked, total int) + events atomic.Uint32 + lst gapList +} + +type gapList struct { + sync.Mutex + head, tail *gapInfo +} + +type gapInfo struct { + sync.Mutex + next *gapInfo + send, dropped int +} + +func (a *trackingACKer) AddEvent(_ publisher.Event, published bool) { + a.events.Inc() + if published { + a.addPublishedEvent() + } else { + a.addDropEvent() + } +} + +// addPublishedEvent increments the 'send' counter in the current gapInfo +// element in the tail of the list. If events have been dropped, we append a +// new empty gapInfo element. +func (a *trackingACKer) addPublishedEvent() { + a.lst.Lock() + + current := a.lst.tail + current.Lock() + if current.dropped > 0 { + tmp := &gapInfo{} + tmp.Lock() + + a.lst.tail.next = tmp + a.lst.tail = tmp + current.Unlock() + current = tmp + } + a.lst.Unlock() + + current.send++ + current.Unlock() +} + +// addDropEvent increments the 'dropped' counter in the gapInfo element in the +// tail of the list. The callback will be run with total=1 and acked=0 if the +// acker state is empty and no events have been send yet. +func (a *trackingACKer) addDropEvent() { + a.lst.Lock() + + current := a.lst.tail + current.Lock() + + if current.send == 0 && current.next == nil { + // send can only be 0 if no no events/gaps present yet + if a.lst.head != a.lst.tail { + panic("gap list expected to be empty") + } + + a.fn(0, 1) + a.lst.Unlock() + current.Unlock() + + a.events.Dec() + return + } + + a.lst.Unlock() + current.dropped++ + current.Unlock() +} + +func (a *trackingACKer) ACKEvents(n int) { + var ( + total = 0 + acked = n + emptyLst bool + ) + + for n > 0 { + if emptyLst { + panic("too many events acked") + } + + a.lst.Lock() + current := a.lst.head + current.Lock() + + // advance list if we detect that the current head will be completely consumed + // by this ACK event. + if n >= current.send { + next := current.next + emptyLst = next == nil + if !emptyLst { + // advance list all event in current entry have been send and list as + // more then 1 gapInfo entry. If only 1 entry is present, list item will be + // reset and reused + a.lst.head = next + } + } + // hand over lock list-entry, so ACK handler and producer can operate + // on potentially different list ends + a.lst.Unlock() + + if n < current.send { + current.send -= n + total += n + n = 0 + } else { + total += current.send + current.dropped + n -= current.send + current.dropped = 0 + current.send = 0 + } + current.Unlock() + } + + a.events.Sub(uint32(total)) + a.fn(acked, total) +} + +func (a *trackingACKer) Close() {} + +// EventPrivateReporter reports all private fields from all events that have +// been published or removed. +// +// The EventPrivateFieldsACKer keeps track of the order of events being send +// and events being acked. If N events have been acked by the output, then +// `total` will include all events dropped in between the last forwarded N +// events and the 'tail' of dropped events. For example (X = send, D = +// dropped): +// +// index: 0 1 2 3 4 5 6 7 8 9 10 11 +// event: X X D D X D D X D X X X +// +// If the output ACKs 3 events, then all events from index 0 to 6 will be reported because: +// - the drop sequence for events 2 and 3 is in between the number of forwarded and ACKed events +// - events 5-6 have been dropped as well, but event 7 is not ACKed yet +func EventPrivateReporter(fn func(acked int, data []interface{})) publisher.ACKer { + a := &eventDataACKer{fn: fn} + a.ACKer = TrackingCounter(a.onACK) + return a +} + +type eventDataACKer struct { + publisher.ACKer + mu sync.Mutex + data []interface{} + fn func(acked int, data []interface{}) +} + +func (a *eventDataACKer) AddEvent(event publisher.Event, published bool) { + a.mu.Lock() + a.data = append(a.data, event.Private) + a.mu.Unlock() + a.ACKer.AddEvent(event, published) +} + +func (a *eventDataACKer) onACK(acked, total int) { + if total == 0 { + return + } + + a.mu.Lock() + data := a.data[:total] + a.data = a.data[total:] + a.mu.Unlock() + + if len(data) > 0 { + a.fn(acked, data) + } +} + +// LastEventPrivateReporter reports only the 'latest' published and acked +// event if a batch of events have been ACKed. +func LastEventPrivateReporter(fn func(acked int, data interface{})) publisher.ACKer { + ignored := 0 + return EventPrivateReporter(func(acked int, data []interface{}) { + for i := len(data) - 1; i >= 0; i-- { + if d := data[i]; d != nil { + fn(ignored+acked, d) + ignored = 0 + return + } + } + + // complete batch has been ignored due to missing data -> add count + ignored += acked + }) +} + +// Combine forwards events to a list of ackers. +func Combine(as ...publisher.ACKer) publisher.ACKer { + return ackerList(as) +} + +type ackerList []publisher.ACKer + +func (l ackerList) AddEvent(event publisher.Event, published bool) { + for _, a := range l { + a.AddEvent(event, published) + } +} + +func (l ackerList) ACKEvents(n int) { + for _, a := range l { + a.ACKEvents(n) + } +} + +func (l ackerList) Close() { + for _, a := range l { + a.Close() + } +} + +// ConnectionOnly ensures that the given ACKer is only used for as long as the +// pipeline Client is active. Once the Client is closed, the ACKer will drop +// its internal state and no more ACK events will be processed. +func ConnectionOnly(a publisher.ACKer) publisher.ACKer { + return &clientOnlyACKer{acker: a} +} + +type clientOnlyACKer struct { + mu sync.Mutex + acker publisher.ACKer +} + +func (a *clientOnlyACKer) AddEvent(event publisher.Event, published bool) { + a.mu.Lock() + defer a.mu.Unlock() + if sub := a.acker; sub != nil { + sub.AddEvent(event, published) + } +} + +func (a *clientOnlyACKer) ACKEvents(n int) { + a.mu.Lock() + sub := a.acker + a.mu.Unlock() + if sub != nil { + sub.ACKEvents(n) + } +} + +func (a *clientOnlyACKer) Close() { + a.mu.Lock() + sub := a.acker + a.acker = nil // drop the internal ACKer on Close and allow the runtime to gc accumulated state. + a.mu.Unlock() + if sub != nil { + sub.Close() + } +} diff --git a/pkg/publisher/acker/acker_test.go b/pkg/publisher/acker/acker_test.go new file mode 100644 index 0000000..f3b5e89 --- /dev/null +++ b/pkg/publisher/acker/acker_test.go @@ -0,0 +1,250 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//nolint:dupl // tests are equivalent +package acker + +import ( + "testing" + + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + "github.com/stretchr/testify/require" +) + +type fakeACKer struct { + AddEventFunc func(event publisher.Event, published bool) + ACKEventsFunc func(n int) + CloseFunc func() +} + +func TestNil(t *testing.T) { + acker := Nil() + require.NotNil(t, acker) + + // check acker can be used without panic: + acker.AddEvent(publisher.Event{}, false) + acker.AddEvent(publisher.Event{}, true) + acker.ACKEvents(3) + acker.Close() +} + +func TestCounting(t *testing.T) { + t.Run("ack count is passed through", func(t *testing.T) { + var n int + acker := RawCounting(func(acked int) { n = acked }) + acker.ACKEvents(3) + require.Equal(t, 3, n) + }) +} + +func TestTracking(t *testing.T) { + t.Run("dropped event is acked immediately if empty", func(t *testing.T) { + var acked, total int + TrackingCounter(func(a, t int) { acked, total = a, t }).AddEvent(publisher.Event{}, false) + require.Equal(t, 0, acked) + require.Equal(t, 1, total) + }) + + t.Run("no dropped events", func(t *testing.T) { + var acked, total int + acker := TrackingCounter(func(a, t int) { acked, total = a, t }) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, true) + acker.ACKEvents(2) + require.Equal(t, 2, acked) + require.Equal(t, 2, total) + }) + + t.Run("acking published includes dropped events in middle", func(t *testing.T) { + var acked, total int + acker := TrackingCounter(func(a, t int) { acked, total = a, t }) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, false) + acker.AddEvent(publisher.Event{}, false) + acker.AddEvent(publisher.Event{}, true) + acker.ACKEvents(2) + require.Equal(t, 2, acked) + require.Equal(t, 4, total) + }) + + t.Run("acking published includes dropped events at end of ACK interval", func(t *testing.T) { + var acked, total int + acker := TrackingCounter(func(a, t int) { acked, total = a, t }) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, false) + acker.AddEvent(publisher.Event{}, false) + acker.AddEvent(publisher.Event{}, true) + acker.ACKEvents(2) + require.Equal(t, 2, acked) + require.Equal(t, 4, total) + }) + + t.Run("partial ACKs", func(t *testing.T) { + var acked, total int + acker := TrackingCounter(func(a, t int) { acked, total = a, t }) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, false) + acker.AddEvent(publisher.Event{}, true) + acker.AddEvent(publisher.Event{}, true) + + acker.ACKEvents(2) + require.Equal(t, 2, acked) + require.Equal(t, 2, total) + + acker.ACKEvents(2) + require.Equal(t, 2, acked) + require.Equal(t, 3, total) + }) +} + +func TestEventPrivateReporter(t *testing.T) { + t.Run("dropped event is acked immediately if empty", func(t *testing.T) { + var acked int + var data []interface{} + acker := EventPrivateReporter(func(a int, d []interface{}) { acked, data = a, d }) + acker.AddEvent(publisher.Event{Private: 1}, false) + require.Equal(t, 0, acked) + require.Equal(t, []interface{}{1}, data) + }) + + t.Run("no dropped events", func(t *testing.T) { + var acked int + var data []interface{} + acker := EventPrivateReporter(func(a int, d []interface{}) { acked, data = a, d }) + acker.AddEvent(publisher.Event{Private: 1}, true) + acker.AddEvent(publisher.Event{Private: 2}, true) + acker.AddEvent(publisher.Event{Private: 3}, true) + acker.ACKEvents(3) + require.Equal(t, 3, acked) + require.Equal(t, []interface{}{1, 2, 3}, data) + }) + + t.Run("private of dropped events is included", func(t *testing.T) { + var acked int + var data []interface{} + acker := EventPrivateReporter(func(a int, d []interface{}) { acked, data = a, d }) + acker.AddEvent(publisher.Event{Private: 1}, true) + acker.AddEvent(publisher.Event{Private: 2}, false) + acker.AddEvent(publisher.Event{Private: 3}, true) + acker.ACKEvents(2) + require.Equal(t, 2, acked) + require.Equal(t, []interface{}{1, 2, 3}, data) + }) +} + +func TestLastEventPrivateReporter(t *testing.T) { + t.Run("dropped event with private is acked immediately if empty", func(t *testing.T) { + var acked int + var datum interface{} + acker := LastEventPrivateReporter(func(a int, d interface{}) { acked, datum = a, d }) + acker.AddEvent(publisher.Event{Private: 1}, false) + require.Equal(t, 0, acked) + require.Equal(t, 1, datum) + }) + + t.Run("dropped event without private is ignored", func(t *testing.T) { + var called bool + acker := LastEventPrivateReporter(func(_ int, _ interface{}) { called = true }) + acker.AddEvent(publisher.Event{Private: nil}, false) + require.False(t, called) + }) + + t.Run("no dropped events", func(t *testing.T) { + var acked int + var data interface{} + acker := LastEventPrivateReporter(func(a int, d interface{}) { acked, data = a, d }) + acker.AddEvent(publisher.Event{Private: 1}, true) + acker.AddEvent(publisher.Event{Private: 2}, true) + acker.AddEvent(publisher.Event{Private: 3}, true) + acker.ACKEvents(3) + require.Equal(t, 3, acked) + require.Equal(t, 3, data) + }) +} + +func TestCombine(t *testing.T) { + t.Run("AddEvent distributes", func(t *testing.T) { + var a1, a2 int + acker := Combine(countACKerOps(&a1, nil, nil), countACKerOps(&a2, nil, nil)) + acker.AddEvent(publisher.Event{}, true) + require.Equal(t, 1, a1) + require.Equal(t, 1, a2) + }) + + t.Run("ACKEvents distributes", func(t *testing.T) { + var a1, a2 int + acker := Combine(countACKerOps(nil, &a1, nil), countACKerOps(nil, &a2, nil)) + acker.ACKEvents(1) + require.Equal(t, 1, a1) + require.Equal(t, 1, a2) + }) + + t.Run("Close distributes", func(t *testing.T) { + var c1, c2 int + acker := Combine(countACKerOps(nil, nil, &c1), countACKerOps(nil, nil, &c2)) + acker.Close() + require.Equal(t, 1, c1) + require.Equal(t, 1, c2) + }) +} + +func TestConnectionOnly(t *testing.T) { + t.Run("passes ACKs if not closed", func(t *testing.T) { + var n int + acker := ConnectionOnly(RawCounting(func(acked int) { n = acked })) + acker.ACKEvents(3) + require.Equal(t, 3, n) + }) + + t.Run("ignores ACKs after close", func(t *testing.T) { + var n int + acker := ConnectionOnly(RawCounting(func(acked int) { n = acked })) + acker.Close() + acker.ACKEvents(3) + require.Equal(t, 0, n) + }) +} + +func countACKerOps(add, acked, close *int) publisher.ACKer { + return &fakeACKer{ + AddEventFunc: func(_ publisher.Event, _ bool) { *add++ }, + ACKEventsFunc: func(_ int) { *acked++ }, + CloseFunc: func() { *close++ }, + } +} + +func (f *fakeACKer) AddEvent(event publisher.Event, published bool) { + if f.AddEventFunc != nil { + f.AddEventFunc(event, published) + } +} + +func (f *fakeACKer) ACKEvents(n int) { + if f.ACKEventsFunc != nil { + f.ACKEventsFunc(n) + } +} + +func (f *fakeACKer) Close() { + if f.CloseFunc != nil { + f.CloseFunc() + } +} diff --git a/pkg/publisher/event.go b/pkg/publisher/event.go new file mode 100644 index 0000000..7c5aa08 --- /dev/null +++ b/pkg/publisher/event.go @@ -0,0 +1,25 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package publisher + +import "github.com/elastic/elastic-agent-libs/mapstr" + +type Event struct { + Fields mapstr.M + Private interface{} +} diff --git a/pkg/publisher/pipeline.go b/pkg/publisher/pipeline.go new file mode 100644 index 0000000..29335bf --- /dev/null +++ b/pkg/publisher/pipeline.go @@ -0,0 +1,177 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package publisher + +import ( + "time" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// Pipeline provides access to libbeat event publishing by creating a Client +// instance. +type Pipeline interface { + ConnectWith(ClientConfig) (Client, error) + Connect() (Client, error) +} + +type PipelineConnector = Pipeline + +// Client holds a connection to the beats publisher pipeline +type Client interface { + Publish(Event) + PublishAll([]Event) + Close() error +} + +// ClientConfig defines common configuration options one can pass to +// Pipeline.ConnectWith to control the clients behavior and provide ACK support. +type ClientConfig struct { + PublishMode PublishMode + + Processing ProcessingConfig + + CloseRef CloseRef + + // WaitClose sets the maximum duration to wait on ACK, if client still has events + // active non-acknowledged events in the publisher pipeline. + // WaitClose is only effective if one of ACKCount, ACKEvents and ACKLastEvents + // is configured + WaitClose time.Duration + + // Configure ACK callback. + ACKHandler ACKer + + // Events configures callbacks for common client callbacks + Events ClientEventer +} + +// ACKer can be registered with a Client when connecting to the pipeline. +// The ACKer will be informed when events are added or dropped by the processors, +// and when an event has been ACKed by the outputs. +// +// Due to event publishing and ACKing are asynchronous operations, the +// operations on ACKer are normally executed in different go routines. ACKers +// are required to be multi-threading safe. +type ACKer interface { + // AddEvent informs the ACKer that a new event has been send to the client. + // AddEvent is called after the processors have handled the event. If the + // event has been dropped by the processor `published` will be set to true. + // This allows the ACKer to do some bookeeping for dropped events. + AddEvent(event Event, published bool) + + // ACK Events from the output and pipeline queue are forwarded to ACKEvents. + // The number of reported events only matches the known number of events downstream. + // ACKers might need to keep track of dropped events by themselves. + ACKEvents(n int) + + // Close informs the ACKer that the Client used to publish to the pipeline has been closed. + // No new events should be published anymore. The ACKEvents method still will be actively called + // as long as there are pending events for the client in the pipeline. The Close signal can be used + // to suppress any ACK event propagation if required. + // Close might be called from another go-routine than AddEvent and ACKEvents. + Close() +} + +// CloseRef allows users to close the client asynchronously. +// A CloseReg implements a subset of function required for context.Context. +type CloseRef interface { + Done() <-chan struct{} + Err() error +} + +// ProcessingConfig provides additional event processing settings a client can +// pass to the publisher pipeline on Connect. +type ProcessingConfig struct { + // EventMetadata configures additional fields/tags to be added to published events. + EventMetadata mapstr.EventMetadata + + // Meta provides additional meta data to be added to the Meta field in the beat.Event + // structure. + Meta mapstr.M + + // Fields provides additional 'global' fields to be added to every event + Fields mapstr.M + + // DynamicFields provides additional fields to be added to every event, supporting live updates + DynamicFields *mapstr.Pointer + + // Processors passes additional processor to the client, to be executed before + // the pipeline processors. + Processor ProcessorList + + // KeepNull determines whether published events will keep null values or omit them. + KeepNull bool + + // Disables the addition of host.name if it was enabled for the publisher. + DisableHost bool + + // Private contains additional information to be passed to the processing + // pipeline builder. + Private interface{} +} + +// ClientEventer provides access to internal client events. +type ClientEventer interface { + Closing() // Closing indicates the client is being shutdown next + Closed() // Closed indicates the client being fully shutdown + + Published() // event has been successfully forwarded to the publisher pipeline + FilteredOut(Event) // event has been filtered out/dropped by processors + DroppedOnPublish(Event) // event has been dropped, while waiting for the queue +} + +type ProcessorList interface { + Processor + Close() error + All() []Processor +} + +// Processor defines the minimal required interface for processor, that can be +// registered with the publisher pipeline. +type Processor interface { + String() string // print full processor description + Run(in *Event) (event *Event, err error) +} + +// PublishMode enum sets some requirements on the client connection to the beats +// publisher pipeline +type PublishMode uint8 + +const ( + // DefaultGuarantees are up to the pipeline configuration itself. + DefaultGuarantees PublishMode = iota + + // OutputChooses mode fully depends on the output and its configuration. + // Events might be dropped based on the users output configuration. + // In this mode no events are dropped within the pipeline. Events are only removed + // after the output has ACKed the events to the pipeline, even if the output + // did drop the events. + OutputChooses + + // GuaranteedSend ensures events are retried until acknowledged by the output. + // Normally guaranteed sending should be used with some client ACK-handling + // to update state keeping track of the sending status. + GuaranteedSend + + // DropIfFull drops an event to be send if the pipeline is currently full. + // This ensures a beats internals can continue processing if the pipeline has + // filled up. Useful if an event stream must be processed to keep internal + // state up-to-date. + DropIfFull +) diff --git a/pkg/publisher/testing/connector.go b/pkg/publisher/testing/connector.go new file mode 100644 index 0000000..c0d9377 --- /dev/null +++ b/pkg/publisher/testing/connector.go @@ -0,0 +1,134 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package testing + +import ( + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + "github.com/elastic/elastic-agent-libs/atomic" +) + +// ClientCounter can be used to create a publisher.PipelineConnector that count +// pipeline connects and disconnects. +type ClientCounter struct { + total atomic.Int + active atomic.Int +} + +// FakeConnector implements the publisher.PipelineConnector interface. +// The ConnectFunc is called for each connection attempt, and must be provided +// by tests that wish to use FakeConnector. If ConnectFunc is nil tests will panic +// if there is a connection attempt. +type FakeConnector struct { + ConnectFunc func(publisher.ClientConfig) (publisher.Client, error) +} + +// FakeClient implements the publisher.Client interface. The implementation of a +// custom PublishFunc and CloseFunc are optional. +type FakeClient struct { + // If set PublishFunc is called for each event that is published by a producer. + PublishFunc func(publisher.Event) + + // If set CloseFunc is called on Close. Otherwise Close returns nil. + CloseFunc func() error +} + +var _ publisher.PipelineConnector = FakeConnector{} +var _ publisher.Client = (*FakeClient)(nil) + +// ConnectWith calls the ConnectFunc with the given configuration. +func (c FakeConnector) ConnectWith(cfg publisher.ClientConfig) (publisher.Client, error) { + return c.ConnectFunc(cfg) +} + +// Connect calls the ConnectFunc with an empty configuration. +func (c FakeConnector) Connect() (publisher.Client, error) { + return c.ConnectWith(publisher.ClientConfig{}) +} + +// Publish calls PublishFunc, if PublishFunc is not nil. +func (c *FakeClient) Publish(event publisher.Event) { + if c.PublishFunc != nil { + c.PublishFunc(event) + } +} + +// Close calls CloseFunc, if CloseFunc is not nil. Otherwise it returns nil. +func (c *FakeClient) Close() error { + if c.CloseFunc == nil { + return nil + } + return c.CloseFunc() +} + +// PublishAll calls PublishFunc for each event in the given slice. +func (c *FakeClient) PublishAll(events []publisher.Event) { + for _, event := range events { + c.Publish(event) + } +} + +// FailingConnector creates a pipeline that will always fail with the +// configured error value. +func FailingConnector(err error) publisher.PipelineConnector { + return &FakeConnector{ + ConnectFunc: func(_ publisher.ClientConfig) (publisher.Client, error) { + return nil, err + }, + } +} + +// ConstClient returns a pipeline that always returns the pre-configured publisher.Client instance. +func ConstClient(client publisher.Client) publisher.PipelineConnector { + return &FakeConnector{ + ConnectFunc: func(_ publisher.ClientConfig) (publisher.Client, error) { + return client, nil + }, + } +} + +// ChClient create a publisher.Client that will forward all events to the given channel. +func ChClient(ch chan publisher.Event) publisher.Client { + return &FakeClient{ + PublishFunc: func(event publisher.Event) { + ch <- event + }, + } +} + +// Active returns the number of currently active connections. +func (c *ClientCounter) Active() int { return c.active.Load() } + +// Total returns the total number of calls to Connect. +func (c *ClientCounter) Total() int { return c.total.Load() } + +// BuildConnector create a pipeline that updates the active and tocal +// connection counters on Connect and Close calls. +func (c *ClientCounter) BuildConnector() publisher.PipelineConnector { + return FakeConnector{ + ConnectFunc: func(_ publisher.ClientConfig) (publisher.Client, error) { + c.total.Inc() + c.active.Inc() + return &FakeClient{ + CloseFunc: func() error { + c.active.Dec() + return nil + }, + }, nil + }, + } +} diff --git a/pkg/publisher/testing/testing.go b/pkg/publisher/testing/testing.go new file mode 100644 index 0000000..3fd7f25 --- /dev/null +++ b/pkg/publisher/testing/testing.go @@ -0,0 +1,95 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package testing + +import "github.com/elastic/elastic-agent-inputs/pkg/publisher" + +// ChanClient implements Client interface, forwarding published events to some + +type TestPublisher struct { + client publisher.Client +} + +// given channel only. +type ChanClient struct { + done chan struct{} + Channel chan publisher.Event + publishCallback func(event publisher.Event) +} + +func PublisherWithClient(client publisher.Client) publisher.Pipeline { + return &TestPublisher{client} +} + +func (pub *TestPublisher) Connect() (publisher.Client, error) { + return pub.client, nil +} + +func (pub *TestPublisher) ConnectWith(_ publisher.ClientConfig) (publisher.Client, error) { + return pub.client, nil +} + +func NewChanClientWithCallback(bufSize int, callback func(event publisher.Event)) *ChanClient { + chanClient := NewChanClientWith(make(chan publisher.Event, bufSize)) + chanClient.publishCallback = callback + + return chanClient +} + +func NewChanClient(bufSize int) *ChanClient { + return NewChanClientWith(make(chan publisher.Event, bufSize)) +} + +func NewChanClientWith(ch chan publisher.Event) *ChanClient { + if ch == nil { + ch = make(chan publisher.Event, 1) + } + c := &ChanClient{ + done: make(chan struct{}), + Channel: ch, + } + return c +} + +func (c *ChanClient) Close() error { + close(c.done) + return nil +} + +// PublishEvent will publish the event on the channel. Options will be ignored. +// Always returns true. +func (c *ChanClient) Publish(event publisher.Event) { + select { + case <-c.done: + case c.Channel <- event: + if c.publishCallback != nil { + c.publishCallback(event) + <-c.Channel + } + } +} + +func (c *ChanClient) PublishAll(event []publisher.Event) { + for _, e := range event { + c.Publish(e) + } +} + +func (c *ChanClient) ReceiveEvent() publisher.Event { + return <-c.Channel +} diff --git a/pkg/publisher/testing/testing_test.go b/pkg/publisher/testing/testing_test.go new file mode 100644 index 0000000..3c34c19 --- /dev/null +++ b/pkg/publisher/testing/testing_test.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package testing + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-inputs/pkg/publisher" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +var cnt = 0 + +func testEvent() publisher.Event { + event := publisher.Event{ + Fields: mapstr.M{ + "message": "test", + "idx": cnt, + }, + } + cnt++ + return event +} + +// Test that ChanClient writes an event to its Channel. +func TestChanClientPublishEvent(t *testing.T) { + cc := NewChanClient(1) + e1 := testEvent() + cc.Publish(e1) + assert.Equal(t, e1, cc.ReceiveEvent()) +} + +// Test that ChanClient write events to its Channel. +func TestChanClientPublishEvents(t *testing.T) { + cc := NewChanClient(1) + + e1, e2 := testEvent(), testEvent() + go cc.PublishAll([]publisher.Event{e1, e2}) + assert.Equal(t, e1, cc.ReceiveEvent()) + assert.Equal(t, e2, cc.ReceiveEvent()) +} diff --git a/pkg/statestore/backend/backend.go b/pkg/statestore/backend/backend.go new file mode 100644 index 0000000..c40d851 --- /dev/null +++ b/pkg/statestore/backend/backend.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package backend + +// Registry provides access to stores managed by the backend storage. +type Registry interface { + // Access opens a store. The store will be closed by the frontend, once all + // accessed stores have been closed. + // + // The Store instance returned must be threadsafe. + Access(name string) (Store, error) + + // Close is called on shutdown after all stores have been closed. + // An implementation of Registry is not required to check for the stores to be closed. + Close() error +} + +// ValueDecoder is used to decode values into go structs or maps within a transaction. +// A ValueDecoder is supposed to be invalidated by beats after the loop operations has returned. +type ValueDecoder interface { + Decode(to interface{}) error +} + +// Store provides access to key value pairs. +type Store interface { + // Close should close the store and release all used resources. + Close() error + + // Has checks if the key exists. No error must be returned if the key does + // not exists, but the bool return must be false. + // An error return value must indicate internal errors only. The store is + // assumed to be in a 'bad' but recoverable state if 'Has' fails. + Has(key string) (bool, error) + + // Get decodes the value for the given key into value. + // Besides internal implementation specific errors an error is assumed + // to be returned if the key does not exist or the type of the value + // passed is incompatible to the actual value in the store (decoding error). + Get(key string, value interface{}) error + + // Set inserts or overwrites a key pair in the store. + // The `value` parameters can be assumed to be a struct or a map. Besides + // internal implementation specific errors, an error should be returned if + // the value given can not be encoded. + Set(key string, value interface{}) error + + // Remove removes and entry from the store. + Remove(string) error + + // Each loops over all key value pairs in the store calling fn for each pair. + // The ValueDecoder is used by fn to optionally decode the value into a + // custom struct or map. The decoder must be executable multiple times, but + // is assumed to be invalidated once fn returns + // The loop shall return if fn returns an error or false. + Each(fn func(string, ValueDecoder) (bool, error)) error +} diff --git a/pkg/statestore/backend/memlog/diskstore.go b/pkg/statestore/backend/memlog/diskstore.go new file mode 100644 index 0000000..e4843fa --- /dev/null +++ b/pkg/statestore/backend/memlog/diskstore.go @@ -0,0 +1,699 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/cleanup" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// diskstore manages the on-disk state of the memlog store. +type diskstore struct { + log *logp.Logger + + // store configuration + checkpointPred CheckpointPredicate + fileMode os.FileMode + bufferSize int + + // on disk file tracking information + home string // home path of the store + logFilePath string // current log file + oldDataFiles []dataFileInfo // unused data files that can be removed + activeDataFile dataFileInfo // most recent data file that needs to be kept on disk + + // nextTxID is the sequential counter that tracks + // all updates to the store. The nextTxID is added to operation being logged + // used as name for the data files. + nextTxID uint64 + + // log file access. The log file is updated using an in memory write buffer. + logFile *os.File + logBuf *bufio.Writer + + // internal state and metrics + logFileSize uint64 + logEntries uint + logInvalid bool + logNeedsTruncate bool +} + +// dataFileInfo is used to track and sort on disk data files. +// We should have only one data file on disk, but in case delete operations +// have failed or not finished dataFileInfo is used to detect the ordering. +// +// dataFileInfo can be ordered on txid. When sorting isTxIDLessEqual should be +// used, to get the correct ordering even in the case of integer overflows. +// For sorting a slice of dataFileInfo use sortDataFileInfos. +type dataFileInfo struct { + path string + txid uint64 +} + +// storeEntry is used to write entries to the checkpoint file only. +type storeEntry struct { + Key string `struct:"_key"` + Fields mapstr.M `struct:",inline"` +} + +// storeMeta is read from the meta file. +type storeMeta struct { + Version string `struct:"version"` +} + +// logAction is prepended to each operation logged to the update file. +// It contains the update ID, a sequential counter to track correctness, +// and the action name. +type logAction struct { + Op string `json:"op"` + ID uint64 `json:"id"` +} + +const ( + logFileName = "log.json" + metaFileName = "meta.json" + activeDataFileName = "active.dat" + activeDataTmpFileName = "active.dat.new" + checkpointTmpFileName = "checkpoint.new" + + storeVersion = "1" + + keyField = "_key" +) + +// newDiskStore initializes the disk store structure only. The store must have +// been opened already. It tries to open the update log file for append +// operations. If opening the update log file fails, it is marked as +// 'corrupted', triggering a checkpoint operation on the first update to the store. +func newDiskStore( + log *logp.Logger, + home string, + dataFiles []dataFileInfo, + txid uint64, + mode os.FileMode, + entries uint, + logInvalid bool, + bufferSize uint, + checkpointPred CheckpointPredicate, +) (*diskstore, error) { + var active dataFileInfo + if L := len(dataFiles); L > 0 { + active = dataFiles[L-1] + dataFiles = dataFiles[:L-1] + } + + s := &diskstore{ + log: log.With("path", home), + home: home, + logFilePath: filepath.Join(home, logFileName), + oldDataFiles: dataFiles, + activeDataFile: active, + nextTxID: txid + 1, + fileMode: mode, + bufferSize: int(bufferSize), + logFile: nil, + logBuf: nil, + logEntries: entries, + logInvalid: logInvalid, + logNeedsTruncate: false, // only truncate on next checkpoint + checkpointPred: checkpointPred, + } + + // delete temporary files from an older instances that was interrupted + // during a checkpoint process. + // Note: we do not delete old data files yet, in case we need them for debugging, + // or to manually restore some older state after disk outages. + if err := os.Remove(filepath.Join(home, checkpointTmpFileName)); err != nil && !os.IsNotExist(err) { + return nil, err + } + if err := os.Remove(filepath.Join(home, activeDataTmpFileName)); err != nil && !os.IsNotExist(err) { + return nil, err + } + + _ = s.tryOpenLog() + return s, nil +} + +// tryOpenLog access the update log. The log file is truncated if a checkpoint operation has been +// executed last. +// The log file is marked as invalid if opening it failed. This will trigger a checkpoint operation +// and another call to tryOpenLog in the future. +func (s *diskstore) tryOpenLog() error { + flags := os.O_RDWR | os.O_CREATE + if s.logNeedsTruncate { + flags |= os.O_TRUNC + } + + f, err := os.OpenFile(s.logFilePath, flags, s.fileMode) + if err != nil { + s.log.Errorf("Failed to open file %v: %v", s.logFilePath, err) + return err + } + + ok := false + defer cleanup.IfNot(&ok, func() { + f.Close() + }) + + _, err = f.Seek(0, os.SEEK_END) + if err != nil { + return err + } + + if s.logNeedsTruncate { + s.logEntries = 0 // reset counter if file was truncated on Open + s.logFileSize = 0 + } else { + info, err := f.Stat() + if err != nil { + return err + } + + s.logFileSize = uint64(info.Size()) + } + + ok = true + s.logNeedsTruncate = false + s.logFile = f + s.logBuf = bufio.NewWriterSize(&ensureWriter{s.logFile}, s.bufferSize) + return nil +} + +// mustCheckpoint returns true if the store is required to execute a checkpoint +// operation, either by predicate or by some internal state detecting a problem +// with the log file. +func (s *diskstore) mustCheckpoint() bool { + return s.logInvalid || s.checkpointPred(s.logFileSize) +} + +func (s *diskstore) Close() error { + if s.logFile != nil { + // always sync log file on ordinary shutdown. + err := s.logBuf.Flush() + if err == nil { + err = syncFile(s.logFile) + } + s.logFile.Close() + s.logFile = nil + s.logBuf = nil + return err + } + return nil +} + +// log operation adds another entry to the update log file. +// The log file is marked as invalid if the write fails. This will trigger a +// checkpoint operation in the future. +func (s *diskstore) LogOperation(op op) error { + if s.logInvalid { + return errLogInvalid + } + + if s.logFile == nil { + // We continue in case we have errors accessing the log file, but mark the + // store as invalid. This will force a full state checkpoint. + // The call to tryOpenLog prints some error log, we only use the error as + // indicator to invalidate the disk store, so we can try to recover by + // checkpointing. + if err := s.tryOpenLog(); err != nil { + s.logInvalid = true + return err + } + } + + writer := s.logBuf + counting := &countWriter{w: writer} + defer func() { + s.logFileSize += counting.n + }() + + ok := false + defer cleanup.IfNot(&ok, func() { + s.logInvalid = true + }) + + enc := newJSONEncoder(counting) + if err := enc.Encode(logAction{Op: op.name(), ID: s.nextTxID}); err != nil { + return err + } + _ = writer.WriteByte('\n') + + if err := enc.Encode(op); err != nil { + return err + } + _ = writer.WriteByte('\n') + + if err := writer.Flush(); err != nil { + return err + } + + ok = true + s.logEntries++ + s.nextTxID++ + return nil +} + +// WriteCheckpoint serializes all state into a json file. The file contains an +// array with all states known to the memory storage. +// WriteCheckpoint first serializes all state to a temporary file, and finally +// moves the temporary data file into the correct location. No files +// are overwritten or replaced. Instead the change sequence number is used for +// the filename, and older data files will be deleted after success. +// +// The active marker file is overwritten after all updates did succeed. The +// marker file contains the filename of the current valid data-file. +// NOTE: due to limitation on some Operating system or file systems, the active +// marker is not a symlink, but an actual file. +func (s *diskstore) WriteCheckpoint(state map[string]entry) error { + tmpPath, err := s.checkpointTmpFile(filepath.Join(s.home, checkpointTmpFileName), state) + if err != nil { + return err + } + + // silently try to delete the temporary checkpoint file on error. + // Deletion of tmpPath will fail if the rename operation did succeed. + defer os.Remove(tmpPath) + + // The checkpoint is assigned the next available transaction id. This + // guarantees that all existing log entries are 'older' then the checkpoint + // file and subsequenent operations. The first operation after a successful + // checkpoint will be (fileTxID + 1). + fileTxID := s.nextTxID + fileName := fmt.Sprintf("%v.json", fileTxID) + checkpointPath := filepath.Join(s.home, fileName) + + if err := os.Rename(tmpPath, checkpointPath); err != nil { + return err + } + trySyncPath(s.home) + + // clear transaction log once finished + s.checkpointClearLog() + + // finish current on-disk transaction by increasing the txid + s.nextTxID++ + + if s.activeDataFile.path != "" { + s.oldDataFiles = append(s.oldDataFiles, s.activeDataFile) + } + s.activeDataFile = dataFileInfo{ + path: checkpointPath, + txid: fileTxID, + } + + // delete old transaction files + _ = updateActiveMarker(s.log, s.home, s.activeDataFile.path) + s.removeOldDataFiles() + + trySyncPath(s.home) + return nil +} + +func (s *diskstore) checkpointTmpFile(tempfile string, states map[string]entry) (string, error) { + f, err := os.OpenFile(tempfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_SYNC, s.fileMode) + if err != nil { + return "", err + } + + ok := false + defer cleanup.IfNot(&ok, func() { + f.Close() + }) + + writer := bufio.NewWriterSize(&ensureWriter{f}, s.bufferSize) + enc := newJSONEncoder(writer) + if _, err = writer.Write([]byte{'['}); err != nil { + return "", err + } + + first := true + for key, entry := range states { + prefix := []byte(",\n") + if first { + prefix = prefix[1:] + first = false + } + if _, err = writer.Write(prefix); err != nil { + return "", err + } + + err = enc.Encode(storeEntry{ + Key: key, + Fields: entry.value, + }) + if err != nil { + return "", err + } + } + + if _, err = writer.Write([]byte("\n]")); err != nil { + return "", err + } + + if err = writer.Flush(); err != nil { + return "", err + } + + if err = syncFile(f); err != nil { + return "", err + } + + ok = true + if err = f.Close(); err != nil { + return "", err + } + + return tempfile, nil +} + +func (s *diskstore) checkpointClearLog() { + if s.logFile == nil { + s.logNeedsTruncate = true + return + } + + err := s.logFile.Truncate(0) + if err == nil { + _, err = s.logFile.Seek(0, io.SeekStart) + } + + if err != nil { + s.logFile.Close() + s.logFile = nil + s.logBuf = nil + s.logNeedsTruncate = true + s.logInvalid = true + } + + s.logEntries = 0 + s.logFileSize = 0 +} + +// updateActiveMarker overwrites the active.dat file in the home directory with +// the path of the most recent checkpoint file. +// The active file will be written to ``/active.dat. +func updateActiveMarker(log *logp.Logger, homePath, checkpointFilePath string) error { + activeLink := filepath.Join(homePath, activeDataFileName) + tmpLink := filepath.Join(homePath, activeDataTmpFileName) + log = log.With("temporary", tmpLink, "data_file", checkpointFilePath, "link_file", activeLink) + + if checkpointFilePath == "" { + if err := os.Remove(activeLink); err != nil { // try, remove active.dat if present. + log.Errorf("Failed to remove old pointer file: %v", err) + } + return nil + } + + // Atomically try to update the pointer file to the most recent data file. + // We 'simulate' the atomic update by create the temporary active.dat.new file, + // which we rename to active.dat. If active.dat.tmp exists we remove it. + if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) { + log.Errorf("Failed to remove old temporary active.dat.tmp file: %v", err) + return err + } + if err := ioutil.WriteFile(tmpLink, []byte(checkpointFilePath), 0600); err != nil { + log.Errorf("Failed to write temporary pointer file: %v", err) + return err + } + if err := os.Rename(tmpLink, activeLink); err != nil { + log.Errorf("Failed to replace link file: %v", err) + return err + } + + trySyncPath(homePath) + return nil +} + +// removeOldDataFiles sorts the data files by their update sequence number and +// finally deletes all but the newest file from the storage directory. +func (s *diskstore) removeOldDataFiles() { + for i := range s.oldDataFiles { + path := s.oldDataFiles[i].path + err := os.Remove(path) + if err != nil && !os.IsNotExist(err) { + s.log.With("file", path).Errorf("Failed to delete old data file: %v", err) + s.oldDataFiles = s.oldDataFiles[i:] + return + } + } + s.oldDataFiles = nil +} + +// listDataFiles returns a sorted list of data files with txid per file. +// The list is sorted by txid, in ascending order (taking integer overflows +// into account). +func listDataFiles(home string) ([]dataFileInfo, error) { + files, err := filepath.Glob(filepath.Join(home, "*.json")) + if err != nil { + return nil, err + } + + var infos []dataFileInfo + for i := range files { + info, err := os.Lstat(files[i]) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + continue + } + + name := filepath.Base(files[i]) + name = name[:len(name)-5] // remove '.json' extension + + id, err := strconv.ParseUint(name, 10, 64) + if err == nil { + infos = append(infos, dataFileInfo{ + path: files[i], + txid: id, + }) + } + } + + // empty or most recent snapshot was complete (old data file has been deleted) + if len(infos) <= 1 { + return infos, nil + } + + // sort files by transaction ID + sortDataFileInfos(infos) + return infos, nil +} + +// sortDataFileInfos sorts the slice by the files txid. +func sortDataFileInfos(infos []dataFileInfo) { + sort.Slice(infos, func(i, j int) bool { + return isTxIDLessEqual(infos[i].txid, infos[j].txid) + }) +} + +// loadDataFile create a new hashtable with all key/value pairs found. +func loadDataFile(path string, tbl map[string]entry) error { + if path == "" { + return nil + } + + err := readDataFile(path, func(key string, state mapstr.M) { + tbl[key] = entry{value: state} + }) + return err +} + +var ErrCorruptStore = errors.New("corrupted data file") + +func readDataFile(path string, fn func(string, mapstr.M)) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + var states []map[string]interface{} + dec := json.NewDecoder(f) + if err := dec.Decode(&states); err != nil { + return fmt.Errorf("%w: %v", ErrCorruptStore, err) + } + + for _, state := range states { + keyRaw := state[keyField] + key, ok := keyRaw.(string) + if !ok { + continue + } + + delete(state, keyField) + fn(key, mapstr.M(state)) + } + + return nil +} + +// loadLogFile applies all recorded transaction to an already initialized +// memStore. +// The txid is the transaction ID of the last known valid data file. +// Transactions older then txid will be ignored. +// loadLogFile returns the last committed txid in logTxid and the total number +// of operations in logCount. +func loadLogFile( + store *memstore, + txid uint64, + home string, +) (logTxid uint64, entries uint, err error) { + err = readLogFile(home, func(rawOp op, id uint64) error { + // ignore old entries in case the log file truncation was not executed between a beat restart. + if isTxIDLessEqual(id, txid) { + return nil + } + + if id != txid+1 { + return errTxIDInvalid + } + txid = id + + switch op := rawOp.(type) { + case *opSet: + entries++ + store.Set(op.K, op.V) + case *opRemove: + entries++ + store.Remove(op.K) + } + return nil + }) + if err != nil { + return txid, entries, err + } + + return txid, entries, err +} + +// readLogFile iterates all operations found in the transaction log. +func readLogFile(home string, fn func(op, uint64) error) error { + path := filepath.Join(home, logFileName) + f, err := os.Open(path) + if os.IsNotExist(err) { + return nil + } + defer f.Close() + + dec := json.NewDecoder(f) + for dec.More() { + var act logAction + if err := dec.Decode(&act); err != nil { + return err + } + + var op op + switch act.Op { + case opValSet: + op = &opSet{} + case opValRemove: + op = &opRemove{} + } + + if err := dec.Decode(op); err != nil { + return err + } + + if err := fn(op, act.ID); err != nil { + return err + } + } + + return nil +} + +func checkMeta(meta storeMeta) error { + if meta.Version != storeVersion { + return fmt.Errorf("store version %v not supported", meta.Version) + } + + return nil +} + +func writeMetaFile(home string, mode os.FileMode) error { + path := filepath.Join(home, metaFileName) + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, mode) + if err != nil { + return err + } + + ok := false + defer cleanup.IfNot(&ok, func() { + f.Close() + }) + + enc := newJSONEncoder(&ensureWriter{f}) + err = enc.Encode(storeMeta{ + Version: storeVersion, + }) + if err != nil { + return err + } + + if err := syncFile(f); err != nil { + return err + } + + ok = true + if err := f.Close(); err != nil { + return err + } + + trySyncPath(home) + return nil +} + +func readMetaFile(home string) (storeMeta, error) { + var meta storeMeta + path := filepath.Join(home, metaFileName) + + f, err := os.Open(path) + if err != nil { + return meta, err + } + defer f.Close() + + dec := json.NewDecoder(f) + if err := dec.Decode(&meta); err != nil { + return meta, fmt.Errorf("can not read store meta file: %w", err) + } + + return meta, nil +} + +// isTxIDLessEqual compares two IDs by checking that their distance is < 2^63. +// It always returns true if +// - a == b +// - a < b (mod 2^63) +// - b > a after an integer rollover that is still within the distance of <2^63-1 +func isTxIDLessEqual(a, b uint64) bool { + return int64(a-b) <= 0 +} diff --git a/pkg/statestore/backend/memlog/doc.go b/pkg/statestore/backend/memlog/doc.go new file mode 100644 index 0000000..514098b --- /dev/null +++ b/pkg/statestore/backend/memlog/doc.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package memlog implements the memlog statestore backend. +// The store provided by memlog is a in-memory key-value store +// that logs all operations to an append only log file. +// Once the log file is considered full the store executes a checkpoint +// operation. The checkpoint operation serializes all state to a data file. +// +// The memory store in memlog holds all key-value pairs in a hashtable, with +// value represented by map[string]interface{}. As the store must be 'valid' +// based on the state of the last update operations (Set, Remove), it +// guarantees that no references into data structures passed via Set are held. +// Instead structured data is serialized/deserialized into a +// map[string]interface{}. The serialized states contain only primitive types +// like intX, uintX, float, bool, string, slices, or map[string]interface{} +// itself. As a side effect this also guarantees that the internal can always +// be serialized to disk after updating the in memory representation. +// +// On disk we have a meta file, an update log file, data files, and an active +// marker file in the store directory. +// +// The meta file only contains the store version number. +// +// Normally all operations that update the store in memory state are appended +// to the update log file. +// The file stores all entries in JSON format. Each entry starts with an action +// entry, followed by an data entry. +// The action entry has the schema: `{"op": "", id: }`. Supporter +// operations are 'set' or 'remove'. The `id` contains a sequential counter +// that must always be increased by 1. +// The data entry for the 'set' operation has the format: `{"K": "", "V": { ... }}`. +// The data entry for the 'remove' operation has the format: `{"K": ""}`. +// Updates to the log file are not synced to disk. Having all updates available +// between restarts/crashes also depends on the capabilities of the operation +// system and file system. When opening the store we read up until it is +// possible, reconstructing a last known valid state the beat can continue +// from. This can lead to duplicates if the machine/filesystem has had an +// outage with state not yet fully synchronised to disk. Ordinary restarts +// should not lead to any problems. +// If any error is encountered when reading the log file, the next updates to the store +// will trigger a checkpoint operation and reset the log file. +// +// The store might contain multiple data files, but only the last data file is +// supposed to be valid. Older data files will continuously tried to be cleaned up +// on checkpoint operations. +// The data files filenames do include the change sequence number. Which allows +// us to sort them by name. The checkpoint operation of memlog, writes the full +// state into a new data file, that consists of an JSON array with all known +// key-value pairs. Each JSON object in the array consists of the value +// object, with memlog private fields added. Private fields start with `_`. At +// the moment the only private field is `_key`, which is used to identify the +// key-value pair. +// NOTE: Creating a new file guarantees that Beats can progress when creating a +// new checkpoint file. Some filesystems tend to block the +// delete/replace operation when the file is accessed by another process +// (e.g. common problem with AV Scanners on Windows). By creating a new +// file we circumvent this problem. Failures in deleting old files is +// ok, and we will try to delete old data files again in the future. +// +// The active marker file is not really used by the store. It is written for +// debugging purposes and contains the filepath of the last written data file +// that is supposed to be valid. +// +// When opening the store we first validate the meta file and read the "last" +// data file into the in-memory hashtable. Older data files are ignored. The +// filename with the update sequence number is used to sort data files. +// NOTE: the active marker file is not used, as the checkpoint operation is +// supposed to be an atomic operation that is finalized once the data +// file is moved to its correct location. +// +// After loading the data file we loop over all operations in the log file. +// Operations with a smaller sequence number are ignored when iterating the log +// file. If any subsequent entries in the log file have a sequence number difference != +// 1, we assume the log file to be corrupted and stop the loop. All processing +// continues from the last known accumulated state. +// +// When closing the store we make a last attempt at fsyncing the log file (just +// in case), close the log file and clear all in memory state. +// +// The store provided by memlog is threadsafe and uses a RWMutex. We allow only +// one active writer, but multiple concurrent readers. +package memlog diff --git a/pkg/statestore/backend/memlog/error.go b/pkg/statestore/backend/memlog/error.go new file mode 100644 index 0000000..29f665b --- /dev/null +++ b/pkg/statestore/backend/memlog/error.go @@ -0,0 +1,27 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import "errors" + +var ( + errRegClosed = errors.New("registry has been closed") + errLogInvalid = errors.New("can not add operation to log file, a checkpoint is required") + errTxIDInvalid = errors.New("invalid update sequence number") + errKeyUnknown = errors.New("key unknown") +) diff --git a/pkg/statestore/backend/memlog/json.go b/pkg/statestore/backend/memlog/json.go new file mode 100644 index 0000000..33bd0e0 --- /dev/null +++ b/pkg/statestore/backend/memlog/json.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "io" + + "github.com/elastic/go-structform/gotype" + "github.com/elastic/go-structform/json" +) + +type jsonEncoder struct { + out io.Writer + folder *gotype.Iterator +} + +func newJSONEncoder(out io.Writer) *jsonEncoder { + e := &jsonEncoder{out: out} + e.reset() + return e +} + +func (e *jsonEncoder) reset() { + visitor := json.NewVisitor(e.out) + visitor.SetEscapeHTML(false) + + var err error + + // create new encoder with custom time.Time encoding + e.folder, err = gotype.NewIterator(visitor) + if err != nil { + panic(err) + } +} + +func (e *jsonEncoder) Encode(v interface{}) error { + return e.folder.Fold(v) +} diff --git a/pkg/statestore/backend/memlog/memlog.go b/pkg/statestore/backend/memlog/memlog.go new file mode 100644 index 0000000..e7a61bd --- /dev/null +++ b/pkg/statestore/backend/memlog/memlog.go @@ -0,0 +1,134 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "os" + "path/filepath" + "sync" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" + "github.com/elastic/elastic-agent-libs/logp" +) + +// Registry configures access to memlog based stores. +type Registry struct { + log *logp.Logger + + mu sync.Mutex + active bool + + settings Settings + + wg sync.WaitGroup +} + +// Settings configures a new Registry. +type Settings struct { + // Registry root directory. Stores will be single sub-directories. + Root string + + // FileMode is used to configure the file mode for new files generated by the + // regisry. File mode 0600 will be used if this field is not set. + FileMode os.FileMode + + // BufferSize configures the IO buffer size when accessing the underlying + // storage files. Defaults to 4096 if not set. + BufferSize uint + + // Checkpoint predicate that can trigger a registry file rotation. If not + // configured, memlog will automatically trigger a checkpoint every 10MB. + Checkpoint CheckpointPredicate + + // If set memlog will not check the version of the meta file. + IgnoreVersionCheck bool +} + +// CheckpointPredicate is the type for configurable checkpoint checks. +// The store executes a checkpoint operation when the predicate returns true. +type CheckpointPredicate func(fileSize uint64) bool + +const defaultFileMode os.FileMode = 0600 + +const defaultBufferSize = 4 * 1024 + +func defaultCheckpoint(filesize uint64) bool { + const limit = 10 * 1 << 20 // set rotation limit to 10MB by default + return filesize >= limit +} + +// New configures a memlog Registry that can be used to open stores. +func New(log *logp.Logger, settings Settings) (*Registry, error) { + if settings.FileMode == 0 { + settings.FileMode = defaultFileMode + } + if settings.Checkpoint == nil { + settings.Checkpoint = defaultCheckpoint + } + if settings.BufferSize == 0 { + settings.BufferSize = defaultBufferSize + } + + root, err := filepath.Abs(settings.Root) + if err != nil { + return nil, err + } + + settings.Root = root + return &Registry{ + log: log, + active: true, + settings: settings, + }, nil +} + +// Access creates or opens a new store. A new sub-directory for the store if +// created, if the store does not exist. +// Returns an error is any file access fails. +func (r *Registry) Access(name string) (backend.Store, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if !r.active { + return nil, errRegClosed + } + + logger := r.log.With("store", name) + + home := filepath.Join(r.settings.Root, name) + fileMode := r.settings.FileMode + bufSz := r.settings.BufferSize + store, err := openStore(logger, home, fileMode, bufSz, r.settings.IgnoreVersionCheck, r.settings.Checkpoint) + if err != nil { + return nil, err + } + + return store, nil +} + +// Close closes the registry. No new store can be accessed during close. +// Close blocks until all stores have been closed. +func (r *Registry) Close() error { + r.mu.Lock() + r.active = false + r.mu.Unlock() + + // block until all stores have been closed + r.wg.Wait() + return nil +} diff --git a/pkg/statestore/backend/memlog/memlog.test b/pkg/statestore/backend/memlog/memlog.test new file mode 100755 index 0000000..0b10b30 Binary files /dev/null and b/pkg/statestore/backend/memlog/memlog.test differ diff --git a/pkg/statestore/backend/memlog/memlog_test.go b/pkg/statestore/backend/memlog/memlog_test.go new file mode 100644 index 0000000..05585a8 --- /dev/null +++ b/pkg/statestore/backend/memlog/memlog_test.go @@ -0,0 +1,256 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "encoding/json" + "io" + "io/ioutil" + "math" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore" + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" + "github.com/elastic/elastic-agent-inputs/pkg/statestore/internal/storecompliance" + "github.com/elastic/elastic-agent-libs/logp" +) + +func init() { + err := logp.DevelopmentSetup() + if err != nil { + panic(err) + } +} + +func TestCompliance_Default(t *testing.T) { + storecompliance.TestBackendCompliance(t, func(testPath string) (backend.Registry, error) { + return New(logp.NewLogger("test"), Settings{Root: testPath}) + }) +} + +func TestCompliance_AlwaysCheckpoint(t *testing.T) { + storecompliance.TestBackendCompliance(t, func(testPath string) (backend.Registry, error) { + return New(logp.NewLogger("test"), Settings{ + Root: testPath, + Checkpoint: func(filesize uint64) bool { + return true + }, + }) + }) +} + +func TestLoadVersion1(t *testing.T) { + dataHome := "testdata/1" + + list, err := ioutil.ReadDir(dataHome) + if err != nil { + t.Fatal(err) + } + + cases := list[:0] + for _, info := range list { + if info.IsDir() { + cases = append(cases, info) + } + } + + for _, info := range cases { + name := filepath.Base(info.Name()) + t.Run(name, func(t *testing.T) { + testLoadVersion1Case(t, filepath.Join(dataHome, info.Name())) + }) + } +} + +func testLoadVersion1Case(t *testing.T, dataPath string) { + path, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create temporary test directory: %v", err) + } + defer os.RemoveAll(path) + + t.Logf("Test tmp dir: %v", path) + + if err := copyPath(path, dataPath); err != nil { + t.Fatalf("Failed to copy test file to the temporary directory: %v", err) + } + + // load expected test results + raw, err := ioutil.ReadFile(filepath.Join(path, "expected.json")) + if err != nil { + t.Fatalf("Failed to load expected.json: %v", err) + } + + expected := struct { + Txid uint64 + Datafile string + Entries map[string]interface{} + }{} + if err := json.Unmarshal(raw, &expected); err != nil { + t.Fatalf("Failed to parse expected.json: %v", err) + } + + // load store: + store, err := openStore(logp.NewLogger("test"), path, 0660, 4096, true, func(_ uint64) bool { + return false + }) + if err != nil { + t.Fatalf("Failed to load test store: %v", err) + } + defer store.Close() + + disk := store.disk + disk.removeOldDataFiles() + + // validate store: + assert.Equal(t, expected.Txid, disk.nextTxID-1) + if expected.Datafile != "" { + assert.Equal(t, filepath.Join(path, expected.Datafile), disk.activeDataFile.path) + } + + // check all keys in expected are known and do match stored values: + func() { + for key, val := range expected.Entries { + var tmp interface{} + err := store.Get(key, &tmp) + require.NoError(t, err, "error reading entry (key=%v)", key) + + assert.Equal(t, val, tmp, "failed when checking key '%s'", key) + } + }() + + // check store does not contain any additional keys + func() { + err = store.Each(func(key string, val statestore.ValueDecoder) (bool, error) { + _, exists := expected.Entries[key] + if !exists { + t.Errorf("unexpected key: %s", key) + } + return true, nil + }) + assert.NoError(t, err) + }() +} + +func TestTxIDLessEqual(t *testing.T) { + cases := map[string]struct { + a, b uint64 + want bool + }{ + "is equal": {10, 10, true}, + "is less": {8, 9, true}, + "is bigger": {9, 8, false}, + "is less 0 with integer overflow": { + math.MaxUint64 - 2, 0, true, + }, + "is less random value with integer overflow": { + math.MaxUint64 - 2, 10, true, + }, + "is less with large ids": { + math.MaxUint64 - 10, math.MaxUint64 - 9, true, + }, + "is bigger with large ids": { + math.MaxUint64 - 9, math.MaxUint64 - 10, false, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + got := isTxIDLessEqual(test.a, test.b) + if got != test.want { + t.Fatalf("%v <= %v should be %v", test.a, test.b, test.want) + } + }) + } +} + +func copyPath(to, from string) error { + info, err := os.Stat(from) + if err != nil { + return err + } + + if info.IsDir() { + return copyDir(to, from) + } + if info.Mode().IsRegular() { + return copyFile(to, from) + } + + // ignore other file types + return nil +} + +func copyDir(to, from string) error { + if !isDir(to) { + info, err := os.Stat(from) + if err != nil { + return err + } + + if err := os.MkdirAll(to, info.Mode()); err != nil { + return err + } + } + + list, err := ioutil.ReadDir(from) + if err != nil { + return err + } + + for _, file := range list { + name := file.Name() + err := copyPath(filepath.Join(to, name), filepath.Join(from, name)) + if err != nil { + return err + } + } + return nil +} + +func copyFile(to, from string) error { + in, err := os.Open(from) + if err != nil { + return err + } + defer in.Close() + + info, err := in.Stat() + if err != nil { + return err + } + + out, err := os.OpenFile(to, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode()) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/pkg/statestore/backend/memlog/op.go b/pkg/statestore/backend/memlog/op.go new file mode 100644 index 0000000..e4bf91d --- /dev/null +++ b/pkg/statestore/backend/memlog/op.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import "github.com/elastic/elastic-agent-libs/mapstr" + +type ( + op interface { + name() string + } + + // opSet encodes the 'Set' operations in the update log. + opSet struct { + K string + V mapstr.M + } + + // opRemove encodes the 'Remove' operation in the update log. + opRemove struct { + K string + } +) + +// operation type names +const ( + opValSet = "set" + opValRemove = "remove" +) + +func (*opSet) name() string { return opValSet } +func (*opRemove) name() string { return opValRemove } diff --git a/pkg/statestore/backend/memlog/store.go b/pkg/statestore/backend/memlog/store.go new file mode 100644 index 0000000..e2de297 --- /dev/null +++ b/pkg/statestore/backend/memlog/store.go @@ -0,0 +1,281 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/transform/typeconv" +) + +// store implements an actual memlog based store. +// It holds all key value pairs in memory in a memstore struct. +// All changes to the memstore are logged to the diskstore. +// The store execute a checkpoint operation if the checkpoint predicate +// triggers the operation, or if some error in the update log file has been +// detected by the diskstore. +// +// The store allows only one writer, but multiple concurrent readers. +type store struct { + lock sync.RWMutex + disk *diskstore + mem memstore +} + +// memstore is the in memory key value store +type memstore struct { + table map[string]entry +} + +type entry struct { + value map[string]interface{} +} + +// openStore opens a store from the home path. +// The directory and intermediate directories will be created if it does not exist. +// The open routine loads the full key-value store into memory by first reading the data file and finally applying all outstanding updates +// from the update log file. +// If an error in in the log file is detected, the store opening routine continues from the last known valid state and will trigger a checkpoint +// operation on subsequent writes, also truncating the log file. +// Old data files are scheduled for deletion later. +func openStore(log *logp.Logger, home string, mode os.FileMode, bufSz uint, ignoreVersionCheck bool, checkpoint CheckpointPredicate) (*store, error) { + fi, err := os.Stat(home) + if os.IsNotExist(err) { + err = os.MkdirAll(home, os.ModeDir|0770) + if err != nil { + return nil, err + } + + err = writeMetaFile(home, mode) + if err != nil { + return nil, err + } + } else if !fi.Mode().IsDir() { + return nil, fmt.Errorf("'%v' is not a directory", home) + } else { + if err := pathEnsurePermissions(filepath.Join(home, metaFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update meta file permissions: %w", err) + } + } + + if !ignoreVersionCheck { + meta, err := readMetaFile(home) + if err != nil { + return nil, err + } + if err := checkMeta(meta); err != nil { + return nil, err + } + } + + if err := pathEnsurePermissions(filepath.Join(home, activeDataFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update active file permissions: %w", err) + } + + dataFiles, err := listDataFiles(home) + if err != nil { + return nil, err + } + for _, df := range dataFiles { + if err := pathEnsurePermissions(df.path, mode); err != nil { + return nil, fmt.Errorf("failed to update data file permissions: %w", err) + } + } + if err := pathEnsurePermissions(filepath.Join(home, logFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update log file permissions: %w", err) + } + + tbl := map[string]entry{} + var txid uint64 + if L := len(dataFiles); L > 0 { + active := dataFiles[L-1] + txid = active.txid + if err := loadDataFile(active.path, tbl); err != nil { + if errors.Is(err, ErrCorruptStore) { + corruptFilePath := active.path + ".corrupted" + err := os.Rename(active.path, corruptFilePath) + if err != nil { + logp.Debug("Failed to backup corrupt data file '%s': %+v", active.path, err) + } + logp.Warn("Data file is corrupt. It has been renamed to %s. Attempting to restore partial state from log file.", corruptFilePath) + } else { + return nil, err + } + } else { + logp.Info("Loading data file of '%v' succeeded. Active transaction id=%v", home, txid) + } + } + + var entries uint + memstore := memstore{tbl} + txid, entries, err = loadLogFile(&memstore, txid, home) + logp.Info("Finished loading transaction log file for '%v'. Active transaction id=%v", home, txid) + + if err != nil { + // Error indicates the log file was incomplete or corrupted. + // Anyways, we already have the table in a valid state and will + // continue opening the store from here. + logp.Warn("Incomplete or corrupted log file in %v. Continue with last known complete and consistent state. Reason: %v", home, err) + } + + diskstore, err := newDiskStore(log, home, dataFiles, txid, mode, entries, err != nil, bufSz, checkpoint) + if err != nil { + return nil, err + } + + return &store{ + disk: diskstore, + mem: memstore, + }, nil +} + +// Close closes access to the update log file and clears the in memory key +// value store. Access to the store after close can lead to a panic. +func (s *store) Close() error { + s.lock.Lock() + defer s.lock.Unlock() + s.mem = memstore{} + return s.disk.Close() +} + +// Has checks if the key is known. The in memory store does not report any +// errors. +func (s *store) Has(key string) (bool, error) { + s.lock.RLock() + defer s.lock.RUnlock() + return s.mem.Has(key), nil +} + +// Get retrieves and decodes the key-value pair into to. +func (s *store) Get(key string, to interface{}) error { + s.lock.RLock() + defer s.lock.RUnlock() + + dec := s.mem.Get(key) + if dec == nil { + return errKeyUnknown + } + return dec.Decode(to) +} + +// Set inserts or overwrites a key-value pair. +// If encoding was successful the in-memory state will be updated and a +// set-operation is logged to the diskstore. +func (s *store) Set(key string, value interface{}) error { + var tmp mapstr.M + if err := typeconv.Convert(&tmp, value); err != nil { + return err + } + + s.lock.Lock() + defer s.lock.Unlock() + + s.mem.Set(key, tmp) + return s.logOperation(&opSet{K: key, V: tmp}) +} + +// Remove removes a key from the in memory store and logs a remove operation to +// the diskstore. The operation does not check if the key exists. +func (s *store) Remove(key string) error { + s.lock.Lock() + defer s.lock.Unlock() + + s.mem.Remove(key) + return s.logOperation(&opRemove{K: key}) +} + +// Checkpoint triggers a state checkpoint operation. All state will be written +// to a new transaction data file and fsync'ed. The log file will be reset after +// a successful write. +func (s *store) Checkpoint() error { + s.lock.Lock() + defer s.lock.Unlock() + + return s.disk.WriteCheckpoint(s.mem.table) +} + +// lopOperation ensures that the diskstore reflects the recent changes to the +// in memory store by either triggering a checkpoint operations or adding the +// operation type to the update log file. +func (s *store) logOperation(op op) error { + if s.disk.mustCheckpoint() { + err := s.disk.WriteCheckpoint(s.mem.table) + if err != nil { + // if writing the new checkpoint file failed we try to fallback to + // appending the log operation. + // idea: make append configurable and retry checkpointing with backoff. + _ = s.disk.LogOperation(op) + } + + return err + } + + return s.disk.LogOperation(op) +} + +// Each iterates over all key-value pairs in the store. +func (s *store) Each(fn func(string, backend.ValueDecoder) (bool, error)) error { + s.lock.RLock() + defer s.lock.RUnlock() + + for k, entry := range s.mem.table { + cont, err := fn(k, entry) + if !cont || err != nil { + return err + } + } + + return nil +} + +func (m *memstore) Has(key string) bool { + _, exists := m.table[key] + return exists +} + +func (m *memstore) Get(key string) backend.ValueDecoder { + entry, exists := m.table[key] + if !exists { + return nil + } + return entry +} + +func (m *memstore) Set(key string, value mapstr.M) { + m.table[key] = entry{value: value} +} + +func (m *memstore) Remove(key string) bool { + _, exists := m.table[key] + if !exists { + return false + } + delete(m.table, key) + return true +} + +func (e entry) Decode(to interface{}) error { + return typeconv.Convert(to, e.value) +} diff --git a/pkg/statestore/backend/memlog/testdata/1/Readme.md b/pkg/statestore/backend/memlog/testdata/1/Readme.md new file mode 100644 index 0000000..7532ab9 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/Readme.md @@ -0,0 +1 @@ +Sample disk stores version 1 diff --git a/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json new file mode 100644 index 0000000..90bd45d --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md new file mode 100644 index 0000000..9c7b458 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md @@ -0,0 +1,2 @@ +Store file with transaction ID and log with with transaction id 3. +The data file 2.json is missing, which leads to all entries in the log file to be ignored. diff --git a/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json new file mode 100644 index 0000000..f691496 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json new file mode 100644 index 0000000..5b5d8f4 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json @@ -0,0 +1,6 @@ +{"op": "set", "id": 3} +{"K":"key3","V":{"a":3}} +{"op": "set", "id": 3} +{"K":"key3","V":{"a":3}} +{"op": "set", "id": 3} +{"K":"key5","V":{"a":5}} diff --git a/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json new file mode 100644 index 0000000..bcb050a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/1.json b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/1.json new file mode 100644 index 0000000..3277e34 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1",a":1}, +{"_key":"key2","a":2} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/Readme.md b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/Readme.md new file mode 100644 index 0000000..dd15265 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/Readme.md @@ -0,0 +1 @@ +Store with corrupt data file and valid log file with logged updates. diff --git a/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/expected.json b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/expected.json new file mode 100644 index 0000000..ade004a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 4, + "datafile": "1.json", + "entries": { + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/log.json b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/log.json new file mode 100644 index 0000000..c7edd4f --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/log.json @@ -0,0 +1,6 @@ +{"op":"set", "id": 2} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 3} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 4} +{"K":"key5","V":{"a":5}} diff --git a/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/meta.json b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/meta.json new file mode 100644 index 0000000..bcb050a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/corrupt_datafile/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/data_and_log/1.json b/pkg/statestore/backend/memlog/testdata/1/data_and_log/1.json new file mode 100644 index 0000000..90bd45d --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/data_and_log/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/data_and_log/Readme.md b/pkg/statestore/backend/memlog/testdata/1/data_and_log/Readme.md new file mode 100644 index 0000000..5272ae5 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/data_and_log/Readme.md @@ -0,0 +1 @@ +Store with valid data file and valid log file with logged updates. diff --git a/pkg/statestore/backend/memlog/testdata/1/data_and_log/expected.json b/pkg/statestore/backend/memlog/testdata/1/data_and_log/expected.json new file mode 100644 index 0000000..c7a3023 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/data_and_log/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 4, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/data_and_log/log.json b/pkg/statestore/backend/memlog/testdata/1/data_and_log/log.json new file mode 100644 index 0000000..c7edd4f --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/data_and_log/log.json @@ -0,0 +1,6 @@ +{"op":"set", "id": 2} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 3} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 4} +{"K":"key5","V":{"a":5}} diff --git a/pkg/statestore/backend/memlog/testdata/1/data_and_log/meta.json b/pkg/statestore/backend/memlog/testdata/1/data_and_log/meta.json new file mode 100644 index 0000000..bcb050a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/data_and_log/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/datafile_only/1.json b/pkg/statestore/backend/memlog/testdata/1/datafile_only/1.json new file mode 100644 index 0000000..0460d15 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/datafile_only/1.json @@ -0,0 +1,8 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2}, +{"_key":"key3","a":3}, +{"_key":"key4","a":4}, +{"_key":"key5","a":5} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/datafile_only/Readme.md b/pkg/statestore/backend/memlog/testdata/1/datafile_only/Readme.md new file mode 100644 index 0000000..d5cc1f5 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/datafile_only/Readme.md @@ -0,0 +1,2 @@ +Valid store without log.json. All entries are read from 1.json only. + diff --git a/pkg/statestore/backend/memlog/testdata/1/datafile_only/expected.json b/pkg/statestore/backend/memlog/testdata/1/datafile_only/expected.json new file mode 100644 index 0000000..8c643ce --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/datafile_only/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/datafile_only/meta.json b/pkg/statestore/backend/memlog/testdata/1/datafile_only/meta.json new file mode 100644 index 0000000..bcb050a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/datafile_only/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/incomplete_op/1.json b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/1.json new file mode 100644 index 0000000..90bd45d --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md new file mode 100644 index 0000000..a3df15b --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md @@ -0,0 +1,2 @@ +Valid data file with incomplete log.json. The last entry in the log file has +the data part missing, which will fail the log file to be read. diff --git a/pkg/statestore/backend/memlog/testdata/1/incomplete_op/expected.json b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/expected.json new file mode 100644 index 0000000..f691496 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/incomplete_op/log.json b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/log.json new file mode 100644 index 0000000..5046231 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/log.json @@ -0,0 +1 @@ +{"op":"set", "id": 2} diff --git a/pkg/statestore/backend/memlog/testdata/1/incomplete_op/meta.json b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/meta.json new file mode 100644 index 0000000..bcb050a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/incomplete_op/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md new file mode 100644 index 0000000..0c8ac8f --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md @@ -0,0 +1,3 @@ +Checkpoint file does not exist yet. All entries are read from the log file. +Contents is missing with the last operation in the log file triggereing a parse error. + diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json new file mode 100644 index 0000000..356fb55 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 6, + "datafile": "", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json new file mode 100644 index 0000000..a0cc0a3 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json @@ -0,0 +1,13 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"set", "id": 4} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 5} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 6} +{"K":"key5","V":{"a":5}} +{"op":"set", diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json new file mode 100644 index 0000000..cd576d0 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json @@ -0,0 +1 @@ +{"version":"1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_only/Readme.md b/pkg/statestore/backend/memlog/testdata/1/logfile_only/Readme.md new file mode 100644 index 0000000..7250420 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_only/Readme.md @@ -0,0 +1,2 @@ +Checkpoint file does not exist yet. All entries are read from the log file. + diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_only/expected.json b/pkg/statestore/backend/memlog/testdata/1/logfile_only/expected.json new file mode 100644 index 0000000..356fb55 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_only/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 6, + "datafile": "", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_only/log.json b/pkg/statestore/backend/memlog/testdata/1/logfile_only/log.json new file mode 100644 index 0000000..f00679f --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_only/log.json @@ -0,0 +1,12 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"set", "id": 4} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 5} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 6} +{"K":"key5","V":{"a":5}} diff --git a/pkg/statestore/backend/memlog/testdata/1/logfile_only/meta.json b/pkg/statestore/backend/memlog/testdata/1/logfile_only/meta.json new file mode 100644 index 0000000..cd576d0 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/logfile_only/meta.json @@ -0,0 +1 @@ +{"version":"1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/old_datafiles/1.json b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/1.json new file mode 100644 index 0000000..0460d15 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/1.json @@ -0,0 +1,8 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2}, +{"_key":"key3","a":3}, +{"_key":"key4","a":4}, +{"_key":"key5","a":5} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/old_datafiles/2.json b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/2.json new file mode 100644 index 0000000..c3d3f12 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/2.json @@ -0,0 +1,3 @@ +[ +{"_key":"key0","a":0} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/old_datafiles/3.json b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/3.json new file mode 100644 index 0000000..9ac6abf --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/3.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":30}, +{"_key":"key1","a":31}, +{"_key":"key2","a":32} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md new file mode 100644 index 0000000..cdea115 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md @@ -0,0 +1,2 @@ +Store with old data files that failed to be cleaned up. Only entries from data file 3.json should be loaded. + diff --git a/pkg/statestore/backend/memlog/testdata/1/old_datafiles/expected.json b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/expected.json new file mode 100644 index 0000000..f671203 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 3, + "datafile": "3.json", + "entries": { + "key0": {"a": 30}, + "key1": {"a": 31}, + "key2": {"a": 32} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/old_datafiles/meta.json b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/meta.json new file mode 100644 index 0000000..bcb050a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_datafiles/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json new file mode 100644 index 0000000..90bd45d --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md new file mode 100644 index 0000000..9d99a9a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md @@ -0,0 +1,3 @@ +Due to restart the log file was not truncated after the last checkpoint was +written. Update with ID 0 adds the removed key0 again to the store. All entries +in log.json should be ignored. diff --git a/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json new file mode 100644 index 0000000..4e95470 --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 5, + "datafile": "5.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json new file mode 100644 index 0000000..426ce6a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json @@ -0,0 +1,8 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"remove", "id": 4} +{"K":"key0"} diff --git a/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json new file mode 100644 index 0000000..bcb050a --- /dev/null +++ b/pkg/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/pkg/statestore/backend/memlog/util.go b/pkg/statestore/backend/memlog/util.go new file mode 100644 index 0000000..d788c4e --- /dev/null +++ b/pkg/statestore/backend/memlog/util.go @@ -0,0 +1,122 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "errors" + "io" + "os" + "runtime" + "syscall" +) + +// ensureWriter writes the buffer to the underlying writer +// for as long as w returns a retryable error (e.g. EAGAIN) +// or the input buffer has been exhausted. +// +// XXX: this code was written and tested with go1.13 and go1.14, which does not +// handled EINTR. Some users report EINTR getting triggered more often in +// go1.14 due to changes in the signal handling for implementing +// preemption. +// In future versions EINTR will be handled by go for us. +// See: https://github.com/golang/go/issues/38033 +type ensureWriter struct { + w io.Writer +} + +// countWriter keeps track of the amount of bytes written over time. +type countWriter struct { + n uint64 + w io.Writer +} + +func (c *countWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += uint64(n) + return n, err +} + +func (e *ensureWriter) Write(p []byte) (int, error) { + var N int + for len(p) > 0 { + n, err := e.w.Write(p) + N, p = N+n, p[n:] + if err != nil && !isRetryErr(err) { + return N, err + } + } + return N, nil +} + +func isRetryErr(err error) bool { + return errors.Is(err, syscall.EINTR) || errors.Is(err, syscall.EAGAIN) +} + +// trySyncPath provides a best-effort fsync on path (directory). The fsync is required by some +// filesystems, so to update the parents directory metadata to actually +// contain the new file being rotated in. +func trySyncPath(path string) { + f, err := os.Open(path) + if err != nil { + return // ignore error, sync on dir must not be necessarily supported by the FS + } + defer f.Close() + _ = syncFile(f) +} + +// pathEnsurePermissions checks if the file permissions for the given file match wantPerm. +// The permissions are updated using chmod if needed. +// No file will be created if the file does not yet exist. +func pathEnsurePermissions(path string, wantPerm os.FileMode) error { + f, err := os.OpenFile(path, os.O_RDWR, wantPerm) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + defer f.Close() + return fileEnsurePermissions(f, wantPerm) +} + +// fileEnsurePermissions checks if the file permissions for the given file +// matches wantPerm. If not fileEnsurePermissions tries to update +// the current permissions via chmod. +// The file is not created or updated if it does not exist. +func fileEnsurePermissions(f *os.File, wantPerm os.FileMode) error { + if runtime.GOOS == "windows" { + return nil + } + + fi, err := f.Stat() + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + wantPerm = wantPerm & os.ModePerm + perm := fi.Mode() & os.ModePerm + if wantPerm == perm { + return nil + } + + return f.Chmod((fi.Mode() &^ os.ModePerm) | wantPerm) +} diff --git a/pkg/statestore/backend/memlog/util_darwin.go b/pkg/statestore/backend/memlog/util_darwin.go new file mode 100644 index 0000000..93f7ccc --- /dev/null +++ b/pkg/statestore/backend/memlog/util_darwin.go @@ -0,0 +1,75 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "errors" + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +var errno0 = syscall.Errno(0) + +// syncFile implements the fsync operation for darwin. On darwin fsync is not +// reliable, instead the fcntl syscall with F_FULLFSYNC must be used. +func syncFile(f *os.File) error { + for { + _, err := unix.FcntlInt(f.Fd(), unix.F_FULLFSYNC, 0) + rootCause := errorRootCause(err) + if err == nil || isIOError(rootCause) { + return err + } + + if isRetryErr(err) { + continue + } + + err = f.Sync() + if isRetryErr(err) { + continue + } + return err + } +} + +func isIOError(err error) bool { + return errors.Is(err, unix.EIO) || + // space/quota + errors.Is(err, unix.ENOSPC) || errors.Is(err, unix.EDQUOT) || errors.Is(err, unix.EFBIG) || + // network + errors.Is(err, unix.ECONNRESET) || errors.Is(err, unix.ENETDOWN) || errors.Is(err, unix.ENETUNREACH) +} + +// normalizeSysError returns the underlying error or nil, if the underlying +// error indicates it is no error. +func errorRootCause(err error) error { + for err != nil { + u, ok := err.(interface{ Unwrap() error }) //nolint:errorlint // keep old behaviour + if !ok { + break + } + err = u.Unwrap() + } + + if err == nil || errors.Is(err, errno0) { + return nil + } + return err +} diff --git a/pkg/statestore/backend/memlog/util_other.go b/pkg/statestore/backend/memlog/util_other.go new file mode 100644 index 0000000..0f39792 --- /dev/null +++ b/pkg/statestore/backend/memlog/util_other.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux || dragonfly || freebsd || netbsd || openbsd || solaris || aix +// +build linux dragonfly freebsd netbsd openbsd solaris aix + +package memlog + +import ( + "os" +) + +// syncFile implements the fsync operation for most *nix systems. +// The call is retried if EINTR or EAGAIN is returned. +func syncFile(f *os.File) error { + // best effort. + for { + err := f.Sync() + if err == nil || !isRetryErr(err) { + return err + } + } +} diff --git a/pkg/statestore/backend/memlog/util_test.go b/pkg/statestore/backend/memlog/util_test.go new file mode 100644 index 0000000..2f34b6e --- /dev/null +++ b/pkg/statestore/backend/memlog/util_test.go @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "errors" + "syscall" + "testing" +) + +// A mock Writer implementation that always returns a configurable +// error on the first write call, to test error handling in ensureWriter. +type mockErrorWriter struct { + errorType error + reportedError bool +} + +func (mew *mockErrorWriter) Write(data []byte) (n int, err error) { + if !mew.reportedError { + mew.reportedError = true + return 0, mew.errorType + } + return len(data), nil +} + +func TestEnsureWriter_RetriableError(t *testing.T) { + // EAGAIN is retriable, ensureWriter.Write should succeed. + errorWriter := &mockErrorWriter{errorType: syscall.EAGAIN} + bytes := []byte{1, 2, 3} + writer := &ensureWriter{errorWriter} + written, err := writer.Write(bytes) + if err != nil { + t.Fatalf("ensureWriter shouldn't propagate retriable errors") + } + if written != len(bytes) { + t.Fatalf("Expected %d bytes written, got %d", len(bytes), written) + } +} + +func TestEnsureWriter_NonRetriableError(t *testing.T) { + // EINVAL is not retriable, ensureWriter.Write should return an error. + errorWriter := &mockErrorWriter{errorType: syscall.EINVAL} + bytes := []byte{1, 2, 3} + writer := &ensureWriter{errorWriter} + written, err := writer.Write(bytes) + if !errors.Is(err, syscall.EINVAL) { + t.Fatalf("ensureWriter should propagate nonretriable errors") + } + if written != 0 { + t.Fatalf("Expected 0 bytes written, got %d", written) + } +} + +func TestEnsureWriter_NoError(t *testing.T) { + // This tests the case where the underlying writer returns with no error, + // but without writing the full buffer. + var bytes []byte = []byte{1, 2, 3} + errorWriter := &mockErrorWriter{errorType: nil} + writer := &ensureWriter{errorWriter} + written, err := writer.Write(bytes) + if err != nil { + t.Fatalf("ensureWriter should only error if the underlying writer does") + } + if written != len(bytes) { + t.Fatalf("Expected %d bytes written, got %d", len(bytes), written) + } +} diff --git a/pkg/statestore/backend/memlog/util_windows.go b/pkg/statestore/backend/memlog/util_windows.go new file mode 100644 index 0000000..ce0d108 --- /dev/null +++ b/pkg/statestore/backend/memlog/util_windows.go @@ -0,0 +1,26 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import "os" + +// syncFile implements the fsync operation for Windows. Internally +// FlushFileBuffers will be used. +func syncFile(f *os.File) error { + return f.Sync() // stdlib already uses FlushFileBuffers, yay +} diff --git a/pkg/statestore/cleanup/cleanup.go b/pkg/statestore/cleanup/cleanup.go new file mode 100644 index 0000000..10e6d64 --- /dev/null +++ b/pkg/statestore/cleanup/cleanup.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package cleanup provides common helpers for common cleanup patterns on defer +// +// Use the helpers with `defer`. For example use IfNot with `defer`, such that +// cleanup functions will be executed if `check` is false, no matter if an +// error has been returned or an panic has occurred. +// +// initOK := false +// defer cleanup.IfNot(&initOK, func() { +// cleanup +// }) +// +// ... // init structures... +// +// initOK = true // notify handler cleanup code must not be executed +package cleanup + +// If will run the cleanup function if the bool value is true. +func If(check *bool, cleanup func()) { + if *check { + cleanup() + } +} + +// IfNot will run the cleanup function if the bool value is false. +func IfNot(check *bool, cleanup func()) { + if !(*check) { + cleanup() + } +} + +// IfPred will run the cleanup function if pred returns true. +func IfPred(pred func() bool, cleanup func()) { + if pred() { + cleanup() + } +} + +// IfNotPred will run the cleanup function if pred returns false. +func IfNotPred(pred func() bool, cleanup func()) { + if !pred() { + cleanup() + } +} + +// WithError returns a cleanup function calling a custom handler if an error occurred. +func WithError(fn func(error), cleanup func() error) func() { + return func() { + if err := cleanup(); err != nil { + fn(err) + } + } +} + +// IgnoreError silently ignores errors in the cleanup function. +func IgnoreError(cleanup func() error) func() { + return func() { _ = cleanup() } +} diff --git a/pkg/statestore/cleanup/cleanup_test.go b/pkg/statestore/cleanup/cleanup_test.go new file mode 100644 index 0000000..b64bae1 --- /dev/null +++ b/pkg/statestore/cleanup/cleanup_test.go @@ -0,0 +1,65 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cleanup_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/cleanup" +) + +func TestIfBool(t *testing.T) { + testcases := []struct { + title string + fn func(*bool, func()) + value bool + cleanup bool + }{ + { + "IfNot runs cleanup", + cleanup.IfNot, false, true, + }, + { + "IfNot does not run cleanup", + cleanup.IfNot, true, false, + }, + { + "If runs cleanup", + cleanup.If, true, true, + }, + { + "If does not run cleanup", + cleanup.If, false, false, + }, + } + + for _, test := range testcases { + test := test + t.Run(test.title, func(t *testing.T) { + executed := false + func() { + v := test.value + defer test.fn(&v, func() { executed = true }) + }() + + assert.Equal(t, test.cleanup, executed) + }) + } +} diff --git a/pkg/statestore/cleanup/multi.go b/pkg/statestore/cleanup/multi.go new file mode 100644 index 0000000..54f831f --- /dev/null +++ b/pkg/statestore/cleanup/multi.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cleanup + +// FailClean keeps track of functions to be executed of FailClean did +// not receive a success signal. +type FailClean struct { + success bool + fns []func() +} + +// Signal sends a success or fail signal to FailClean. +func (f *FailClean) Signal(success bool) { + f.success = success +} + +// Add adds another cleanup handler. The last added handler will be run first. +func (f *FailClean) Add(fn func()) { + f.fns = append(f.fns, fn) +} + +// Cleanup runs all cleanup handlers in reverse order. +func (f *FailClean) Cleanup() { + if f.success { + return + } + + for i := len(f.fns) - 1; i >= 0; i-- { + f.fns[i]() + } +} diff --git a/pkg/statestore/error.go b/pkg/statestore/error.go new file mode 100644 index 0000000..85438ac --- /dev/null +++ b/pkg/statestore/error.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package statestore + +import ( + "errors" + "fmt" +) + +// ErrorAccess indicates that an error occurred when trying to open a Store. +type ErrorAccess struct { + name string + cause error +} + +// Store reports the name of the store that could not been accessed. +func (e *ErrorAccess) Store() string { return e.name } + +// Unwrap returns the cause for the error or nil if the cause is unknown or has +// not been reported by the backend +func (e *ErrorAccess) Unwrap() error { return e.cause } + +// Error creates a descriptive error string. +func (e *ErrorAccess) Error() string { + if e.cause == nil { + return fmt.Sprintf("failed to open store '%v'", e.name) + } + return fmt.Sprintf("failed to open store '%v': %v", e.name, e.cause) +} + +// ErrorClosed indicates that the operation failed because the store has already been closed. +type ErrorClosed struct { + name string + operation string +} + +// Store reports the name of the store that has been closed. +func (e *ErrorClosed) Store() string { return e.name } + +// Operation returns a 'readable' name for the operation that failed to access the closed store. +func (e *ErrorClosed) Operation() string { return e.operation } + +// Error creates a descriptive error string. +func (e *ErrorClosed) Error() string { + return fmt.Sprintf("can not executed %v operation on closed store '%v'", e.operation, e.name) +} + +// ErrorOperation is returned when a generic store operation failed. +type ErrorOperation struct { + name string + operation string + cause error +} + +// Store reports the name of the store. +func (e *ErrorOperation) Store() string { return e.name } + +// Operation returns a 'readable' name for the operation that failed. +func (e *ErrorOperation) Operation() string { return e.operation } + +// Unwrap returns the cause of the failure. +func (e *ErrorOperation) Unwrap() error { return e.cause } + +// Error creates a descriptive error string. +func (e *ErrorOperation) Error() string { + return fmt.Sprintf("failed in %v operation on store '%v': %v", e.operation, e.name, e.cause) +} + +// IsClosed returns true if the cause for an Error is ErrorClosed. +func IsClosed(err error) bool { + var tmp *ErrorClosed + return errors.As(err, &tmp) +} diff --git a/pkg/statestore/internal/storecompliance/reg.go b/pkg/statestore/internal/storecompliance/reg.go new file mode 100644 index 0000000..a769361 --- /dev/null +++ b/pkg/statestore/internal/storecompliance/reg.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package storecompliance + +import ( + "testing" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" +) + +// Registry helper for writing tests. +// The registry uses a testing.T and provides some MustX methods that fail if +// an error occurred. +type Registry struct { + T testing.TB + backend.Registry +} + +// Store helper for writing tests. +// The store needs a reference to the Registry with the current test context. +// The Store provides additional helpers for reopening the store, MustX methods +// that will fail the test if an error has occurred. +type Store struct { + backend.Store + + Registry *Registry + name string +} + +// Access uses the backend Registry to create a new Store. +func (r *Registry) Access(name string) (*Store, error) { + s, err := r.Registry.Access(name) + if err != nil { + return nil, err + } + return &Store{Store: s, Registry: r, name: name}, nil +} + +// MustAccess opens a Store. It fails the test if an error has occurred. +func (r *Registry) MustAccess(name string) *Store { + store, err := r.Access(name) + must(r.T, err, "open store") + return store +} + +// Close closes the testing store. +func (s *Store) Close() { + err := s.Store.Close() + must(s.Registry.T, err, "closing store %q failed", s.name) +} + +// ReopenIf reopens the store if b is true. +func (s *Store) ReopenIf(b bool) { + if b { + s.Reopen() + } +} + +// Reopen reopens the store by closing the backend store and using the registry +// backend to access the same store again. +func (s *Store) Reopen() { + t := s.Registry.T + + s.Close() + if t.Failed() { + t.Fatal("Test already failed") + } + + store, err := s.Registry.Registry.Access(s.name) + must(s.Registry.T, err, "reopen failed") + + s.Store = store +} + +// MustHave fails the test if an error occurred in a call to Has. +func (s *Store) MustHave(key string) bool { + b, err := s.Has(key) + must(s.Registry.T, err, "unexpected error on store/has call") + return b +} + +// MustGet fails the test if an error occurred in a call to Get. +func (s *Store) MustGet(key string, into interface{}) { + err := s.Get(key, into) + must(s.Registry.T, err, "unexpected error on store/get call") +} + +// MustSet fails the test if an error occurred in a call to Set. +func (s *Store) MustSet(key string, from interface{}) { + err := s.Set(key, from) + must(s.Registry.T, err, "unexpected error on store/set call") +} + +// MustRemove fails the test if an error occurred in a call to Remove. +func (s *Store) MustRemove(key string) { + err := s.Store.Remove(key) + must(s.Registry.T, err, "unexpected error remove key") +} diff --git a/pkg/statestore/internal/storecompliance/storecompliance.go b/pkg/statestore/internal/storecompliance/storecompliance.go new file mode 100644 index 0000000..4fa177d --- /dev/null +++ b/pkg/statestore/internal/storecompliance/storecompliance.go @@ -0,0 +1,188 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package storecompliance provides a common test suite that a store +// implementation must succeed in order to be compliant to the beats +// statestore. The Internal tests are used by statestore/storetest and +// statestore/backend/memlog. +// +// The package adds the `-keep` and `-dir ` CLI flags: +// - `-dir `: configure path where to create test folders in (defaults +// to OS specific temporary directory) +// - `-keep`: The test directories will not be deleted after a test has +// finished. The test directory is added to the test logs. +// +package storecompliance + +import ( + "errors" + "flag" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" +) + +// BackendFactory is used by TestBackendCompliance to create +// store instances for testing. Each store will be configured +// with an unique temporary directory. +type BackendFactory func(testPath string) (backend.Registry, error) + +var defaultTempDir string +var keepTmpDir bool + +func init() { + flag.StringVar(&defaultTempDir, "dir", "", "Temporary directory for use by the test") + flag.BoolVar(&keepTmpDir, "keep", false, "Keep temporary test directories") +} + +// TestBackendCompliance runs a set of tests the verifies that the store +// implementation can be used by beats. +// Most tests are executed twice if they modify data. Once with keeping the +// store open between operations, and once with reopening the store between +// updates. +// For a store backend that supports different 'modes' that can impact storage, +// the compliance tests should be run with the different modes enabled. +// +// Note: The tests only check for interoperability. Implementations should add +// additional tests as well. +func TestBackendCompliance(t *testing.T, factory BackendFactory) { + t.Run("init and close registry", WithPath(factory, func(t *testing.T, reg *Registry) { + })) + + t.Run("open stores", WithPath(factory, func(t *testing.T, reg *Registry) { + store := reg.MustAccess("test1") + defer store.Close() + + store2 := reg.MustAccess("test2") + defer store2.Close() + })) + + t.Run("set-get", withBackend(factory, testSetGet)) + t.Run("remove", withBackend(factory, testRemove)) + t.Run("iteration", withBackend(factory, testIteration)) +} + +func testSetGet(t *testing.T, factory BackendFactory) { + t.Run("unknown key", WithStore(factory, func(t *testing.T, store *Store) { + has := store.MustHave("key") + assert.False(t, has) + })) + + runWithBools(t, "reopen", func(t *testing.T, reopen bool) { + t.Run("has key after set", WithStore(factory, func(t *testing.T, store *Store) { + type entry struct{ A int } + store.MustSet("key", entry{A: 1}) + + store.ReopenIf(reopen) + + has := store.MustHave("key") + assert.True(t, has) + })) + + t.Run("set and get one entry only", WithStore(factory, func(t *testing.T, store *Store) { + type entry struct{ A int } + key := "key" + value := entry{A: 1} + + store.MustSet(key, value) + store.ReopenIf(reopen) + + var actual entry + store.MustGet(key, &actual) + assert.Equal(t, value, actual) + })) + }) +} + +func testRemove(t *testing.T, factory BackendFactory) { + t.Run("no error when removing unknown key", WithStore(factory, func(t *testing.T, store *Store) { + store.MustRemove("key") + })) + + runWithBools(t, "reopen", func(t *testing.T, reopen bool) { + t.Run("remove key", WithStore(factory, func(t *testing.T, store *Store) { + type entry struct{ A int } + key := "key" + store.MustSet(key, entry{A: 1}) + store.ReopenIf(reopen) + store.MustRemove(key) + store.ReopenIf(reopen) + has := store.MustHave(key) + assert.False(t, has) + })) + }) +} + +func testIteration(t *testing.T, factory BackendFactory) { + data := map[string]interface{}{ + "a": map[string]interface{}{"field": "hello"}, + "b": map[string]interface{}{"field": "world"}, + } + + addTestData := func(store *Store, reopen bool, data map[string]interface{}) { + for k, v := range data { + store.MustSet(k, v) + } + store.ReopenIf(reopen) + } + + runWithBools(t, "reopen", func(t *testing.T, reopen bool) { + t.Run("all keys", WithStore(factory, func(t *testing.T, store *Store) { + addTestData(store, reopen, data) + + got := map[string]interface{}{} + err := store.Each(func(key string, dec backend.ValueDecoder) (bool, error) { + var tmp interface{} + if err := dec.Decode(&tmp); err != nil { + return false, err + } + + got[key] = tmp + return true, nil + }) + + assert.NoError(t, err) + assert.Equal(t, data, got) + })) + + t.Run("stop on error", WithStore(factory, func(t *testing.T, store *Store) { + addTestData(store, reopen, data) + + count := 0 + err := store.Each(func(_ string, _ backend.ValueDecoder) (bool, error) { + count++ + return true, errors.New("oops") + }) + assert.Equal(t, 1, count) + assert.Error(t, err) + })) + + t.Run("stop on bool", WithStore(factory, func(t *testing.T, store *Store) { + addTestData(store, reopen, data) + + count := 0 + err := store.Each(func(_ string, _ backend.ValueDecoder) (bool, error) { + count++ + return false, nil + }) + assert.Equal(t, 1, count) + assert.NoError(t, err) + })) + }) +} diff --git a/pkg/statestore/internal/storecompliance/util.go b/pkg/statestore/internal/storecompliance/util.go new file mode 100644 index 0000000..a6355fd --- /dev/null +++ b/pkg/statestore/internal/storecompliance/util.go @@ -0,0 +1,156 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package storecompliance + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/cleanup" +) + +// RunWithPath uses the factory to create and configure a registry with a +// temporary test path. The test function fn is called with the new registry. +// The registry is closed once the test finishes and the temporary is deleted +// afterwards (unless the `-keep` CLI flag is used). +func RunWithPath(t *testing.T, factory BackendFactory, fn func(*Registry)) { + reg, cleanup := SetupRegistry(t, factory) + defer cleanup() + fn(reg) +} + +// WithPath wraps a registry aware test function into a normalized test +// function that can be used with `t.Run`. +// The factory is used to create and configure the registry with a temporary +// test path. The registry is closed and the temporary test directory is delete +// if the test function returns or panics. +func WithPath(factory BackendFactory, fn func(*testing.T, *Registry)) func(t *testing.T) { + return func(t *testing.T) { + reg, cleanup := SetupRegistry(t, factory) + defer cleanup() + fn(t, reg) + } +} + +// SetupRegistry creates a testing Registry for the current testing.T context. +// A cleanup function that must be run via defer is returned as well. +// +// Exanple: +// reg, cleanup := SetupRegistry(t, factory) +// defer cleanup() +// ... +func SetupRegistry(t testing.TB, factory BackendFactory) (*Registry, func()) { + path, err := ioutil.TempDir(defaultTempDir, "") + if err != nil { + t.Fatalf("Failed to create temporary test directory: %v", err) + } + + ok := false + + t.Logf("Test tmp dir: %v", path) + if !keepTmpDir { + defer cleanup.IfNot(&ok, func() { + os.RemoveAll(path) + }) + } + + reg, err := factory(path) + if err != nil { + t.Fatalf("Failed to create registry: %v", err) + } + + ok = true + return &Registry{T: t, Registry: reg}, func() { + if !keepTmpDir { + defer os.RemoveAll(path) + } + reg.Close() + } +} + +// RunWithStore uses the factory to create a registry and temporary store, that +// is used with fn. The temporary directory used for the store is deleted once +// fn returns. +func RunWithStore(t *testing.T, factory BackendFactory, fn func(*Store)) { + store, cleanup := SetupTestStore(t, factory) + defer cleanup() + fn(store) +} + +// WithStore wraps a store aware test function into a normalized test function +// that can be used with `t.Run`. WithStore is based on WithPath, but will +// create and pass a test store (named "test") to the test function. The test +// store is closed once the test function returns or panics. +func WithStore(factory BackendFactory, fn func(*testing.T, *Store)) func(*testing.T) { + return func(t *testing.T) { + store, cleanup := SetupTestStore(t, factory) + defer cleanup() + fn(t, store) + } +} + +// SetupTestStore creates a testing Store for the current testing.T context. +// A cleanup function that must be run via defer is returned as well. +// +// Exanple: +// store, cleanup := SetupStore(t, factory) +// defer cleanup() +// ... +func SetupTestStore(t testing.TB, factory BackendFactory) (*Store, func()) { + reg, cleanupReg := SetupRegistry(t, factory) + store, err := reg.Access("test") + if err != nil { + defer cleanupReg() + must(t, err, "failed to create test store") + return nil, nil + } + + return store, func() { + defer cleanupReg() + store.Close() + } +} + +func withBackend(factory BackendFactory, fn func(*testing.T, BackendFactory)) func(*testing.T) { + return func(t *testing.T) { + fn(t, factory) + } +} + +func runWithBools(t *testing.T, name string, fn func(*testing.T, bool)) { + withBools(name, fn)(t) +} + +func withBools(name string, fn func(*testing.T, bool)) func(t *testing.T) { + return func(t *testing.T) { + for _, b := range []bool{false, true} { + b := b + t.Run(fmt.Sprintf("%v=%v", name, b), func(t *testing.T) { + fn(t, b) + }) + } + } +} + +func must(t testing.TB, err error, msg string, args ...interface{}) { + if err != nil { + t.Fatal(fmt.Sprintf(msg, args...), ":", err) + } +} diff --git a/pkg/statestore/mock_test.go b/pkg/statestore/mock_test.go new file mode 100644 index 0000000..007fb1e --- /dev/null +++ b/pkg/statestore/mock_test.go @@ -0,0 +1,95 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package statestore + +import ( + "fmt" + + "github.com/stretchr/testify/mock" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" +) + +type mockRegistry struct { + mock.Mock +} + +type mockStore struct { + mock.Mock +} + +func newMockRegistry() *mockRegistry { return &mockRegistry{} } + +func (m *mockRegistry) OnAccess(name string) *mock.Call { return m.On("Access", name) } +func (m *mockRegistry) Access(name string) (backend.Store, error) { + args := m.Called(name) + + var store backend.Store + if ifc := args.Get(0); ifc != nil { + var ok bool + store, ok = ifc.(backend.Store) + if !ok { + panic(fmt.Errorf("cannot convert interface to backend.Store: %v", ifc)) + } + } + + return store, args.Error(1) +} + +func (m *mockRegistry) OnClose() *mock.Call { return m.On("Close") } +func (m *mockRegistry) Close() error { + args := m.Called() + return args.Error(0) +} + +func newMockStore() *mockStore { return &mockStore{} } + +func (m *mockStore) OnClose() *mock.Call { return m.On("Close") } +func (m *mockStore) Close() error { + args := m.Called() + return args.Error(0) +} + +func (m *mockStore) OnHas(key string) *mock.Call { return m.On("Has", key) } +func (m *mockStore) Has(key string) (bool, error) { + args := m.Called(key) + return args.Bool(0), args.Error(1) +} + +func (m *mockStore) OnGet(key string) *mock.Call { return m.On("Get", key) } +func (m *mockStore) Get(key string, into interface{}) error { + args := m.Called(key) + return args.Error(0) +} + +func (m *mockStore) OnRemove(key string) *mock.Call { return m.On("Remove", key) } +func (m *mockStore) Remove(key string) error { + args := m.Called(key) + return args.Error(0) +} + +func (m *mockStore) OnSet(key string) *mock.Call { return m.On("Set", key) } +func (m *mockStore) Set(key string, from interface{}) error { + args := m.Called(key) + return args.Error(0) +} + +func (m *mockStore) Each(fn func(string, backend.ValueDecoder) (bool, error)) error { + args := m.Called(fn) + return args.Error(0) +} diff --git a/pkg/statestore/registry.go b/pkg/statestore/registry.go new file mode 100644 index 0000000..8828847 --- /dev/null +++ b/pkg/statestore/registry.go @@ -0,0 +1,89 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package statestore + +import ( + "sync" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" +) + +// Registry manages multiple key-value stores. +// When working with a registry, one must access a store. Depending on backend +// a store can be an index, a table, or a directory. All access to a store is +// handled by transaction. +type Registry struct { + backend backend.Registry + + mu sync.Mutex + active map[string]*sharedStore // active/open stores + wg sync.WaitGroup +} + +// ValueDecoder is used to decode retrieved from an actual store. A +// ValueDecoder instance is valid for the lifetime of the transaction only. +type ValueDecoder = backend.ValueDecoder + +// NewRegistry creates a new Registry with a configured backend. +func NewRegistry(backend backend.Registry) *Registry { + return &Registry{ + backend: backend, + active: map[string]*sharedStore{}, + } +} + +// Close closes the backend storage. Close blocks until all stores in use are closed. +func (r *Registry) Close() error { + r.wg.Wait() // wait for all stores being closed + return r.backend.Close() +} + +// Get opens a shared store. A store is closed and released only after all it's +// users have closed the store. +func (r *Registry) Get(name string) (*Store, error) { + r.mu.Lock() + defer r.mu.Unlock() + + shared := r.active[name] + if shared == nil { + backend, err := r.backend.Access(name) + if err != nil { + return nil, &ErrorAccess{name: name, cause: err} + } + + shared = newSharedStore(r, name, backend) + defer func() { + _ = shared.Release() + }() + + r.active[name] = shared + r.wg.Add(1) + } + + return newStore(shared), nil +} + +func (r *Registry) unregisterStore(s *sharedStore) { + _, exists := r.active[s.name] + if !exists { + panic("removing an unknown store") + } + + delete(r.active, s.name) + r.wg.Done() +} diff --git a/pkg/statestore/registry_test.go b/pkg/statestore/registry_test.go new file mode 100644 index 0000000..bb87f47 --- /dev/null +++ b/pkg/statestore/registry_test.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package statestore + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAccessStore(t *testing.T) { + t.Run("single access", func(t *testing.T) { + mr := newMockRegistry() + ms := newMockStore() + mr.OnClose().Once().Return(nil) + mr.OnAccess("test").Once().Return(ms, nil) + ms.OnClose().Once().Return(nil) + + reg := NewRegistry(mr) + store, _ := reg.Get("test") + assert.NoError(t, store.Close()) + assert.NoError(t, reg.Close()) + + mr.AssertExpectations(t) + ms.AssertExpectations(t) + }) + + t.Run("shared store instance", func(t *testing.T) { + mr := newMockRegistry() + ms := newMockStore() + mr.OnClose().Once().Return(nil) + + // test instance sharing. Store must be opened and closed only once + mr.OnAccess("test").Once().Return(ms, nil) + ms.OnClose().Once().Return(nil) + + reg := NewRegistry(mr) + s1, _ := reg.Get("test") + s2, _ := reg.Get("test") + assert.NoError(t, s1.Close()) + assert.NoError(t, s2.Close()) + assert.NoError(t, reg.Close()) + + mr.AssertExpectations(t) + ms.AssertExpectations(t) + }) + + t.Run("close non-shared store needs open", func(t *testing.T) { + mr := newMockRegistry() + ms := newMockStore() + mr.OnClose().Once().Return(nil) + + // test instance sharing. Store must be opened and closed only once + mr.OnAccess("test").Twice().Return(ms, nil) + ms.OnClose().Twice().Return(nil) + + reg := NewRegistry(mr) + + store, err := reg.Get("test") + assert.NoError(t, err) + assert.NoError(t, store.Close()) + + store, err = reg.Get("test") + assert.NoError(t, err) + assert.NoError(t, store.Close()) + + assert.NoError(t, reg.Close()) + + mr.AssertExpectations(t) + ms.AssertExpectations(t) + }) + + t.Run("separate stores are not shared", func(t *testing.T) { + mr := newMockRegistry() + mr.OnClose().Once().Return(nil) + + ms1 := newMockStore() + ms1.OnClose().Once().Return(nil) + mr.OnAccess("s1").Once().Return(ms1, nil) + + ms2 := newMockStore() + ms2.OnClose().Once().Return(nil) + mr.OnAccess("s2").Once().Return(ms2, nil) + + reg := NewRegistry(mr) + s1, err := reg.Get("s1") + assert.NoError(t, err) + s2, err := reg.Get("s2") + assert.NoError(t, err) + assert.NoError(t, s1.Close()) + assert.NoError(t, s2.Close()) + assert.NoError(t, reg.Close()) + + mr.AssertExpectations(t) + ms1.AssertExpectations(t) + ms2.AssertExpectations(t) + }) +} diff --git a/pkg/statestore/store.go b/pkg/statestore/store.go new file mode 100644 index 0000000..8a270ca --- /dev/null +++ b/pkg/statestore/store.go @@ -0,0 +1,176 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package statestore + +import ( + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" + "github.com/elastic/go-concert/atomic" + "github.com/elastic/go-concert/unison" +) + +type sharedStore struct { + reg *Registry + refCount atomic.Int + + name string + backend backend.Store +} + +// Store instance. The backend is shared between multiple instances of this store. +// The backend will be closed only after the last instance have been closed. +// No transaction can be created once the store instance has been closed. +// A Store is not thread-safe. Each go-routine accessing a store should create +// an instance using `Registry.Get`. +type Store struct { + shared *sharedStore + // wait group to ensure active operations can finish, but not started anymore after the store has been closed. + active unison.SafeWaitGroup +} + +func newSharedStore(reg *Registry, name string, backend backend.Store) *sharedStore { + return &sharedStore{ + reg: reg, + refCount: atomic.MakeInt(1), + name: name, + backend: backend, + } +} + +func newStore(shared *sharedStore) *Store { + shared.Retain() + return &Store{ + shared: shared, + } +} + +// Close deactivates the current store. No new transacation can be generated. +// Already active transaction will continue to function until Closed. +// The backing store will be closed once all stores and active transactions have been closed. +func (s *Store) Close() error { + if err := s.active.Add(1); err != nil { + return &ErrorClosed{operation: "store/close", name: s.shared.name} + } + s.active.Close() + s.active.Done() + + s.active.Wait() + return s.shared.Release() +} + +// Has checks if the given key exists. +// Has returns an error if the store has already been closed or the storage backend returns an error. +func (s *Store) Has(key string) (bool, error) { + const operation = "store/has" + if err := s.active.Add(1); err != nil { + return false, &ErrorClosed{operation: operation, name: s.shared.name} + } + defer s.active.Done() + + has, err := s.shared.backend.Has((key)) + if err != nil { + return false, &ErrorOperation{name: s.shared.name, operation: operation, cause: err} + } + return has, nil +} + +// Get unpacks the value for a given key into "into". +// Get returns an error if the store has already been closed, the key does not +// exist, or the storage backend returns an error. +func (s *Store) Get(key string, into interface{}) error { + const operation = "store/get" + if err := s.active.Add(1); err != nil { + return &ErrorClosed{operation: operation, name: s.shared.name} + } + defer s.active.Done() + + err := s.shared.backend.Get(key, into) + if err != nil { + return &ErrorOperation{name: s.shared.name, operation: operation, cause: err} + } + return nil +} + +// Set inserts or overwrite a key value pair. +// Set returns an error if the store has been closed, the value can not be +// encoded by the store, or the storage backend did failed. +func (s *Store) Set(key string, from interface{}) error { + const operation = "store/get" + if err := s.active.Add(1); err != nil { + return &ErrorClosed{operation: operation, name: s.shared.name} + } + defer s.active.Done() + + if err := s.shared.backend.Set((key), from); err != nil { + return &ErrorOperation{name: s.shared.name, operation: operation, cause: err} + } + return nil +} + +// Remove removes a key value pair from the store. Remove does not error if the +// key is unknown to the store. +// An error is returned if the store has already been closed or the operation +// itself fails in the storage backend. +func (s *Store) Remove(key string) error { + const operation = "store/remove" + if err := s.active.Add(1); err != nil { + return &ErrorClosed{operation: operation, name: s.shared.name} + } + defer s.active.Done() + + if err := s.shared.backend.Remove((key)); err != nil { + return &ErrorOperation{name: s.shared.name, operation: operation, cause: err} + } + return nil +} + +// Each iterates over all key-value pairs in the store. +// The iteration stops if fn returns false or an error value != nil. +// If the store has been closed already an error is returned. +func (s *Store) Each(fn func(string, ValueDecoder) (bool, error)) error { + if err := s.active.Add(1); err != nil { + return &ErrorClosed{operation: "store/each", name: s.shared.name} + } + defer s.active.Done() + + return s.shared.backend.Each(fn) +} + +func (s *sharedStore) Retain() { + s.refCount.Inc() +} + +func (s *sharedStore) Release() error { + if s.refCount.Dec() == 0 && s.tryUnregister() { + return s.backend.Close() + } + return nil +} + +// tryUnregister removed the store from the registry. tryUnregister returns false +// if the store has been retained in the meantime. True is returned if the store +// can be closed for sure. +func (s *sharedStore) tryUnregister() bool { + s.reg.mu.Lock() + defer s.reg.mu.Unlock() + if s.refCount.Load() > 0 { + return false + } + + s.reg.unregisterStore(s) + return true +} diff --git a/pkg/statestore/store_test.go b/pkg/statestore/store_test.go new file mode 100644 index 0000000..8ce4312 --- /dev/null +++ b/pkg/statestore/store_test.go @@ -0,0 +1,223 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package statestore + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/storetest" +) + +func TestStore_Close(t *testing.T) { + t.Run("close succeeds", func(t *testing.T) { + makeClosedTestStore(t) + }) + t.Run("fails if store has been closed", func(t *testing.T) { + assert.Error(t, makeClosedTestStore(t).Close()) + }) +} + +func TestStore_Has(t *testing.T) { + t.Run("fails if store has been closed", func(t *testing.T) { + store := makeClosedTestStore(t) + _, err := store.Has("test") + assertClosed(t, err) + }) + t.Run("error is passed through", func(t *testing.T) { + ms := newMockStore() + ms.OnHas("test").Return(false, errors.New("oops")) + defer ms.AssertExpectations(t) + + store := makeTestMockedStore(t, ms) + defer store.Close() + + _, err := store.Has("test") + assert.Error(t, err) + }) + t.Run("return result from backend", func(t *testing.T) { + data := map[string]interface{}{"known_key": "test"} + store := makeTestStore(t, data) + defer store.Close() + + got, err := store.Has("known_key") + assert.NoError(t, err) + assert.True(t, got) + + got, err = store.Has("unknown_key") + assert.NoError(t, err) + assert.False(t, got) + }) +} + +func TestStore_Get(t *testing.T) { + t.Run("fails if store has been closed", func(t *testing.T) { + store := makeClosedTestStore(t) + var tmp interface{} + assertClosed(t, store.Get("test", &tmp)) + }) + t.Run("error is passed through", func(t *testing.T) { + ms := newMockStore() + defer ms.AssertExpectations(t) + + store := makeTestMockedStore(t, ms) + defer store.Close() + + ms.OnGet("test").Return(errors.New("oops")) + var tmp interface{} + err := store.Get("test", &tmp) + assert.Error(t, err) + }) + t.Run("return result from backend", func(t *testing.T) { + data := map[string]interface{}{"known_key": "test"} + store := makeTestStore(t, data) + defer store.Close() + + var got interface{} + err := store.Get("known_key", &got) + assert.NoError(t, err) + assert.Equal(t, "test", got) + }) +} + +func TestStore_Set(t *testing.T) { + t.Run("fails if store has been closed", func(t *testing.T) { + store := makeClosedTestStore(t) + var tmp interface{} + assertClosed(t, store.Set("test", &tmp)) + }) + t.Run("error is passed through", func(t *testing.T) { + ms := newMockStore() + defer ms.AssertExpectations(t) + + store := makeTestMockedStore(t, ms) + defer store.Close() + + ms.OnSet("test").Return(errors.New("oops")) + err := store.Set("test", nil) + assert.Error(t, err) + }) + t.Run("set key in backend", func(t *testing.T) { + data := map[string]interface{}{} + store := makeTestStore(t, data) + defer store.Close() + + err := store.Set("key", "value") + assert.NoError(t, err) + assert.Equal(t, "value", data["key"]) + }) +} + +func TestStore_Remove(t *testing.T) { + t.Run("fails if store has been closed", func(t *testing.T) { + store := makeClosedTestStore(t) + assertClosed(t, store.Remove("test")) + }) + t.Run("error is passed through", func(t *testing.T) { + ms := newMockStore() + ms.OnRemove("test").Return(errors.New("oops")) + defer ms.AssertExpectations(t) + + store := makeTestMockedStore(t, ms) + defer store.Close() + + assert.Error(t, store.Remove("test")) + }) + t.Run("remove key from backend", func(t *testing.T) { + data := map[string]interface{}{"key": "test"} + store := makeTestStore(t, data) + + err := store.Remove("key") + assert.NoError(t, err) + assert.Equal(t, 0, len(data)) + }) +} + +func TestStore_Each(t *testing.T) { + t.Run("fails if store has been closed", func(t *testing.T) { + store := makeClosedTestStore(t) + assertClosed(t, store.Each(func(string, ValueDecoder) (bool, error) { + return true, nil + })) + }) + t.Run("correctly iterate pairs", func(t *testing.T) { + data := map[string]interface{}{ + "a": map[string]interface{}{"field": "hello"}, + "b": map[string]interface{}{"field": "test"}, + } + store := makeTestStore(t, data) + defer store.Close() + + got := map[string]interface{}{} + err := store.Each(func(key string, dec ValueDecoder) (bool, error) { + var tmp interface{} + if err := dec.Decode(&tmp); err != nil { + t.Fatalf("failed to read value from store: %v", err) + } + got[key] = tmp + return true, nil + }) + + assert.NoError(t, err) + assert.Equal(t, data, got) + }) +} + +func makeTestStore(t *testing.T, data map[string]interface{}) *Store { + memstore := &storetest.MapStore{Table: data} + reg := NewRegistry(&storetest.MemoryStore{ + Stores: map[string]*storetest.MapStore{ + "test": memstore, + }, + }) + store, err := reg.Get("test") + if err != nil { + t.Fatalf("Failed to create test store: %v", err) + } + return store +} + +func makeTestMockedStore(t *testing.T, ms *mockStore) *Store { + mr := newMockRegistry() + mr.OnAccess("test").Once().Return(ms, nil) + + reg := NewRegistry(mr) + s, err := reg.Get("test") + require.NoError(t, err) + + ms.OnClose().Return(nil) + return s +} + +func makeClosedTestStore(t *testing.T) *Store { + s := makeTestMockedStore(t, newMockStore()) + require.NoError(t, s.Close()) + return s +} + +func assertClosed(t *testing.T, err error) { + if err == nil { + t.Fatal("expected error") + } + if !IsClosed(err) { + t.Fatalf("The error does not seem to indicate a failure because of a closed store. Error: %v", err) + } +} diff --git a/pkg/statestore/storetest/storetest.go b/pkg/statestore/storetest/storetest.go new file mode 100644 index 0000000..cc2252c --- /dev/null +++ b/pkg/statestore/storetest/storetest.go @@ -0,0 +1,214 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package storetest provides helpers for testing functionality that requires a statestore. +package storetest + +import ( + "errors" + "sync" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" + "github.com/elastic/elastic-agent-libs/transform/typeconv" +) + +// MemoryStore provides a dummy backend store that holds all access stores and +// data in memory. The Stores field is accessible for introspection or custom +// initialization. Stores should not be modified while a test is active. +// For validation one can use the statestore API or introspect the tables directly. +// +// The zero value is MemoryStore is a valid store instance. The Stores field +// will be initialized lazily if it has not been setup upfront. +// +// Example: Create store for testing: +// store := statestore.NewRegistry(storetest.NewMemoryStoreBackend()) +type MemoryStore struct { + Stores map[string]*MapStore + mu sync.Mutex +} + +// MapStore implements a single in memory storage. The MapStore holds all +// key-value pairs in a map[string]interface{}. +type MapStore struct { + mu sync.RWMutex + closed bool + Table map[string]interface{} +} + +type valueUnpacker struct { + from interface{} +} + +// CreateValueDecoder creates a backend.ValueDecoder that can be used to unpack +// an value into a custom go type. +func CreateValueDecoder(v interface{}) backend.ValueDecoder { + return valueUnpacker{v} +} + +var errMapStoreClosed = errors.New("store closed") +var errUnknownKey = errors.New("unknown key") + +// NewMemoryStoreBackend creates a new backend.Registry instance that can be +// used with the statestore. +func NewMemoryStoreBackend() *MemoryStore { + return &MemoryStore{} +} + +func (m *MemoryStore) init() { + if m.Stores == nil { + m.Stores = map[string]*MapStore{} + } +} + +// Access returns a MapStore that for the given name. A new store is created +// and registered in the Stores table, if the store name is new to MemoryStore. +func (m *MemoryStore) Access(name string) (backend.Store, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.init() + + store, exists := m.Stores[name] + if !exists { + store = &MapStore{} + m.Stores[name] = store + } else { + store.Reopen() + } + return store, nil +} + +// Close closes the store. +func (m *MemoryStore) Close() error { return nil } + +func (s *MapStore) init() { + if s.Table == nil { + s.Table = map[string]interface{}{} + } +} + +// Reopen marks the MapStore as open in case it has been closed already. All +// key-value pairs and store operations are accessible after reopening the +// store. +func (s *MapStore) Reopen() { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = false +} + +// Close marks the store as closed. The Store API calls like Has, Get, Set, and +// Remove will fail until the store is reopenned. +func (s *MapStore) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + return nil +} + +// IsClosed returns true if the store is marked as closed. +func (s *MapStore) IsClosed() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.closed +} + +// Has checks if the key value pair is known to the store. +// It returns an error if the store is marked as closed. +func (s *MapStore) Has(key string) (bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if s.closed { + return false, errMapStoreClosed + } + + s.init() + _, exists := s.Table[key] + return exists, nil +} + +// Get returns a key value pair from the store. An error is returned if the +// store has been closed, the key is unknown, or an decoding error occurred. +func (s *MapStore) Get(key string, into interface{}) error { + s.mu.RLock() + defer s.mu.RUnlock() + if s.closed { + return errMapStoreClosed + } + + s.init() + val, exists := s.Table[key] + if !exists { + return errUnknownKey + } + return typeconv.Convert(into, val) +} + +// Set inserts or overwrites a key-value pair. +// An error is returned if the store is marked as closed or the value being +// passed in can not be encoded. +func (s *MapStore) Set(key string, from interface{}) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return errMapStoreClosed + } + + s.init() + var tmp interface{} + if err := typeconv.Convert(&tmp, from); err != nil { + return err + } + s.Table[key] = tmp + return nil +} + +// Remove removes a key value pair from the store. +// An error is returned if the store is marked as closed. +func (s *MapStore) Remove(key string) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return errMapStoreClosed + } + + s.init() + delete(s.Table, key) + return nil +} + +// Each iterates all key value pairs in the store calling fn. +// The iteration stops if fn returns false or an error. +// Each returns an error if the store is closed, or fn returns an error. +func (s *MapStore) Each(fn func(string, backend.ValueDecoder) (bool, error)) error { + s.mu.RLock() + defer s.mu.RUnlock() + if s.closed { + return errMapStoreClosed + } + + s.init() + for k, v := range s.Table { + cont, err := fn(k, CreateValueDecoder(v)) + if !cont || err != nil { + return err + } + } + return nil +} + +func (d valueUnpacker) Decode(to interface{}) error { + return typeconv.Convert(to, d.from) +} diff --git a/pkg/statestore/storetest/storetest_test.go b/pkg/statestore/storetest/storetest_test.go new file mode 100644 index 0000000..3d981b9 --- /dev/null +++ b/pkg/statestore/storetest/storetest_test.go @@ -0,0 +1,59 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package storetest + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-inputs/pkg/statestore/backend" + "github.com/elastic/elastic-agent-inputs/pkg/statestore/internal/storecompliance" + "github.com/elastic/elastic-agent-libs/logp" +) + +func init() { + err := logp.DevelopmentSetup() + if err != nil { + panic(err) + } +} + +func TestCompliance(t *testing.T) { + storecompliance.TestBackendCompliance(t, func(testPath string) (backend.Registry, error) { + return NewMemoryStoreBackend(), nil + }) +} + +func TestStore_IsClosed(t *testing.T) { + t.Run("false by default", func(t *testing.T) { + store := &MapStore{} + assert.False(t, store.IsClosed()) + }) + t.Run("true after close", func(t *testing.T) { + store := &MapStore{} + store.Close() + assert.True(t, store.IsClosed()) + }) + t.Run("true after reopen", func(t *testing.T) { + store := &MapStore{} + store.Close() + store.Reopen() + assert.False(t, store.IsClosed()) + }) +} diff --git a/tools/tools.go b/tools/tools.go index 2f77091..b71aaa0 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -20,4 +20,7 @@ package tools -import _ "go.elastic.co/go-licence-detector" +import ( + _ "github.com/elastic/go-licenser" + _ "go.elastic.co/go-licence-detector" +)