diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..c31e1f7 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index e340e6b..67e87d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea .java-version /target +/target diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..2d8531b --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..8a81040 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml new file mode 100644 index 0000000..2e46793 --- /dev/null +++ b/.idea/jarRepositories.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d5975a1 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,15 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..2b63946 --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..b932771 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,105 @@ +# Salesforce Open Source Community Code of Conduct + +## About the Code of Conduct + +Equality is a core value at Salesforce. We believe a diverse and inclusive +community fosters innovation and creativity, and are committed to building a +culture where everyone feels included. + +Salesforce open-source projects are committed to providing a friendly, safe, and +welcoming environment for all, regardless of gender identity and expression, +sexual orientation, disability, physical appearance, body size, ethnicity, nationality, +race, age, religion, level of experience, education, socioeconomic status, or +other similar personal characteristics. + +The goal of this code of conduct is to specify a baseline standard of behavior so +that people with different social values and communication styles can work +together effectively, productively, and respectfully in our open source community. +It also establishes a mechanism for reporting issues and resolving conflicts. + +All questions and reports of abusive, harassing, or otherwise unacceptable behavior +in a Salesforce open-source project may be reported by contacting the Salesforce +Open Source Conduct Committee at ossconduct@salesforce.com. + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of gender +identity and expression, sexual orientation, disability, physical appearance, +body size, ethnicity, nationality, race, age, religion, level of experience, education, +socioeconomic status, or other similar personal characteristics. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy toward other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Personal attacks, insulting/derogatory comments, or trolling +* Public or private harassment +* Publishing, or threatening to publish, others' private information—such as +a physical or electronic address—without explicit permission +* Other conduct which could reasonably be considered inappropriate in a +professional setting +* Advocating for or encouraging any of the above behaviors + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned with this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project email +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the Salesforce Open Source Conduct Committee +at ossconduct@salesforce.com. All complaints will be reviewed and investigated +and will result in a response that is deemed necessary and appropriate to the +circumstances. The committee is obligated to maintain confidentiality with +regard to the reporter of an incident. Further details of specific enforcement +policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership and the Salesforce Open Source Conduct +Committee. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant-home], +version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html. +It includes adaptions and additions from [Go Community Code of Conduct][golang-coc], +[CNCF Code of Conduct][cncf-coc], and [Microsoft Open Source Code of Conduct][microsoft-coc]. + +This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us]. + +[contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/) +[golang-coc]: https://golang.org/conduct +[cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md +[microsoft-coc]: https://opensource.microsoft.com/codeofconduct/ +[cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..fc37a1d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +[How to contribute](https://mac-project.ai/docs/contribute) \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index db4cdb7..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 Amir Khan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..ae7332a --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2024 Salesforce, Inc. +All rights reserved. + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..14f4d0c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +## Security + +Please report any security issue to [security@salesforce.com](mailto:security@salesforce.com) +as soon as it is discovered. This library limits its runtime dependencies in +order to reduce the total cost of ownership as much as can be, but all consumers +should remain vigilant and have their security stakeholders review all third-party +products (3PP) like this one and their dependencies. \ No newline at end of file diff --git a/icon/icon.svg b/icon/icon.svg index c1931b2..e8f3854 100644 --- a/icon/icon.svg +++ b/icon/icon.svg @@ -1,3324 +1,411 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 49ea83f..8e6c98f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,17 +6,11 @@ 4.0.0 com.mule.mulechain mac-web-crawler - 0.1.0 + 0.1.25-SNAPSHOT mule-extension - MAC WebCrawler Connector + mac-web-crawler Extension The MAC WebCrawler Connector enables a Mule application to crawl websites and retrieve content, potentially for creating vector embeddings for structured knowledge extraction. https://mac-project.ai/docs/mac-webcrawler/connector-overview - - - MIT License - https://www.opensource.org/licenses/mit-license.php - - Yogesh Mudaliar @@ -46,7 +40,28 @@ mule-modules-parent 1.3.2 + + + 4.6.0 + + + + + + org.mule + mule-javaee-runtime-bom + ${muleJavaEeBomVersion} + pom + import + + + + + org.mule.sdk + mule-sdk-api + 0.9.0-rc1 + org.jsoup jsoup @@ -57,5 +72,68 @@ jackson-databind 2.15.2 + + com.squareup.okhttp3 + okhttp + 4.12.0 + + + org.json + json + 20240303 + + + commons-io + commons-io + 2.16.1 + + + + org.seleniumhq.selenium + selenium-java + 4.26.0 + + + io.github.bonigarcia + webdrivermanager + 5.9.2 + +<<<<<<< HEAD + + + + anypoint-exchange-v3 + Anypoint Exchange V3 + https://maven.eu1.anypoint.mulesoft.com/api/v3/maven + default + + + + + mulesoft-releases + mulesoft release repository + default + https://repository.mulesoft.org/releases/ + + false + + + + + + exchange-repository + Exchange Repository + https://maven.anypoint.mulesoft.com/api/v1/organizations/9c7c42b3-1b21-4ec1-9183-047ec0921663/maven + default + + + exchange-repository + Exchange Repository + https://maven.anypoint.mulesoft.com/api/v1/organizations/9c7c42b3-1b21-4ec1-9183-047ec0921663/maven + default + + +======= +>>>>>>> master diff --git a/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerExtension.java b/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerExtension.java index 41574e7..00e3eab 100644 --- a/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerExtension.java +++ b/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerExtension.java @@ -3,7 +3,10 @@ import org.mule.runtime.extension.api.annotation.Extension; import org.mule.runtime.extension.api.annotation.Configurations; import org.mule.runtime.extension.api.annotation.dsl.xml.Xml; - +import org.mule.sdk.api.annotation.JavaVersionSupport; +import static org.mule.sdk.api.meta.JavaVersion.JAVA_11; +import static org.mule.sdk.api.meta.JavaVersion.JAVA_17; +import static org.mule.sdk.api.meta.JavaVersion.JAVA_8; /** * This is the main class of an extension, is the entry point from which configurations, connection providers, operations @@ -12,6 +15,7 @@ @Xml(prefix = "mac-web-crawler") @Extension(name = "MAC WebCrawler") @Configurations(MulechainwebcrawlerConfiguration.class) +@JavaVersionSupport({JAVA_8, JAVA_11, JAVA_17}) public class MulechainwebcrawlerExtension { } diff --git a/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerOperations.java b/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerOperations.java index 26f9c00..d8fb67a 100644 --- a/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerOperations.java +++ b/src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerOperations.java @@ -3,6 +3,11 @@ import com.mule.mulechain.crawler.internal.helpers.CrawlResult; import com.mule.mulechain.crawler.internal.helpers.SiteMapNode; import com.mule.mulechain.crawler.internal.helpers.crawlingHelper; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.json.JSONObject; import org.jsoup.UnsupportedMimeTypeException; import org.jsoup.nodes.Document; import org.mule.runtime.extension.api.annotation.Alias; @@ -16,13 +21,16 @@ import java.io.*; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.*; -import static org.mule.runtime.extension.api.annotation.param.MediaType.ANY; +import static org.apache.commons.io.IOUtils.toInputStream; +import static org.mule.runtime.extension.api.annotation.param.MediaType.APPLICATION_JSON; /** - * This class is a container for operations, every public method in this class will be taken as an extension operation. + * This class is a container for operations, every public method in this class + * will be taken as an extension operation. */ public class MulechainwebcrawlerOperations { @@ -34,80 +42,93 @@ private enum CrawlType { private static final Logger LOGGER = LoggerFactory.getLogger(MulechainwebcrawlerOperations.class); /** - * Crawl a website at a specified depth and fetch contents. Specify tags and classes in the configuration to fetch contents from those elements only. + * Crawl a website at a specified depth and fetch contents. Specify tags and + * classes in the configuration to fetch contents from those elements only. * * @throws IOException */ - /* JSoup limitiations / web crawl challenges - - some sites prevent robots - use of User-Agent may be required but not always guaranteed to work - - JavaScript generated content is not read by jsoup - - some sites require cookies or sessions to be present + /* + * JSoup limitiations / web crawl challenges + * - some sites prevent robots - use of User-Agent may be required but not + * always guaranteed to work + * - JavaScript generated content is not read by jsoup + * - some sites require cookies or sessions to be present */ - @MediaType(value = ANY, strict = false) + @MediaType(value = APPLICATION_JSON, strict = false) @Alias("Crawl-website") - public String crawlWebsite(@Config MulechainwebcrawlerConfiguration configuration, - @DisplayName("Website URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url, - @DisplayName("Maximum Depth") @Placement(order = 2) @Example("2") int maxDepth, - @DisplayName("Retrieve Meta Tags") @Placement(order = 3) @Example("Yes") boolean getMetaTags, - @DisplayName("Download Images") @Placement(order = 4) @Example("Yes") boolean downloadImages, - @DisplayName("Download Location") @Placement(order = 5) @Example("/users/mulesoft/downloads") String downloadPath) throws IOException { + public InputStream crawlWebsite(@Config MulechainwebcrawlerConfiguration configuration, + @DisplayName("Website URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url, + @DisplayName("Restrict Crawl under URL") @Placement(order = 2) @Example("False") boolean restrictToPath, + @DisplayName("Dynamic Content Retrieval") @Placement(order = 3) @Example("False") boolean dynamicContent, + @DisplayName("Maximum Depth") @Placement(order = 4) @Example("2") int maxDepth, + @DisplayName("Delay (millisecs)") @Placement(order = 5) @Example("0") int delayMillis, + @DisplayName("Retrieve Meta Tags") @Placement(order = 6) @Example("False") boolean getMetaTags, + @DisplayName("Download Images") @Placement(order = 7) @Example("False") boolean downloadImages, + @DisplayName("Download Location") @Placement(order = 8) @Example("/users/mulesoft/downloads") String downloadPath) + throws IOException { LOGGER.info("Website crawl action"); - // initialise variables Set visitedLinksGlobal = new HashSet<>(); Map> visitedLinksByDepth = new HashMap<>(); List specificTags = configuration.getTags(); - SiteMapNode root = startCrawling(url, 0, maxDepth, visitedLinksByDepth, visitedLinksGlobal, downloadImages, downloadPath, specificTags, getMetaTags, CrawlType.CONTENT); - + String originalUrl = url; + SiteMapNode root = startCrawling(url, originalUrl, 0, maxDepth, restrictToPath, dynamicContent, delayMillis, visitedLinksByDepth, visitedLinksGlobal, downloadImages, + downloadPath, specificTags, getMetaTags, CrawlType.CONTENT); - return crawlingHelper.convertToJSON(root); + return toInputStream(crawlingHelper.convertToJSON(root), StandardCharsets.UTF_8); } - /** - * Fetch the meta tags from a web page. - */ - @MediaType(value = ANY, strict = false) + * Fetch the meta tags from a web page. + */ + @MediaType(value = APPLICATION_JSON, strict = false) @Alias("Get-page-meta-tags") - public String getMetaTags ( - @DisplayName("Page URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url) throws IOException { + public InputStream getMetaTags( + @DisplayName("Page URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url) + throws IOException { LOGGER.info("Get meta tags"); Document document = crawlingHelper.getDocument(url); - return crawlingHelper.convertToJSON(crawlingHelper.getPageMetaTags(document)); + //return crawlingHelper.convertToJSON(crawlingHelper.getPageMetaTags(document)); + return toInputStream(crawlingHelper.convertToJSON(crawlingHelper.getPageMetaTags(document)),StandardCharsets.UTF_8) ; } /** * Retrieve internal links as a site map from the specified url and depth. */ - @MediaType(value = ANY, strict = false) + @MediaType(value = APPLICATION_JSON, strict = false) @Alias("Generate-sitemap") - public String getSiteMap ( - @DisplayName("Website URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url, - @DisplayName("Maximum Depth") @Placement(order = 2) @Example("2") int maxDepth) throws IOException { + public InputStream getSiteMap( + @DisplayName("Website URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url, + @DisplayName("Maximum Depth") @Placement(order = 2) @Example("2") int maxDepth, + @DisplayName("Delay (millisecs)") @Placement(order = 3) @Example("0") int delayMillis) throws IOException { LOGGER.info("Generate sitemap"); // initialise variables Set visitedLinksGlobal = new HashSet<>(); Map> visitedLinksByDepth = new HashMap<>(); - SiteMapNode root = startCrawling(url, 0, maxDepth, visitedLinksByDepth, visitedLinksGlobal, false, null, null, false, CrawlType.LINK); + String originalUrl = url; + SiteMapNode root = startCrawling(url, originalUrl,0, maxDepth, false, false, delayMillis, visitedLinksByDepth, visitedLinksGlobal, false, null, null, + false, CrawlType.LINK); - return crawlingHelper.convertToJSON(root); + return toInputStream(crawlingHelper.convertToJSON(root),StandardCharsets.UTF_8) ; } /** - * Download all images from a web page, or download a single image at the specified link. + * Download all images from a web page, or download a single image at the + * specified link. */ - @MediaType(value = ANY, strict = false) + @MediaType(value = APPLICATION_JSON, strict = false) @Alias("Download-image") - public String downloadWebsiteImages ( - @DisplayName("Page Or Image URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url, - @DisplayName("Download Location") @Placement(order = 2) @Example("/users/mulesoft/downloads") String downloadPath) throws IOException { + public InputStream downloadWebsiteImages( + @DisplayName("Page Or Image URL") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url, + @DisplayName("Download Location") @Placement(order = 2) @Example("/users/mulesoft/downloads") String downloadPath) + throws IOException { String result = ""; @@ -115,42 +136,44 @@ public String downloadWebsiteImages ( // url provided is a website url, so download all images from this document Document document = crawlingHelper.getDocument(url); result = crawlingHelper.convertToJSON(downloadWebsiteImages(document, downloadPath)); - } - catch (UnsupportedMimeTypeException e) { + } catch (UnsupportedMimeTypeException e) { // url provided is direct link to image, so download single image Map linkFileMap = new HashMap<>(); linkFileMap.put(url, downloadSingleImage(url, downloadPath)); result = crawlingHelper.convertToJSON(linkFileMap); } - return result; + return toInputStream(result,StandardCharsets.UTF_8) ; } - /** - * Get insights from a web page including links, word count, number of occurrences of elements. Restrict insights to specific elements in the configuration. + * Get insights from a web page including links, word count, number of + * occurrences of elements. Restrict insights to specific elements in the + * configuration. */ - @MediaType(value = ANY, strict = false) + @MediaType(value = APPLICATION_JSON, strict = false) @Alias("Get-page-insights") - public String getPageInsights( - @Config MulechainwebcrawlerConfiguration configuration, - @DisplayName("Page Url") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url) throws IOException { + public InputStream getPageInsights( + @Config MulechainwebcrawlerConfiguration configuration, + @DisplayName("Page Url") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url) + throws IOException { LOGGER.info("Analyze page"); Document document = crawlingHelper.getDocument(url); - return crawlingHelper.convertToJSON(crawlingHelper.getPageInsights(document, configuration.getTags(), crawlingHelper.PageInsightType.ALL)); + return toInputStream(crawlingHelper.convertToJSON( + crawlingHelper.getPageInsights(document, configuration.getTags(), crawlingHelper.PageInsightType.ALL)), StandardCharsets.UTF_8) ; } - /** * Get contents of a web page. Content is returned in the resulting payload. */ - @MediaType(value = ANY, strict = false) + @MediaType(value = APPLICATION_JSON, strict = false) @Alias("Get-page-content") - public String getPageContent( - @Config MulechainwebcrawlerConfiguration configuration, - @DisplayName("Page Url") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url) throws IOException { + public InputStream getPageContent( + @Config MulechainwebcrawlerConfiguration configuration, + @DisplayName("Page Url") @Placement(order = 1) @Example("https://mac-project.ai/docs") String url) + throws IOException { LOGGER.info("Get page content"); Map contents = new HashMap(); @@ -161,10 +184,9 @@ public String getPageContent( contents.put("title", document.title()); contents.put("content", crawlingHelper.getPageContent(document, configuration.getTags())); - return crawlingHelper.convertToJSON(contents); + return toInputStream(crawlingHelper.convertToJSON(contents), StandardCharsets.UTF_8) ; } - private String savePageContents(Object results, String downloadPath, String title) throws IOException { String pageContents = crawlingHelper.convertToJSON(results); @@ -174,7 +196,6 @@ private String savePageContents(Object results, String downloadPath, String titl // Generate a unique filename using the current timestamp String timestamp = new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date()); - // Create a unique filename based on the sanitized title fileName = crawlingHelper.getSanitizedFilename(title) + "_" + timestamp + ".json"; @@ -195,15 +216,26 @@ private String savePageContents(Object results, String downloadPath, String titl return (file != null) ? file.getName() : "File is null"; } - - //private String startCrawling(String url, int depth, int maxDepth, Set visitedLinks, boolean downloadImages, String downloadPath, List tags) { - private SiteMapNode startCrawling(String url, int depth, int maxDepth, Map> visitedLinksByDepth, Set visitedLinksGlobal, boolean downloadImages, String downloadPath, List contentTags, boolean getMetaTags, CrawlType crawlType ) { + // private String startCrawling(String url, int depth, int maxDepth, Set + // visitedLinks, boolean downloadImages, String downloadPath, List tags) + // { + private SiteMapNode startCrawling(String url, String originalUrl, int depth, int maxDepth, boolean restrictToPath, boolean dynamicContent, int delayMillis, Map> visitedLinksByDepth, + Set visitedLinksGlobal, boolean downloadImages, String downloadPath, List contentTags, + boolean getMetaTags, CrawlType crawlType) { // return if maxDepth reached if (depth > maxDepth) { return null; } + if (restrictToPath) { + // Restrict crawling to URLs under the original URL only + if (!url.startsWith(originalUrl)) { + LOGGER.info("SKIPPING due to strict crawling: " + url); + return null; + } + } + // Initialize the set for the current depth if not already present visitedLinksByDepth.putIfAbsent(depth, new HashSet<>()); @@ -215,25 +247,38 @@ private SiteMapNode startCrawling(String url, int depth, int maxDepth, Map pageData = new HashMap<>(); - LOGGER.info("Fetching content for : " + url); String title = document.title(); @@ -241,14 +286,12 @@ private SiteMapNode startCrawling(String url, int depth, int maxDepth, Map links = new HashSet<>(); - Map linksMap = (Map) crawlingHelper.getPageInsights(document, null, crawlingHelper.PageInsightType.INTERNALLINKS).get("links"); + Map linksMap = (Map) crawlingHelper + .getPageInsights(document, null, crawlingHelper.PageInsightType.INTERNALLINKS).get("links"); + if (linksMap != null) { - links = (Set) linksMap.get("internal"); // Cast to Set + links = (Set) linksMap.get("internal"); // Cast to Set } if (links != null) { for (String nextUrl : links) { // Recursively crawl the link and add as a child - SiteMapNode childNode = startCrawling(nextUrl, depth + 1, maxDepth, visitedLinksByDepth, visitedLinksGlobal, downloadImages, downloadPath, contentTags, getMetaTags, crawlType); + SiteMapNode childNode = startCrawling(nextUrl, originalUrl, depth + 1, maxDepth, restrictToPath, dynamicContent, delayMillis, visitedLinksByDepth, visitedLinksGlobal, + downloadImages, downloadPath, contentTags, getMetaTags, crawlType); if (childNode != null) { node.addChild(childNode); } @@ -314,9 +354,10 @@ private Map downloadWebsiteImages(Document document, String save Map linkFileMap = new HashMap<>(); - Map linksMap = (Map) crawlingHelper.getPageInsights(document, null, crawlingHelper.PageInsightType.IMAGELINKS).get("links"); + Map linksMap = (Map) crawlingHelper + .getPageInsights(document, null, crawlingHelper.PageInsightType.IMAGELINKS).get("links"); if (linksMap != null) { - imageUrls = (Set) linksMap.get("images"); // Cast to Set + imageUrls = (Set) linksMap.get("images"); // Cast to Set } if (imageUrls != null) { @@ -330,7 +371,7 @@ private Map downloadWebsiteImages(Document document, String save return linkFileMap; } - private String downloadSingleImage(String imageUrl, String saveDirectory) throws IOException{ + private String downloadSingleImage(String imageUrl, String saveDirectory) throws IOException { LOGGER.info("Found image : " + imageUrl); File file; try { @@ -385,7 +426,7 @@ private String downloadSingleImage(String imageUrl, String saveDirectory) throws // Extract the filename from the decoded URL String fileName = crawlingHelper.extractFileNameFromUrl(decodedUrl); - //String fileName = decodedUrl.substring(imageUrl.lastIndexOf("/") + 1); + // String fileName = decodedUrl.substring(imageUrl.lastIndexOf("/") + 1); file = new File(saveDirectory, fileName); // Ensure the directory exists @@ -393,7 +434,7 @@ private String downloadSingleImage(String imageUrl, String saveDirectory) throws // Download and save the image try (InputStream in = url.openStream(); - FileOutputStream out = new FileOutputStream(file)) { + FileOutputStream out = new FileOutputStream(file)) { byte[] buffer = new byte[1024]; int bytesRead; @@ -411,5 +452,37 @@ private String downloadSingleImage(String imageUrl, String saveDirectory) throws return (file != null) ? file.getName() : "File is null"; } -} + /** + * Perform a Google search using the SERP API. + * + * @throws IOException + */ + @MediaType(value = MediaType.APPLICATION_JSON, strict = false) + @Alias("Google-search") + public String googleSearch( + @DisplayName("Search Query") @Placement(order = 1) @Example("apple inc") String query, + @DisplayName("API Key") @Placement(order = 2) @Example("your_api_key_here") String apiKey) throws IOException { + LOGGER.info("Performing Google search for query: " + query); + + OkHttpClient client = new OkHttpClient().newBuilder().build(); + okhttp3.MediaType mediaType = okhttp3.MediaType.parse("application/json"); + RequestBody body = RequestBody.create("{\"q\":\"" + query + "\"}", mediaType); + Request request = new Request.Builder() + .url("https://google.serper.dev/search") + .method("POST", body) + .addHeader("X-API-KEY", apiKey) + .addHeader("Content-Type", "application/json") + .build(); + Response response = client.newCall(request).execute(); + + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response); + } + + String responseBody = response.body().string(); + JSONObject jsonResponse = new JSONObject(responseBody); + + return jsonResponse.toString(4); // Pretty print with an indentation of 4 spaces + } +} diff --git a/src/main/java/com/mule/mulechain/crawler/internal/helpers/crawlingHelper.java b/src/main/java/com/mule/mulechain/crawler/internal/helpers/crawlingHelper.java index d7dfb4b..457b4a0 100644 --- a/src/main/java/com/mule/mulechain/crawler/internal/helpers/crawlingHelper.java +++ b/src/main/java/com/mule/mulechain/crawler/internal/helpers/crawlingHelper.java @@ -6,6 +6,9 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +42,39 @@ public static Document getDocument(String url) throws IOException { return document; } + public static Document getDocumentDynamic(String url) throws Exception { + + Document document = null; + // Set ChromeOptions to enable headless mode + ChromeOptions options = new ChromeOptions(); + options.addArguments("--headless"); // Run in headless mode + options.addArguments("--disable-gpu"); // Disable GPU acceleration (optional) + options.addArguments("--no-sandbox"); // Recommended for headless mode in Docker or CI environments + options.addArguments("--disable-dev-shm-usage"); // Recommended for limited resources + options.addArguments("--allow-running-insecure-content"); // Allow HTTP content on HTTPS pages + // Initialize WebDriver + WebDriver driver = new ChromeDriver(options); + + + try { + + // Load the dynamic page + driver.get(url); + // Retrieve the page source and parse with Jsoup + String pageSource = driver.getPageSource(); + document = Jsoup.parse(pageSource, url); + } + catch (Exception e) { + LOGGER.error("Error in loading dynamic content: " + e.toString()); + throw e; + } + finally { + driver.quit(); + } + + return document; + } + public static String extractFileNameFromUrl(String url) { // Extract the filename from the URL path @@ -131,11 +167,14 @@ public static Map getPageInsights(Document document, List0 millisecs + if (delayMillis > 0) { + // Add delay before fetching the next URL + try { + LOGGER.info("Adding delay of " + delayMillis + " ms before fetching contents for the next URL."); + Thread.sleep(delayMillis); + } catch (InterruptedException e) { + LOGGER.error("Thread interrupted during delay: " + e.getMessage()); + Thread.currentThread().interrupt(); // Ensure thread interruption status is reset + } + } + } } diff --git a/target/docs/mac-webcrawler-documentation.adoc b/target/docs/mac-webcrawler-documentation.adoc new file mode 100644 index 0000000..71201a1 --- /dev/null +++ b/target/docs/mac-webcrawler-documentation.adoc @@ -0,0 +1,329 @@ +:toc: left +:toc-title: MAC WebCrawler Module +:toclevels: 2 +:last-update-label!: +:docinfo: +:source-highlighter: coderay +:icons: font + + += MAC WebCrawler Module Documentation Reference + ++++ +This is the main class of an extension, is the entry point from which configurations, connection providers, operations and sources are going to be declared. ++++ + + +== Configurations +--- +[[config]] +=== Config + ++++ +Default configuration ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +|Name | String | The name for this configuration. Connectors reference the configuration with this name. | | *x*{nbsp} +| Tag List a| Array of String | | | {nbsp} +| Expiration Policy a| <> | +++Configures the minimum amount of time that a dynamic configuration instance can remain idle before the runtime considers it eligible for expiration. This does not mean that the platform will expire the instance at the exact moment that it becomes eligible. The runtime will actually purge the instances when it sees it fit.+++ | | {nbsp} +|====================== + + +==== Associated Operations +* <> {nbsp} +* <> {nbsp} +* <> {nbsp} + + + +== Operations + +[[Crawl-website]] +=== Crawl Website +`` + ++++ +Crawl a website at a specified depth and fetch contents. Specify tags and classes in the configuration to fetch contents from those elements only. ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +| Configuration | String | The name of the configuration to use. | | *x*{nbsp} +| Website URL a| String | | | *x*{nbsp} +| Restrict Crawl under URL a| Boolean | | +++false+++ | {nbsp} +| Dynamic Content Retrieval a| Boolean | | +++false+++ | {nbsp} +| Maximum Depth a| Number | | | *x*{nbsp} +| Delay (millisecs) a| Number | | | *x*{nbsp} +| Retrieve Meta Tags a| Boolean | | +++false+++ | {nbsp} +| Download Images a| Boolean | | +++false+++ | {nbsp} +| Download Location a| String | | | *x*{nbsp} +| Output Mime Type a| String | +++The mime type of the payload that this operation outputs.+++ | | {nbsp} +| Output Encoding a| String | +++The encoding of the payload that this operation outputs.+++ | | {nbsp} +| Streaming Strategy a| * <> +* <> +* <> | +++Configure if repeatable streams should be used and their behaviour+++ | | {nbsp} +| Target Variable a| String | +++The name of a variable on which the operation's output will be placed+++ | | {nbsp} +| Target Value a| String | +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ | +++#[payload]+++ | {nbsp} +|====================== + +==== Output +[cols=".^50%,.^50%"] +|====================== +| *Type* a| Binary +|====================== + +==== For Configurations. +* <> {nbsp} + + + +[[Get-page-content]] +=== Get Page Content +`` + ++++ +Get contents of a web page. Content is returned in the resulting payload. ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +| Configuration | String | The name of the configuration to use. | | *x*{nbsp} +| Page Url a| String | | | *x*{nbsp} +| Output Mime Type a| String | +++The mime type of the payload that this operation outputs.+++ | | {nbsp} +| Output Encoding a| String | +++The encoding of the payload that this operation outputs.+++ | | {nbsp} +| Streaming Strategy a| * <> +* <> +* <> | +++Configure if repeatable streams should be used and their behaviour+++ | | {nbsp} +| Target Variable a| String | +++The name of a variable on which the operation's output will be placed+++ | | {nbsp} +| Target Value a| String | +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ | +++#[payload]+++ | {nbsp} +|====================== + +==== Output +[cols=".^50%,.^50%"] +|====================== +| *Type* a| Binary +|====================== + +==== For Configurations. +* <> {nbsp} + + + +[[Get-page-insights]] +=== Get Page Insights +`` + ++++ +Get insights from a web page including links, word count, number of occurrences of elements. Restrict insights to specific elements in the configuration. ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +| Configuration | String | The name of the configuration to use. | | *x*{nbsp} +| Page Url a| String | | | *x*{nbsp} +| Output Mime Type a| String | +++The mime type of the payload that this operation outputs.+++ | | {nbsp} +| Output Encoding a| String | +++The encoding of the payload that this operation outputs.+++ | | {nbsp} +| Streaming Strategy a| * <> +* <> +* <> | +++Configure if repeatable streams should be used and their behaviour+++ | | {nbsp} +| Target Variable a| String | +++The name of a variable on which the operation's output will be placed+++ | | {nbsp} +| Target Value a| String | +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ | +++#[payload]+++ | {nbsp} +|====================== + +==== Output +[cols=".^50%,.^50%"] +|====================== +| *Type* a| Binary +|====================== + +==== For Configurations. +* <> {nbsp} + + + +[[Download-image]] +=== Download Image +`` + ++++ +Download all images from a web page, or download a single image at the specified link. ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +| Page Or Image URL a| String | | | *x*{nbsp} +| Download Location a| String | | | *x*{nbsp} +| Output Mime Type a| String | +++The mime type of the payload that this operation outputs.+++ | | {nbsp} +| Output Encoding a| String | +++The encoding of the payload that this operation outputs.+++ | | {nbsp} +| Streaming Strategy a| * <> +* <> +* <> | +++Configure if repeatable streams should be used and their behaviour+++ | | {nbsp} +| Target Variable a| String | +++The name of a variable on which the operation's output will be placed+++ | | {nbsp} +| Target Value a| String | +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ | +++#[payload]+++ | {nbsp} +|====================== + +==== Output +[cols=".^50%,.^50%"] +|====================== +| *Type* a| Binary +|====================== + + + + +[[Generate-sitemap]] +=== Generate Sitemap +`` + ++++ +Retrieve internal links as a site map from the specified url and depth. ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +| Website URL a| String | | | *x*{nbsp} +| Maximum Depth a| Number | | | *x*{nbsp} +| Delay (millisecs) a| Number | | | *x*{nbsp} +| Output Mime Type a| String | +++The mime type of the payload that this operation outputs.+++ | | {nbsp} +| Output Encoding a| String | +++The encoding of the payload that this operation outputs.+++ | | {nbsp} +| Streaming Strategy a| * <> +* <> +* <> | +++Configure if repeatable streams should be used and their behaviour+++ | | {nbsp} +| Target Variable a| String | +++The name of a variable on which the operation's output will be placed+++ | | {nbsp} +| Target Value a| String | +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ | +++#[payload]+++ | {nbsp} +|====================== + +==== Output +[cols=".^50%,.^50%"] +|====================== +| *Type* a| Binary +|====================== + + + + +[[Get-page-meta-tags]] +=== Get Page Meta Tags +`` + ++++ +Fetch the meta tags from a web page. ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +| Page URL a| String | | | *x*{nbsp} +| Output Mime Type a| String | +++The mime type of the payload that this operation outputs.+++ | | {nbsp} +| Output Encoding a| String | +++The encoding of the payload that this operation outputs.+++ | | {nbsp} +| Streaming Strategy a| * <> +* <> +* <> | +++Configure if repeatable streams should be used and their behaviour+++ | | {nbsp} +| Target Variable a| String | +++The name of a variable on which the operation's output will be placed+++ | | {nbsp} +| Target Value a| String | +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ | +++#[payload]+++ | {nbsp} +|====================== + +==== Output +[cols=".^50%,.^50%"] +|====================== +| *Type* a| Binary +|====================== + + + + +[[Google-search]] +=== Google Search +`` + ++++ +Perform a Google search using the SERP API. ++++ + +==== Parameters +[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"] +|====================== +| Name | Type | Description | Default Value | Required +| Search Query a| String | | | *x*{nbsp} +| API Key a| String | | | *x*{nbsp} +| Output Mime Type a| String | +++The mime type of the payload that this operation outputs.+++ | | {nbsp} +| Target Variable a| String | +++The name of a variable on which the operation's output will be placed+++ | | {nbsp} +| Target Value a| String | +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ | +++#[payload]+++ | {nbsp} +|====================== + +==== Output +[cols=".^50%,.^50%"] +|====================== +| *Type* a| String +|====================== + + + + + +== Types +[[ExpirationPolicy]] +=== Expiration Policy + +[cols=".^20%,.^25%,.^30%,.^15%,.^10%", options="header"] +|====================== +| Field | Type | Description | Default Value | Required +| Max Idle Time a| Number | A scalar time value for the maximum amount of time a dynamic configuration instance should be allowed to be idle before it's considered eligible for expiration | | +| Time Unit a| Enumeration, one of: + +** NANOSECONDS +** MICROSECONDS +** MILLISECONDS +** SECONDS +** MINUTES +** HOURS +** DAYS | A time unit that qualifies the maxIdleTime attribute | | +|====================== + +[[repeatable-in-memory-stream]] +=== Repeatable In Memory Stream + +[cols=".^20%,.^25%,.^30%,.^15%,.^10%", options="header"] +|====================== +| Field | Type | Description | Default Value | Required +| Initial Buffer Size a| Number | This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize. | | +| Buffer Size Increment a| Number | This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full. | | +| Max Buffer Size a| Number | This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit. | | +| Buffer Unit a| Enumeration, one of: + +** BYTE +** KB +** MB +** GB | The unit in which all these attributes are expressed | | +|====================== + +[[repeatable-file-store-stream]] +=== Repeatable File Store Stream + +[cols=".^20%,.^25%,.^30%,.^15%,.^10%", options="header"] +|====================== +| Field | Type | Description | Default Value | Required +| In Memory Size a| Number | Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk. | | +| Buffer Unit a| Enumeration, one of: + +** BYTE +** KB +** MB +** GB | The unit in which maxInMemorySize is expressed | | +|====================== + diff --git a/target/docs/mac-webcrawler-documentation.html b/target/docs/mac-webcrawler-documentation.html new file mode 100644 index 0000000..727f8a6 --- /dev/null +++ b/target/docs/mac-webcrawler-documentation.html @@ -0,0 +1,3327 @@ + + + + + + + +MAC WebCrawler Module Documentation Reference + + + + + + +
+
+
+
+

+This is the main class of an extension, is the entry point from which configurations, connection providers, operations and sources are going to be declared. +

+
+
+
+
+

Configurations

+
+
+
+

Config

+
+

+Default configuration +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Name

String

The name for this configuration. Connectors reference the configuration with this name.

x 

Tag List

+

Array of String

+

 

Expiration Policy

Configures the minimum amount of time that a dynamic configuration instance can remain idle before the runtime considers it eligible for expiration. This does not mean that the platform will expire the instance at the exact moment that it becomes eligible. The runtime will actually purge the instances when it sees it fit.

 

+
+
+

Associated Operations

+
+ +
+
+
+
+
+
+

Operations

+
+
+

Crawl Website

+
+

<mac-web-crawler:crawl-website>

+
+
+

+Crawl a website at a specified depth and fetch contents. Specify tags and classes in the configuration to fetch contents from those elements only. +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Configuration

String

The name of the configuration to use.

x 

Website URL

+

String

+

x 

Restrict Crawl under URL

+

Boolean

+

false

 

Dynamic Content Retrieval

+

Boolean

+

false

 

Maximum Depth

+

Number

+

x 

Delay (millisecs)

+

Number

+

x 

Retrieve Meta Tags

+

Boolean

+

false

 

Download Images

+

Boolean

+

false

 

Download Location

+

String

+

x 

Output Mime Type

+

String

+

The mime type of the payload that this operation outputs.

 

Output Encoding

+

String

+

The encoding of the payload that this operation outputs.

 

Streaming Strategy

Configure if repeatable streams should be used and their behaviour

 

Target Variable

+

String

+

The name of a variable on which the operation's output will be placed

 

Target Value

+

String

+

An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable

#[payload]

 

+
+
+

Output

+ ++++ + + + + + + +

Type

+

Binary

+
+
+
+

For Configurations.

+
+ +
+
+
+
+

Get Page Content

+
+

<mac-web-crawler:get-page-content>

+
+
+

+Get contents of a web page. Content is returned in the resulting payload. +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Configuration

String

The name of the configuration to use.

x 

Page Url

+

String

+

x 

Output Mime Type

+

String

+

The mime type of the payload that this operation outputs.

 

Output Encoding

+

String

+

The encoding of the payload that this operation outputs.

 

Streaming Strategy

Configure if repeatable streams should be used and their behaviour

 

Target Variable

+

String

+

The name of a variable on which the operation's output will be placed

 

Target Value

+

String

+

An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable

#[payload]

 

+
+
+

Output

+ ++++ + + + + + + +

Type

+

Binary

+
+
+
+

For Configurations.

+
+ +
+
+
+
+

Get Page Insights

+
+

<mac-web-crawler:get-page-insights>

+
+
+

+Get insights from a web page including links, word count, number of occurrences of elements. Restrict insights to specific elements in the configuration. +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Configuration

String

The name of the configuration to use.

x 

Page Url

+

String

+

x 

Output Mime Type

+

String

+

The mime type of the payload that this operation outputs.

 

Output Encoding

+

String

+

The encoding of the payload that this operation outputs.

 

Streaming Strategy

Configure if repeatable streams should be used and their behaviour

 

Target Variable

+

String

+

The name of a variable on which the operation's output will be placed

 

Target Value

+

String

+

An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable

#[payload]

 

+
+
+

Output

+ ++++ + + + + + + +

Type

+

Binary

+
+
+
+

For Configurations.

+
+ +
+
+
+
+

Download Image

+
+

<mac-web-crawler:download-image>

+
+
+

+Download all images from a web page, or download a single image at the specified link. +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Page Or Image URL

+

String

+

x 

Download Location

+

String

+

x 

Output Mime Type

+

String

+

The mime type of the payload that this operation outputs.

 

Output Encoding

+

String

+

The encoding of the payload that this operation outputs.

 

Streaming Strategy

Configure if repeatable streams should be used and their behaviour

 

Target Variable

+

String

+

The name of a variable on which the operation's output will be placed

 

Target Value

+

String

+

An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable

#[payload]

 

+
+
+

Output

+ ++++ + + + + + + +

Type

+

Binary

+
+
+
+
+

Generate Sitemap

+
+

<mac-web-crawler:generate-sitemap>

+
+
+

+Retrieve internal links as a site map from the specified url and depth. +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Website URL

+

String

+

x 

Maximum Depth

+

Number

+

x 

Delay (millisecs)

+

Number

+

x 

Output Mime Type

+

String

+

The mime type of the payload that this operation outputs.

 

Output Encoding

+

String

+

The encoding of the payload that this operation outputs.

 

Streaming Strategy

Configure if repeatable streams should be used and their behaviour

 

Target Variable

+

String

+

The name of a variable on which the operation's output will be placed

 

Target Value

+

String

+

An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable

#[payload]

 

+
+
+

Output

+ ++++ + + + + + + +

Type

+

Binary

+
+
+
+
+

Get Page Meta Tags

+
+

<mac-web-crawler:get-page-meta-tags>

+
+
+

+Fetch the meta tags from a web page. +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Page URL

+

String

+

x 

Output Mime Type

+

String

+

The mime type of the payload that this operation outputs.

 

Output Encoding

+

String

+

The encoding of the payload that this operation outputs.

 

Streaming Strategy

Configure if repeatable streams should be used and their behaviour

 

Target Variable

+

String

+

The name of a variable on which the operation's output will be placed

 

Target Value

+

String

+

An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable

#[payload]

 

+
+
+

Output

+ ++++ + + + + + + +

Type

+

Binary

+
+
+
+
+ +
+

<mac-web-crawler:google-search>

+
+
+

+Perform a Google search using the SERP API. +

+
+
+

Parameters

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault ValueRequired

Search Query

+

String

+

x 

API Key

+

String

+

x 

Output Mime Type

+

String

+

The mime type of the payload that this operation outputs.

 

Target Variable

+

String

+

The name of a variable on which the operation's output will be placed

 

Target Value

+

String

+

An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable

#[payload]

 

+
+
+

Output

+ ++++ + + + + + + +

Type

+

String

+
+
+
+
+
+
+

Types

+
+
+

Expiration Policy

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldTypeDescriptionDefault ValueRequired

Max Idle Time

+

Number

+

A scalar time value for the maximum amount of time a dynamic configuration instance should be allowed to be idle before it’s considered eligible for expiration

Time Unit

+

Enumeration, one of:

+
+
+
    +
  • +

    NANOSECONDS

    +
  • +
  • +

    MICROSECONDS

    +
  • +
  • +

    MILLISECONDS

    +
  • +
  • +

    SECONDS

    +
  • +
  • +

    MINUTES

    +
  • +
  • +

    HOURS

    +
  • +
  • +

    DAYS

    +
  • +
+

A time unit that qualifies the maxIdleTime attribute

+
+
+

Repeatable In Memory Stream

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldTypeDescriptionDefault ValueRequired

Initial Buffer Size

+

Number

+

This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize.

Buffer Size Increment

+

Number

+

This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full.

Max Buffer Size

+

Number

+

This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit.

Buffer Unit

+

Enumeration, one of:

+
+
+
    +
  • +

    BYTE

    +
  • +
  • +

    KB

    +
  • +
  • +

    MB

    +
  • +
  • +

    GB

    +
  • +
+

The unit in which all these attributes are expressed

+
+
+

Repeatable File Store Stream

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldTypeDescriptionDefault ValueRequired

In Memory Size

+

Number

+

Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk.

Buffer Unit

+

Enumeration, one of:

+
+
+
    +
  • +

    BYTE

    +
  • +
  • +

    KB

    +
  • +
  • +

    MB

    +
  • +
  • +

    GB

    +
  • +
+

The unit in which maxInMemorySize is expressed

+
+
+
+
+ + + \ No newline at end of file diff --git a/target/temporal-extension-model.json b/target/temporal-extension-model.json new file mode 100644 index 0000000..dde5b94 --- /dev/null +++ b/target/temporal-extension-model.json @@ -0,0 +1,3365 @@ +{ + "name": "MAC WebCrawler", + "description": "This is the main class of an extension, is the entry point from which configurations, connection providers, operations and sources are going to be declared.", + "version": "0.1.25-SNAPSHOT", + "vendor": "Mulesoft", + "category": "COMMUNITY", + "xmlDsl": { + "prefix": "mac-web-crawler", + "namespace": "http://www.mulesoft.org/schema/mule/mac-web-crawler", + "schemaLocation": "http://www.mulesoft.org/schema/mule/mac-web-crawler/current/mule-mac-web-crawler.xsd", + "schemaVersion": "0.1.25-SNAPSHOT", + "xsdFileName": "mule-mac-web-crawler.xsd" + }, + "resources": [], + "subTypes": [], + "privilegedPackages": [], + "privilegedArtifacts": [], + "externalLibraries": [], + "importedTypes": [], + "configurations": [ + { + "stereotype": { + "type": "CONFIG", + "namespace": "MAC-WEB-CRAWLER", + "parent": { + "type": "MODULE_CONFIG", + "namespace": "MULE" + } + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "Array", + "annotations": { + "classInformation": { + "classname": "java.util.List", + "hasDefaultConstructor": false, + "isInterface": true, + "isInstantiable": false, + "isAbstract": true, + "isFinal": false, + "implementedInterfaces": [ + "java.util.Collection" + ], + "parent": "", + "genericTypes": [ + "java.lang.String" + ], + "isMap": false + } + }, + "item": { + "type": "String" + } + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "allowedStereotypeModels": [], + "name": "tags", + "displayModel": { + "displayName": "Tag List" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Object", + "annotations": { + "typeId": "org.mule.runtime.extension.api.runtime.ExpirationPolicy", + "description": { + "value": "Configures the minimum amount of time that a dynamic configuration instance can remain idle before the runtime considers it eligible for expiration. This does not mean that the platform will expire the instance at the exact moment that it becomes eligible. The runtime will actually purge the instances when it sees it fit." + }, + "typeDsl": { + "allowInlineDefinition": true, + "allowTopLevelDefinition": false + } + }, + "fields": [ + { + "key": { + "name": "maxIdleTime" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "classInformation": { + "classname": "java.lang.Long", + "hasDefaultConstructor": false, + "isInterface": false, + "isInstantiable": false, + "isAbstract": false, + "isFinal": true, + "implementedInterfaces": [ + "java.lang.Comparable", + "java.lang.constant.Constable", + "java.lang.constant.ConstantDesc" + ], + "parent": "java.lang.Number", + "genericTypes": [], + "isMap": false + }, + "default": "5" + } + }, + "annotations": { + "description": { + "value": "A scalar time value for the maximum amount of time a dynamic configuration instance should be allowed to be idle before it\u0027s considered eligible for expiration" + }, + "expressionSupport": { + "value": "NOT_SUPPORTED" + } + } + }, + { + "key": { + "name": "timeUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "NANOSECONDS", + "MICROSECONDS", + "MILLISECONDS", + "SECONDS", + "MINUTES", + "HOURS", + "DAYS" + ] + }, + "default": "MINUTES" + } + }, + "annotations": { + "description": { + "value": "A time unit that qualifies the maxIdleTime attribute" + }, + "expressionSupport": { + "value": "NOT_SUPPORTED" + } + } + } + ] + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": false, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "expirationPolicy", + "description": "Configures the minimum amount of time that a dynamic configuration instance can remain idle before the runtime considers it eligible for expiration. This does not mean that the platform will expire the instance at the exact moment that it becomes eligible. The runtime will actually purge the instances when it sees it fit.", + "modelProperties": { + "org.mule.runtime.extension.api.property.QNameModelProperty": { + "value": { + "namespaceURI": "http://www.mulesoft.org/schema/mule/core", + "localPart": "expiration-policy", + "prefix": "mule" + } + }, + "org.mule.runtime.extension.api.property.InfrastructureParameterModelProperty": { + "sequence": 4 + } + } + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + } + ], + "externalLibraryModels": [], + "operations": [ + { + "blocking": true, + "executionType": "CPU_LITE", + "output": { + "type": { + "format": "json", + "type": "Binary", + "annotations": { + "classInformation": { + "classname": "java.io.InputStream", + "hasDefaultConstructor": true, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": false, + "implementedInterfaces": [ + "java.io.Closeable" + ], + "parent": "", + "genericTypes": [], + "isMap": false + } + } + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "outputAttributes": { + "type": { + "format": "java", + "type": "Void" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "transactional": false, + "requiresConnection": false, + "supportsStreaming": true, + "notifications": [], + "nestedComponents": [], + "errors": [], + "stereotype": { + "type": "PROCESSOR", + "namespace": "MULE" + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "url", + "displayModel": { + "displayName": "Website URL", + "example": "https://mac-project.ai/docs" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Boolean", + "annotations": { + "typeId": "boolean" + } + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "defaultValue": "false", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "restrictToPath", + "displayModel": { + "displayName": "Restrict Crawl under URL", + "example": "False" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Boolean", + "annotations": { + "typeId": "boolean" + } + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "defaultValue": "false", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 3, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "dynamicContent", + "displayModel": { + "displayName": "Dynamic Content Retrieval", + "example": "False" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Number", + "annotations": { + "classInformation": { + "classname": "int", + "hasDefaultConstructor": false, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": true, + "implementedInterfaces": [], + "parent": "", + "genericTypes": [], + "isMap": false + }, + "int": {} + } + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 4, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "maxDepth", + "displayModel": { + "displayName": "Maximum Depth", + "example": "2" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Number", + "annotations": { + "classInformation": { + "classname": "int", + "hasDefaultConstructor": false, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": true, + "implementedInterfaces": [], + "parent": "", + "genericTypes": [], + "isMap": false + }, + "int": {} + } + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 5, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "delayMillis", + "displayModel": { + "displayName": "Delay (millisecs)", + "example": "0" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Boolean", + "annotations": { + "typeId": "boolean" + } + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "defaultValue": "false", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 6, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "getMetaTags", + "displayModel": { + "displayName": "Retrieve Meta Tags", + "example": "False" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Boolean", + "annotations": { + "typeId": "boolean" + } + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "defaultValue": "false", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 7, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "downloadImages", + "displayModel": { + "displayName": "Download Images", + "example": "False" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 8, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "downloadPath", + "displayModel": { + "displayName": "Download Location", + "example": "/users/mulesoft/downloads" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 9, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputMimeType", + "description": "The mime type of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 10, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputEncoding", + "description": "The encoding of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Union", + "annotations": { + "typeId": "ByteStreamingStrategy" + }, + "of": [ + { + "type": "Object", + "annotations": { + "typeId": "repeatable-in-memory-stream" + }, + "fields": [ + { + "key": { + "name": "initialBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize." + } + } + }, + { + "key": { + "name": "bufferSizeIncrement" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full." + } + } + }, + { + "key": { + "name": "maxBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "1024" + } + }, + "annotations": { + "description": { + "value": "This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which all these attributes are expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "repeatable-file-store-stream" + }, + "fields": [ + { + "key": { + "name": "inMemorySize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which maxInMemorySize is expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "non-repeatable-stream", + "description": { + "value": "This configuration allows the input stream to be read only once. It will not allow to seek randomly which will limit the transformations that DW can perform on this stream. Use this option for use cases which just require moving data around from one system to another to get optimum performance." + } + }, + "fields": [] + } + ] + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": false, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 11, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "streamingStrategy", + "description": "Configure if repeatable streams should be used and their behaviour", + "modelProperties": { + "org.mule.runtime.extension.api.property.QNameModelProperty": { + "value": { + "namespaceURI": "http://www.mulesoft.org/schema/mule/core", + "localPart": "abstract-byte-streaming-strategy", + "prefix": "mule" + } + }, + "org.mule.runtime.extension.api.property.InfrastructureParameterModelProperty": { + "sequence": 2 + } + } + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + }, + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 12, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "target", + "displayModel": { + "displayName": "Target Variable" + }, + "description": "The name of a variable on which the operation\u0027s output will be placed", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "REQUIRED", + "defaultValue": "#[payload]", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 13, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "targetValue", + "displayModel": { + "displayName": "Target Value" + }, + "description": "An expression that will be evaluated against the operation\u0027s output and the outcome of that expression will be stored in the target variable", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2 + }, + "showInDsl": false, + "name": "Output", + "description": "", + "modelProperties": {} + } + ], + "name": "Crawl-website", + "description": "Crawl a website at a specified depth and fetch contents. Specify tags and classes in the configuration to fetch contents from those elements only.", + "modelProperties": {}, + "kind": "operation" + }, + { + "blocking": true, + "executionType": "CPU_LITE", + "output": { + "type": { + "format": "json", + "type": "Binary", + "annotations": { + "classInformation": { + "classname": "java.io.InputStream", + "hasDefaultConstructor": true, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": false, + "implementedInterfaces": [ + "java.io.Closeable" + ], + "parent": "", + "genericTypes": [], + "isMap": false + } + } + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "outputAttributes": { + "type": { + "format": "java", + "type": "Void" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "transactional": false, + "requiresConnection": false, + "supportsStreaming": true, + "notifications": [], + "nestedComponents": [], + "errors": [], + "stereotype": { + "type": "PROCESSOR", + "namespace": "MULE" + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "url", + "displayModel": { + "displayName": "Page Url", + "example": "https://mac-project.ai/docs" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputMimeType", + "description": "The mime type of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 3, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputEncoding", + "description": "The encoding of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Union", + "annotations": { + "typeId": "ByteStreamingStrategy" + }, + "of": [ + { + "type": "Object", + "annotations": { + "typeId": "repeatable-in-memory-stream" + }, + "fields": [ + { + "key": { + "name": "initialBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize." + } + } + }, + { + "key": { + "name": "bufferSizeIncrement" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full." + } + } + }, + { + "key": { + "name": "maxBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "1024" + } + }, + "annotations": { + "description": { + "value": "This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which all these attributes are expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "repeatable-file-store-stream" + }, + "fields": [ + { + "key": { + "name": "inMemorySize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which maxInMemorySize is expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "non-repeatable-stream", + "description": { + "value": "This configuration allows the input stream to be read only once. It will not allow to seek randomly which will limit the transformations that DW can perform on this stream. Use this option for use cases which just require moving data around from one system to another to get optimum performance." + } + }, + "fields": [] + } + ] + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": false, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 4, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "streamingStrategy", + "description": "Configure if repeatable streams should be used and their behaviour", + "modelProperties": { + "org.mule.runtime.extension.api.property.QNameModelProperty": { + "value": { + "namespaceURI": "http://www.mulesoft.org/schema/mule/core", + "localPart": "abstract-byte-streaming-strategy", + "prefix": "mule" + } + }, + "org.mule.runtime.extension.api.property.InfrastructureParameterModelProperty": { + "sequence": 2 + } + } + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + }, + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 5, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "target", + "displayModel": { + "displayName": "Target Variable" + }, + "description": "The name of a variable on which the operation\u0027s output will be placed", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "REQUIRED", + "defaultValue": "#[payload]", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 6, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "targetValue", + "displayModel": { + "displayName": "Target Value" + }, + "description": "An expression that will be evaluated against the operation\u0027s output and the outcome of that expression will be stored in the target variable", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2 + }, + "showInDsl": false, + "name": "Output", + "description": "", + "modelProperties": {} + } + ], + "name": "Get-page-content", + "description": "Get contents of a web page. Content is returned in the resulting payload.", + "modelProperties": {}, + "kind": "operation" + }, + { + "blocking": true, + "executionType": "CPU_LITE", + "output": { + "type": { + "format": "json", + "type": "Binary", + "annotations": { + "classInformation": { + "classname": "java.io.InputStream", + "hasDefaultConstructor": true, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": false, + "implementedInterfaces": [ + "java.io.Closeable" + ], + "parent": "", + "genericTypes": [], + "isMap": false + } + } + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "outputAttributes": { + "type": { + "format": "java", + "type": "Void" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "transactional": false, + "requiresConnection": false, + "supportsStreaming": true, + "notifications": [], + "nestedComponents": [], + "errors": [], + "stereotype": { + "type": "PROCESSOR", + "namespace": "MULE" + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "url", + "displayModel": { + "displayName": "Page Url", + "example": "https://mac-project.ai/docs" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputMimeType", + "description": "The mime type of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 3, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputEncoding", + "description": "The encoding of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Union", + "annotations": { + "typeId": "ByteStreamingStrategy" + }, + "of": [ + { + "type": "Object", + "annotations": { + "typeId": "repeatable-in-memory-stream" + }, + "fields": [ + { + "key": { + "name": "initialBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize." + } + } + }, + { + "key": { + "name": "bufferSizeIncrement" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full." + } + } + }, + { + "key": { + "name": "maxBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "1024" + } + }, + "annotations": { + "description": { + "value": "This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which all these attributes are expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "repeatable-file-store-stream" + }, + "fields": [ + { + "key": { + "name": "inMemorySize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which maxInMemorySize is expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "non-repeatable-stream", + "description": { + "value": "This configuration allows the input stream to be read only once. It will not allow to seek randomly which will limit the transformations that DW can perform on this stream. Use this option for use cases which just require moving data around from one system to another to get optimum performance." + } + }, + "fields": [] + } + ] + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": false, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 4, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "streamingStrategy", + "description": "Configure if repeatable streams should be used and their behaviour", + "modelProperties": { + "org.mule.runtime.extension.api.property.QNameModelProperty": { + "value": { + "namespaceURI": "http://www.mulesoft.org/schema/mule/core", + "localPart": "abstract-byte-streaming-strategy", + "prefix": "mule" + } + }, + "org.mule.runtime.extension.api.property.InfrastructureParameterModelProperty": { + "sequence": 2 + } + } + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + }, + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 5, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "target", + "displayModel": { + "displayName": "Target Variable" + }, + "description": "The name of a variable on which the operation\u0027s output will be placed", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "REQUIRED", + "defaultValue": "#[payload]", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 6, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "targetValue", + "displayModel": { + "displayName": "Target Value" + }, + "description": "An expression that will be evaluated against the operation\u0027s output and the outcome of that expression will be stored in the target variable", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2 + }, + "showInDsl": false, + "name": "Output", + "description": "", + "modelProperties": {} + } + ], + "name": "Get-page-insights", + "description": "Get insights from a web page including links, word count, number of occurrences of elements. Restrict insights to specific elements in the configuration.", + "modelProperties": {}, + "kind": "operation" + } + ], + "connectionProviders": [], + "messageSources": [], + "name": "config", + "description": "Default configuration", + "modelProperties": {} + } + ], + "operations": [ + { + "blocking": true, + "executionType": "CPU_LITE", + "output": { + "type": { + "format": "json", + "type": "Binary", + "annotations": { + "classInformation": { + "classname": "java.io.InputStream", + "hasDefaultConstructor": true, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": false, + "implementedInterfaces": [ + "java.io.Closeable" + ], + "parent": "", + "genericTypes": [], + "isMap": false + } + } + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "outputAttributes": { + "type": { + "format": "java", + "type": "Void" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "transactional": false, + "requiresConnection": false, + "supportsStreaming": true, + "notifications": [], + "nestedComponents": [], + "errors": [], + "stereotype": { + "type": "PROCESSOR", + "namespace": "MULE" + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "url", + "displayModel": { + "displayName": "Page Or Image URL", + "example": "https://mac-project.ai/docs" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "downloadPath", + "displayModel": { + "displayName": "Download Location", + "example": "/users/mulesoft/downloads" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 3, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputMimeType", + "description": "The mime type of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 4, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputEncoding", + "description": "The encoding of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Union", + "annotations": { + "typeId": "ByteStreamingStrategy" + }, + "of": [ + { + "type": "Object", + "annotations": { + "typeId": "repeatable-in-memory-stream" + }, + "fields": [ + { + "key": { + "name": "initialBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize." + } + } + }, + { + "key": { + "name": "bufferSizeIncrement" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full." + } + } + }, + { + "key": { + "name": "maxBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "1024" + } + }, + "annotations": { + "description": { + "value": "This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which all these attributes are expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "repeatable-file-store-stream" + }, + "fields": [ + { + "key": { + "name": "inMemorySize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which maxInMemorySize is expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "non-repeatable-stream", + "description": { + "value": "This configuration allows the input stream to be read only once. It will not allow to seek randomly which will limit the transformations that DW can perform on this stream. Use this option for use cases which just require moving data around from one system to another to get optimum performance." + } + }, + "fields": [] + } + ] + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": false, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 5, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "streamingStrategy", + "description": "Configure if repeatable streams should be used and their behaviour", + "modelProperties": { + "org.mule.runtime.extension.api.property.QNameModelProperty": { + "value": { + "namespaceURI": "http://www.mulesoft.org/schema/mule/core", + "localPart": "abstract-byte-streaming-strategy", + "prefix": "mule" + } + }, + "org.mule.runtime.extension.api.property.InfrastructureParameterModelProperty": { + "sequence": 2 + } + } + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + }, + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 6, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "target", + "displayModel": { + "displayName": "Target Variable" + }, + "description": "The name of a variable on which the operation\u0027s output will be placed", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "REQUIRED", + "defaultValue": "#[payload]", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 7, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "targetValue", + "displayModel": { + "displayName": "Target Value" + }, + "description": "An expression that will be evaluated against the operation\u0027s output and the outcome of that expression will be stored in the target variable", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2 + }, + "showInDsl": false, + "name": "Output", + "description": "", + "modelProperties": {} + } + ], + "name": "Download-image", + "description": "Download all images from a web page, or download a single image at the specified link.", + "modelProperties": {}, + "kind": "operation" + }, + { + "blocking": true, + "executionType": "CPU_LITE", + "output": { + "type": { + "format": "json", + "type": "Binary", + "annotations": { + "classInformation": { + "classname": "java.io.InputStream", + "hasDefaultConstructor": true, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": false, + "implementedInterfaces": [ + "java.io.Closeable" + ], + "parent": "", + "genericTypes": [], + "isMap": false + } + } + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "outputAttributes": { + "type": { + "format": "java", + "type": "Void" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "transactional": false, + "requiresConnection": false, + "supportsStreaming": true, + "notifications": [], + "nestedComponents": [], + "errors": [], + "stereotype": { + "type": "PROCESSOR", + "namespace": "MULE" + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "url", + "displayModel": { + "displayName": "Website URL", + "example": "https://mac-project.ai/docs" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Number", + "annotations": { + "classInformation": { + "classname": "int", + "hasDefaultConstructor": false, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": true, + "implementedInterfaces": [], + "parent": "", + "genericTypes": [], + "isMap": false + }, + "int": {} + } + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "maxDepth", + "displayModel": { + "displayName": "Maximum Depth", + "example": "2" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Number", + "annotations": { + "classInformation": { + "classname": "int", + "hasDefaultConstructor": false, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": true, + "implementedInterfaces": [], + "parent": "", + "genericTypes": [], + "isMap": false + }, + "int": {} + } + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 3, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "delayMillis", + "displayModel": { + "displayName": "Delay (millisecs)", + "example": "0" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 4, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputMimeType", + "description": "The mime type of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 5, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputEncoding", + "description": "The encoding of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Union", + "annotations": { + "typeId": "ByteStreamingStrategy" + }, + "of": [ + { + "type": "Object", + "annotations": { + "typeId": "repeatable-in-memory-stream" + }, + "fields": [ + { + "key": { + "name": "initialBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize." + } + } + }, + { + "key": { + "name": "bufferSizeIncrement" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full." + } + } + }, + { + "key": { + "name": "maxBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "1024" + } + }, + "annotations": { + "description": { + "value": "This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which all these attributes are expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "repeatable-file-store-stream" + }, + "fields": [ + { + "key": { + "name": "inMemorySize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which maxInMemorySize is expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "non-repeatable-stream", + "description": { + "value": "This configuration allows the input stream to be read only once. It will not allow to seek randomly which will limit the transformations that DW can perform on this stream. Use this option for use cases which just require moving data around from one system to another to get optimum performance." + } + }, + "fields": [] + } + ] + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": false, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 6, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "streamingStrategy", + "description": "Configure if repeatable streams should be used and their behaviour", + "modelProperties": { + "org.mule.runtime.extension.api.property.QNameModelProperty": { + "value": { + "namespaceURI": "http://www.mulesoft.org/schema/mule/core", + "localPart": "abstract-byte-streaming-strategy", + "prefix": "mule" + } + }, + "org.mule.runtime.extension.api.property.InfrastructureParameterModelProperty": { + "sequence": 2 + } + } + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + }, + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 7, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "target", + "displayModel": { + "displayName": "Target Variable" + }, + "description": "The name of a variable on which the operation\u0027s output will be placed", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "REQUIRED", + "defaultValue": "#[payload]", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 8, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "targetValue", + "displayModel": { + "displayName": "Target Value" + }, + "description": "An expression that will be evaluated against the operation\u0027s output and the outcome of that expression will be stored in the target variable", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2 + }, + "showInDsl": false, + "name": "Output", + "description": "", + "modelProperties": {} + } + ], + "name": "Generate-sitemap", + "description": "Retrieve internal links as a site map from the specified url and depth.", + "modelProperties": {}, + "kind": "operation" + }, + { + "blocking": true, + "executionType": "CPU_LITE", + "output": { + "type": { + "format": "json", + "type": "Binary", + "annotations": { + "classInformation": { + "classname": "java.io.InputStream", + "hasDefaultConstructor": true, + "isInterface": false, + "isInstantiable": false, + "isAbstract": true, + "isFinal": false, + "implementedInterfaces": [ + "java.io.Closeable" + ], + "parent": "", + "genericTypes": [], + "isMap": false + } + } + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "outputAttributes": { + "type": { + "format": "java", + "type": "Void" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "transactional": false, + "requiresConnection": false, + "supportsStreaming": true, + "notifications": [], + "nestedComponents": [], + "errors": [], + "stereotype": { + "type": "PROCESSOR", + "namespace": "MULE" + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "url", + "displayModel": { + "displayName": "Page URL", + "example": "https://mac-project.ai/docs" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputMimeType", + "description": "The mime type of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 3, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputEncoding", + "description": "The encoding of the payload that this operation outputs.", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "Union", + "annotations": { + "typeId": "ByteStreamingStrategy" + }, + "of": [ + { + "type": "Object", + "annotations": { + "typeId": "repeatable-in-memory-stream" + }, + "fields": [ + { + "key": { + "name": "initialBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is the amount of memory that will be allocated in order to consume the stream and provide random access to it. If the stream contains more data than can be fit into this buffer, then it will be expanded by according to the bufferSizeIncrement attribute, with an upper limit of maxInMemorySize." + } + } + }, + { + "key": { + "name": "bufferSizeIncrement" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "This is by how much will be buffer size by expanded if it exceeds its initial size. Setting a value of zero or lower will mean that the buffer should not expand, meaning that a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised when the buffer gets full." + } + } + }, + { + "key": { + "name": "maxBufferSize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "1024" + } + }, + "annotations": { + "description": { + "value": "This is the maximum amount of memory that will be used. If more than that is used then a STREAM_MAXIMUM_SIZE_EXCEEDED error will be raised. A value lower or equal to zero means no limit." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which all these attributes are expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "repeatable-file-store-stream" + }, + "fields": [ + { + "key": { + "name": "inMemorySize" + }, + "model": { + "type": "Number", + "annotations": { + "int": {}, + "default": "512" + } + }, + "annotations": { + "description": { + "value": "Defines the maximum memory that the stream should use to keep data in memory. If more than that is consumed then it will start to buffer the content on disk." + } + } + }, + { + "key": { + "name": "bufferUnit" + }, + "model": { + "type": "String", + "annotations": { + "enum": { + "type": "[Ljava.lang.String;", + "values": [ + "BYTE", + "KB", + "MB", + "GB" + ] + }, + "default": "KB" + } + }, + "annotations": { + "description": { + "value": "The unit in which maxInMemorySize is expressed" + } + } + } + ] + }, + { + "type": "Object", + "annotations": { + "typeId": "non-repeatable-stream", + "description": { + "value": "This configuration allows the input stream to be read only once. It will not allow to seek randomly which will limit the transformations that DW can perform on this stream. Use this option for use cases which just require moving data around from one system to another to get optimum performance." + } + }, + "fields": [] + } + ] + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": false, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 4, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "streamingStrategy", + "description": "Configure if repeatable streams should be used and their behaviour", + "modelProperties": { + "org.mule.runtime.extension.api.property.QNameModelProperty": { + "value": { + "namespaceURI": "http://www.mulesoft.org/schema/mule/core", + "localPart": "abstract-byte-streaming-strategy", + "prefix": "mule" + } + }, + "org.mule.runtime.extension.api.property.InfrastructureParameterModelProperty": { + "sequence": 2 + } + } + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + }, + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 5, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "target", + "displayModel": { + "displayName": "Target Variable" + }, + "description": "The name of a variable on which the operation\u0027s output will be placed", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "REQUIRED", + "defaultValue": "#[payload]", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 6, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "targetValue", + "displayModel": { + "displayName": "Target Value" + }, + "description": "An expression that will be evaluated against the operation\u0027s output and the outcome of that expression will be stored in the target variable", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2 + }, + "showInDsl": false, + "name": "Output", + "description": "", + "modelProperties": {} + } + ], + "name": "Get-page-meta-tags", + "description": "Fetch the meta tags from a web page.", + "modelProperties": {}, + "kind": "operation" + }, + { + "blocking": true, + "executionType": "CPU_LITE", + "output": { + "type": { + "format": "json", + "type": "String" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "outputAttributes": { + "type": { + "format": "java", + "type": "Void" + }, + "hasDynamicType": false, + "description": "", + "modelProperties": {} + }, + "transactional": false, + "requiresConnection": false, + "supportsStreaming": false, + "notifications": [], + "nestedComponents": [], + "errors": [], + "stereotype": { + "type": "PROCESSOR", + "namespace": "MULE" + }, + "parameterGroupModels": [ + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "query", + "displayModel": { + "displayName": "Search Query", + "example": "apple inc" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": true, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2, + "tabName": "General" + }, + "allowedStereotypeModels": [], + "name": "apiKey", + "displayModel": { + "displayName": "API Key", + "example": "your_api_key_here" + }, + "description": "", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 3, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "outputMimeType", + "description": "The mime type of the payload that this operation outputs.", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 1 + }, + "showInDsl": false, + "name": "General", + "description": "", + "modelProperties": {} + }, + { + "parameters": [ + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "NOT_SUPPORTED", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 4, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "target", + "displayModel": { + "displayName": "Target Variable" + }, + "description": "The name of a variable on which the operation\u0027s output will be placed", + "modelProperties": {} + }, + { + "type": { + "format": "java", + "type": "String" + }, + "hasDynamicType": false, + "required": false, + "isConfigOverride": false, + "expressionSupport": "REQUIRED", + "defaultValue": "#[payload]", + "role": "BEHAVIOUR", + "dslConfiguration": { + "allowsInlineDefinition": true, + "allowsReferences": true, + "allowTopLevelDefinition": false + }, + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 5, + "tabName": "Advanced" + }, + "allowedStereotypeModels": [], + "name": "targetValue", + "displayModel": { + "displayName": "Target Value" + }, + "description": "An expression that will be evaluated against the operation\u0027s output and the outcome of that expression will be stored in the target variable", + "modelProperties": {} + } + ], + "exclusiveParametersModels": [], + "layoutModel": { + "password": false, + "text": false, + "query": false, + "order": 2 + }, + "showInDsl": false, + "name": "Output", + "description": "", + "modelProperties": {} + } + ], + "name": "Google-search", + "description": "Perform a Google search using the SERP API.", + "modelProperties": {}, + "kind": "operation" + } + ], + "functions": [], + "constructs": [], + "connectionProviders": [], + "messageSources": [], + "notifications": [], + "errors": [ + { + "error": "MAC-WEB-CRAWLER:RETRY_EXHAUSTED", + "parent": "MULE:RETRY_EXHAUSTED", + "handleable": true + }, + { + "error": "MULE:RETRY_EXHAUSTED", + "parent": "MULE:ANY", + "handleable": true + }, + { + "error": "MULE:ANY", + "handleable": true + }, + { + "error": "MAC-WEB-CRAWLER:CONNECTIVITY", + "parent": "MULE:CONNECTIVITY", + "handleable": true + }, + { + "error": "MULE:CONNECTIVITY", + "parent": "MULE:ANY", + "handleable": true + } + ], + "modelProperties": {}, + "types": [] +}