Initial commit

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.vscode/settings.json

@@ -0,0 +1,20 @@
+{
+    "files.exclude": {
+        "**/.git": true,
+        "**/.svn": true,
+        "**/.hg": true,
+        "**/CVS": true,
+        "**/.DS_Store": true,
+        "**/Thumbs.db": true,
+        "**/.apivcs": true,
+        "**/.gitignore": true,
+        "**/.designer.json": true,
+        "**/.exchange.xml": true,
+        "**/.vscode": true,
+        "**/.mvn": true,
+        "**/.tmp": true,
+        "**/.settings": true,
+        "**/.exchange_modules_tmp": true,
+        ".mule": true
+    }
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Amir Khan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,21 @@
+# Mulechain-web-crawler Extension
+
+Add description ...
+
+
+...
+
+
+...
+
+
+Add this dependency to your application pom.xml
+
+```
+<dependency>
+    <groupId>com.mule.mulechain</groupId>
+    <artifactId>mulechain-web-crawler</artifactId>
+    <version>0.0.0</version>
+    <classifier>mule-plugin</classifier>
+</dependency>
+```

pom.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.mule.mulechain</groupId>
+    <artifactId>mulechain-web-crawler</artifactId>
+    <version>0.0.0</version>
+    <packaging>mule-extension</packaging>
+    <name>Mulechain-web-crawler Extension</name>
+
+    <parent>
+        <groupId>org.mule.extensions</groupId>
+        <artifactId>mule-modules-parent</artifactId>
+        <version>1.1.3</version>
+    </parent>
+    <dependencies>
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.17.2</version>
+        </dependency>    </dependencies>
+	<repositories>
+		<repository>
+			<id>anypoint-exchange-v3</id>
+			<name>Anypoint Exchange V3</name>
+			<url>https://maven.eu1.anypoint.mulesoft.com/api/v3/maven</url>
+			<layout>default</layout>
+		</repository>
+	</repositories>
+	<pluginRepositories>
+		<pluginRepository>
+			<id>mulesoft-releases</id>
+			<name>mulesoft release repository</name>
+			<layout>default</layout>
+			<url>https://repository.mulesoft.org/releases/</url>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+		</pluginRepository>
+	</pluginRepositories>
+	<distributionManagement>
+		<snapshotRepository>
+			<id>exchange-repository</id>
+			<name>Exchange Repository</name>
+			<url>https://maven.anypoint.mulesoft.com/api/v1/organizations/9c7c42b3-1b21-4ec1-9183-047ec0921663/maven</url>
+			<layout>default</layout>
+		</snapshotRepository>
+		<repository>
+			<id>exchange-repository</id>
+			<name>Exchange Repository</name>
+			<url>https://maven.anypoint.mulesoft.com/api/v1/organizations/9c7c42b3-1b21-4ec1-9183-047ec0921663/maven</url>
+			<layout>default</layout>
+		</repository>
+	</distributionManagement>
+</project>

src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerConfiguration.java

@@ -0,0 +1,19 @@
+package com.mule.mulechain.crawler.internal;
+
+import org.mule.runtime.extension.api.annotation.Operations;
+import org.mule.runtime.extension.api.annotation.param.Parameter;
+
+/**
+ * This class represents an extension configuration, values set in this class are commonly used across multiple
+ * operations since they represent something core from the extension.
+ */
+@Operations(MulechainwebcrawlerOperations.class)
+public class MulechainwebcrawlerConfiguration {
+
+  @Parameter
+  private String configId;
+
+  public String getConfigId(){
+    return configId;
+  }
+}

src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerExtension.java

@@ -0,0 +1,17 @@
+package com.mule.mulechain.crawler.internal;
+
+import org.mule.runtime.extension.api.annotation.Extension;
+import org.mule.runtime.extension.api.annotation.Configurations;
+import org.mule.runtime.extension.api.annotation.dsl.xml.Xml;
+
+
+/**
+ * This is the main class of an extension, is the entry point from which configurations, connection providers, operations
+ * and sources are going to be declared.
+ */
+@Xml(prefix = "mulechain-web-crawler")
+@Extension(name = "Mulechain Crawler")
+@Configurations(MulechainwebcrawlerConfiguration.class)
+public class MulechainwebcrawlerExtension {
+
+}

src/main/java/com/mule/mulechain/crawler/internal/MulechainwebcrawlerOperations.java

@@ -0,0 +1,29 @@
+package com.mule.mulechain.crawler.internal;
+
+import static org.mule.runtime.extension.api.annotation.param.MediaType.ANY;
+
+import org.mule.runtime.extension.api.annotation.Alias;
+import org.mule.runtime.extension.api.annotation.param.MediaType;
+import org.mule.runtime.extension.api.annotation.param.Config;
+import java.io.IOException;  
+import org.jsoup.Jsoup;  
+import org.jsoup.nodes.Document;  
+
+/**
+ * This class is a container for operations, every public method in this class will be taken as an extension operation.
+ */
+public class MulechainwebcrawlerOperations {
+
+  /**
+   * Example of an operation that uses the configuration and a connection instance to perform some action.
+   * @throws IOException 
+   */
+
+  @MediaType(value = ANY, strict = false)
+  @Alias("Crawl-website")  
+  public String crawlWebsite(String url, @Config MulechainwebcrawlerConfiguration configuration) throws IOException{
+
+        return "";
+  }
+
+}

src/main/java/com/mule/mulechain/crawler/internal/helpers/crawlingHelper.java

@@ -0,0 +1,19 @@
+package com.mule.mulechain.crawler.internal.helpers;
+import java.io.IOException;
+
+import org.jsoup.Jsoup;  
+import org.jsoup.nodes.Document;  
+
+public class crawlingHelper {
+
+    private static String getTitle(String url, String outputFolder) throws IOException{
+        Document doc = connectUrlGetDocument(url);  
+        String title = doc.title();  
+        //System.out.println("title is: " + title);     
+        return title;
+    }
+
+    private static Document connectUrlGetDocument(String url) throws IOException {
+        return Jsoup.connect(url).get();
+    }
+}

src/test/java/com/mule/mulechain/crawler/MulechainwebcrawlerOperationsTestCase.java

@@ -0,0 +1,36 @@
+package com.mule.mulechain.crawler;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import org.mule.functional.junit4.MuleArtifactFunctionalTestCase;
+import org.junit.Test;
+
+public class MulechainwebcrawlerOperationsTestCase extends MuleArtifactFunctionalTestCase {
+
+  /**
+   * Specifies the mule config xml with the flows that are going to be executed in the tests, this file lives in the test resources.
+   */
+  @Override
+  protected String getConfigFile() {
+    return "test-mule-config.xml";
+  }
+
+  @Test
+  public void executeSayHiOperation() throws Exception {
+    String payloadValue = ((String) flowRunner("sayHiFlow").run()
+                                      .getMessage()
+                                      .getPayload()
+                                      .getValue());
+    assertThat(payloadValue, is("Hello Mariano Gonzalez!!!"));
+  }
+
+  @Test
+  public void executeRetrieveInfoOperation() throws Exception {
+    String payloadValue = ((String) flowRunner("retrieveInfoFlow")
+                                      .run()
+                                      .getMessage()
+                                      .getPayload()
+                                      .getValue());
+    assertThat(payloadValue, is("Using Configuration [configId] with Connection id [aValue:100]"));
+  }
+}

src/test/resources/test-mule-config.xml

@@ -0,0 +1,19 @@
+<mule xmlns="http://www.mulesoft.org/schema/mule/core"
+      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+      xmlns:mulechain-web-crawler="http://www.mulesoft.org/schema/mule/mulechain-web-crawler"
+      xsi:schemaLocation="http://www.mulesoft.org/schema/mule/core http://www.mulesoft.org/schema/mule/core/current/mule.xsd
+          http://www.mulesoft.org/schema/mule/mulechain-web-crawler http://www.mulesoft.org/schema/mule/mulechain-web-crawler/current/mule-mulechain-web-crawler.xsd">
+
+    <mulechain-web-crawler:config name="config" configId="configId">
+        <mulechain-web-crawler:connection requiredParameter="aValue" />
+    </mulechain-web-crawler:config>
+
+    <flow name="sayHiFlow">
+        <mulechain-web-crawler:say-hi person="Mariano Gonzalez"/>
+    </flow>
+
+    <flow name="retrieveInfoFlow">
+        <mulechain-web-crawler:retrieve-info config-ref="config"/>
+    </flow>
+
+</mule>

target/docs/mulechain-crawler-documentation.adoc

@@ -0,0 +1,93 @@
+:toc:               left
+:toc-title:         Mulechain Crawler Module
+:toclevels:         2
+:last-update-label!:
+:docinfo:
+:source-highlighter: coderay
+:icons: font
+
+
+= Mulechain Crawler Module Documentation Reference
+
++++
+This is the main class of an extension, is the entry point from which configurations, connection providers, operations and sources are going to be declared.
++++
+
+
+== Configurations
+---
+[[config]]
+=== Config
+
++++
+Default configuration
++++
+
+==== Parameters
+[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"]
+|======================
+| Name | Type | Description | Default Value | Required
+|Name | String | The name for this configuration. Connectors reference the configuration with this name. | | *x*{nbsp}
+| Config Id a| String |  |  | *x*{nbsp}
+| Expiration Policy a| <<ExpirationPolicy>> |  +++Configures the minimum amount of time that a dynamic configuration instance can remain idle before the runtime considers it eligible for expiration. This does not mean that the platform will expire the instance at the exact moment that it becomes eligible. The runtime will actually purge the instances when it sees it fit.+++ |  | {nbsp}
+|======================
+
+
+==== Associated Operations
+* <<Crawl-website>> {nbsp}
+
+
+
+== Operations
+
+[[Crawl-website]]
+=== Crawl Website
+`<mulechain-web-crawler:crawl-website>`
+
++++
+Example of an operation that uses the configuration and a connection instance to perform some action.
++++
+
+==== Parameters
+[cols=".^20%,.^20%,.^35%,.^20%,^.^5%", options="header"]
+|======================
+| Name | Type | Description | Default Value | Required
+| Configuration | String | The name of the configuration to use. | | *x*{nbsp}
+| Url a| String |  |  | *x*{nbsp}
+| Output Folder a| String |  |  | *x*{nbsp}
+| Output Mime Type a| String |  +++The mime type of the payload that this operation outputs.+++ |  | {nbsp}
+| Target Variable a| String |  +++The name of a variable on which the operation's output will be placed+++ |  | {nbsp}
+| Target Value a| String |  +++An expression that will be evaluated against the operation's output and the outcome of that expression will be stored in the target variable+++ |  +++#[payload]+++ | {nbsp}
+|======================
+
+==== Output
+[cols=".^50%,.^50%"]
+|======================
+| *Type* a| String
+|======================
+
+==== For Configurations.
+* <<config>> {nbsp}
+
+
+
+
+== Types
+[[ExpirationPolicy]]
+=== Expiration Policy
+
+[cols=".^20%,.^25%,.^30%,.^15%,.^10%", options="header"]
+|======================
+| Field | Type | Description | Default Value | Required
+| Max Idle Time a| Number | A scalar time value for the maximum amount of time a dynamic configuration instance should be allowed to be idle before it's considered eligible for expiration |  | 
+| Time Unit a| Enumeration, one of:
+
+** NANOSECONDS
+** MICROSECONDS
+** MILLISECONDS
+** SECONDS
+** MINUTES
+** HOURS
+** DAYS | A time unit that qualifies the maxIdleTime attribute |  | 
+|======================
+