feat(shirelang): add crawl functionality and processor support #59

core/src/main/kotlin/com/phodal/shirecore/config/interaction/PostFunction.kt

@@ -5,4 +5,4 @@ import com.intellij.openapi.util.TextRange
 /**
  * Don't remove public modifier, it's required Kotlin compile, in IDEA will failed.
  */
-public typealias PostFunction = (response: String?, textRange: TextRange?) -> Unit
+public typealias PostFunction = (response: String?, textRange: TextRange?) -> Unit

shirelang/src/main/kotlin/com/phodal/shirelang/compiler/hobbit/execute/CrawlProcessor.kt

@@ -0,0 +1,28 @@
+package com.phodal.shirelang.compiler.hobbit.execute
+
+import com.intellij.openapi.application.runInEdt
+import com.phodal.shirecore.agent.agenttool.browse.BrowseTool
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.runBlocking
+
+object CrawlProcessor {
+    suspend fun doExecute(url: String): String? {
+        var body: String? = null
+        runInEdt {
+            val parse = BrowseTool.parse(url)
+            body = parse.body
+        }
+
+        return body
+    }
+
+    fun crawl(urls: Array<out String>): List<String> {
+        return runBlocking {
+            coroutineScope {
+                urls.mapNotNull {
+                    doExecute(it)
+                }
+            }
+        }
+    }
+}

shirelang/src/main/kotlin/com/phodal/shirelang/compiler/hobbit/execute/PatternFuncProcessor.kt

@@ -214,6 +214,9 @@ open class PatternFuncProcessor(open val myProject: Project, open val hole: Hobb
             is PatternActionFunc.Redact -> {
                 RedactProcessor.redact(myProject, lastResult)
             }
+            is PatternActionFunc.Crawl -> {
+                CrawlProcessor.crawl(action.urls)
+            }
         }
     }
 

shirelang/src/main/kotlin/com/phodal/shirelang/compiler/parser/HobbitHoleParser.kt

@@ -607,6 +607,10 @@ object HobbitHoleParser {
                 PatternActionFunc.Redact(first)
             }
 
+            "crawl" -> {
+                PatternActionFunc.Crawl(*args.toTypedArray())
+            }
+
             null -> {
                 logger.warn("parsePatternAction, Unknown pattern action: ${expr.funcCall}")
                 return null

shirelang/src/main/kotlin/com/phodal/shirelang/compiler/patternaction/PatternActionFunc.kt

@@ -145,6 +145,13 @@ sealed class PatternActionFunc(open val funcName: String) {
      */
     class Redact(val strategy: String) : PatternActionFunc("redact")
 
+    /**
+     * The Crawl function is used to crawl a list of urls, get markdown from html and save it to a file.
+     *
+     * @param urls The urls to crawl.
+     */
+    class Crawl(vararg val urls: String) : PatternActionFunc("crawl")
+
     /**
      * User Custom Functions
      */