Started on new crawler

polterguy · Jan 30, 2024 · 0c24953 · 0c24953
1 parent 27a93e6
commit 0c24953
Show file tree

Hide file tree

Showing 6 changed files with 198 additions and 1,343 deletions.
diff --git a/backend/files/system/openai/magic.startup/crawling/magic.ai.crawl-site.hl b/backend/files/system/openai/magic.startup/crawling/magic.ai.crawl-site.hl
@@ -224,11 +224,6 @@ slots.create:magic.ai.crawl-site
                      signal:magic.ai.url.scrape
                         url:x:@.dp/#
                         type:x:@.arguments/*/type
-                        images:bool:true
-                        code:bool:true
-                        lists:bool:true
-                        main:bool:true
-                        empty-completion:bool:false
                         threshold:x:@.arguments/*/threshold
                         feedback-channel:x:@.arguments/*/feedback-channel
 

diff --git a/backend/files/system/openai/magic.startup/crawling/magic.ai.html.extract-code.hl b/backend/files/system/openai/magic.startup/crawling/magic.ai.html.extract-code.hl
diff --git a/backend/files/system/openai/magic.startup/crawling/magic.ai.html.extract-images.hl b/backend/files/system/openai/magic.startup/crawling/magic.ai.html.extract-images.hl