feat(slb-495): update html parsing library

apps/converter/htmlToMarkdown.js

@@ -1,3 +1,4 @@
+import { extract } from '@extractus/article-extractor'
 import crypto from 'crypto';
 import fs from 'fs-extra';
 import imageType from 'image-type';
@@ -14,6 +15,18 @@ const __dirname = isLagoon
   ? '/app/web/sites/default/files/converted'
   : path.dirname(__filename);
 
+
+async function extractMainContentFromUrl(url) {
+      try {
+        const mainContent = await extract(url);
+        console.log("🚀 ~ extractMainContentFromUrl ~ mainContent:", mainContent)
+        return mainContent ? mainContent.content : '';
+      } catch (err) {
+        console.error(err)
+      }
+      return '';
+  }
+
 async function extractMainContent(htmlString) {
   const bodyRegex = /<body[^>]*>([\s\S]*?)<\/body>/i;
   const match = htmlString.match(bodyRegex);
@@ -69,13 +82,16 @@ export async function htmlToMarkdown(url) {
   }
 
   // Fetch HTML content
+  /*
   const response = await fetch(url);
   if (!response.ok) {
     throw new Error(`Failed to fetch page: ${response.statusText}`);
   }
   const fullHtml = await response.text();
 
   const html = await extractMainContent(fullHtml);
+  */
+  const html = await extractMainContentFromUrl(url);
   // Generate folder name based on HTML content
   const folderName = generateFolderName(html);
   const outputDir = path.join(__dirname, folderName);

apps/converter/index.js

@@ -239,13 +239,12 @@ app.get('/html-convert', async (req, res) => {
       element.htmlValue = html;
     });
 
-
     const enhanced = await enhanceMdastNodesRecursive(mdast, outputDir);
+    const flatten = await flattenMdastNodesRecursive(enhanced);
 
     // Return the processed content along with conversion info
     res.json({
-      //content: mdast.children,
-      content: enhanced.children,
+      content: flatten.children,
       outputDirectory: outputDir,
       warnings: warnings,
     });

apps/converter/package.json

@@ -12,6 +12,7 @@
     "dev": "node --watch index.js"
   },
   "dependencies": {
+    "@extractus/article-extractor": "^8.0.16",
     "@textlint/markdown-to-ast": "^14.3.0",
     "express": "^4.21.1",
     "fs-extra": "^11.2.0",

packages/drupal/silverback_ai/modules/silverback_ai_import/silverback_ai_import.module

@@ -136,13 +136,15 @@ function _silverback_ai_import_form_submit(array $form, FormStateInterface $form
     $file = \Drupal::entityTypeManager()->getStorage('file')->load($fid);
     $ast = $service->getAstFromFilePath($file);
     $content->create($ast->content, $entity);
-    $flatten = $service->flattenAst($ast->content);
-    $content->create($flatten, $entity);
+
+    // $flatten = $service->flattenAst($ast->content);
+    // $content->create($flatten, $entity);
   }
   elseif (!empty($url_value) && $type == 'url') {
     $ast = $service->getAstFromUrl($url_value);
     $content->create($ast->content, $entity);
-    $flatten = $service->flattenAst($ast->content);
-    $content->create($flatten, $entity);
+
+    // $flatten = $service->flattenAst($ast->content);
+    // $content->create($flatten, $entity);
   }
 }