Merge pull request #33 from James-LG/james/doctype

fix(html): Allow verbose doctype declaration
James-LG · Jun 15, 2024 · 1c9bbc8 · 1c9bbc8
2 parents 3a5a3be + bd6a116
commit 1c9bbc8
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 9 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "skyscraper"
-version = "0.6.3"
+version = "0.6.4"
 authors = ["James La Novara-Gsell <[email protected]>"]
 edition = "2021"
 description = "XPath for HTML web scraping"

diff --git a/src/html/parse/mod.rs b/src/html/parse/mod.rs
@@ -380,16 +380,16 @@ fn is_doctype(
             if iden != "html" {
                 return Err(ParseError::MissingHtmlAfterDoctype);
             }
-            let token = tokens.next().ok_or(ParseError::UnexpectedEndOfTokens)?;
 
-            if !matches!(token, Token::TagClose) {
-                return Err(ParseError::MissingTagCloseAfterDoctype);
+            for token in tokens {
+                if let Token::TagClose = token {
+                    return Ok(true);
+                }
             }
+            return Err(ParseError::MissingTagCloseAfterDoctype);
         } else {
             return Err(ParseError::MissingHtmlAfterDoctype);
         }
-
-        return Ok(true);
     }
 
     Ok(false)

diff --git a/src/xpath/grammar/expressions/comparison_expressions.rs b/src/xpath/grammar/expressions/comparison_expressions.rs
@@ -10,14 +10,13 @@ use nom::{
 use crate::{
     xpath::{
         grammar::{
-            data_model::{AnyAtomicType, Node, XpathItem},
+            data_model::{AnyAtomicType, XpathItem},
             expressions::string_concat_expressions::string_concat_expr,
             recipes::Res,
             terminal_symbols::symbol_separator,
-            NonTreeXpathNode, XpathItemTreeNodeData,
         },
         xpath_item_set::XpathItemSet,
-        ExpressionApplyError, XpathExpressionContext, XpathItemTree,
+        ExpressionApplyError, XpathExpressionContext,
     },
     xpath_item_set,
 };

diff --git a/tests/html_tests.rs b/tests/html_tests.rs
@@ -49,3 +49,35 @@ fn text_should_unescape_characters() {
     let html_text = document.get_html_node(&child).unwrap().extract_as_text();
     assert_eq!(html_text.value, r##"&"'<>"##);
 }
+
+#[test]
+fn doctype_should_skip_regular_doctype() {
+    // arrange
+    let text = r##"
+        <!DOCTYPE html>
+        <div>hi</div>"##;
+
+    // act
+    let document = html::parse(text).unwrap();
+
+    // assert
+    let root_node = document.root_node;
+    let html_tag = document.get_html_node(&root_node).unwrap().extract_as_tag();
+    assert_eq!(html_tag.name, "div");
+}
+
+#[test]
+fn doctype_should_skip_verbose_doctype() {
+    // arrange
+    let text = r##"
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <div>hi</div>"##;
+
+    // act
+    let document = html::parse(text).unwrap();
+
+    // assert
+    let root_node = document.root_node;
+    let html_tag = document.get_html_node(&root_node).unwrap().extract_as_tag();
+    assert_eq!(html_tag.name, "div");
+}