= (3, 7):
+if cython.compiled:
+ @cython.cfunc
+ @cython.inline
+ def attributeMap():
+ if version.PY_VERSION_HEX >= 0x03070000:
+ return {}
+ else:
+ return OrderedDict()
+elif version_info >= (3, 7):
attributeMap = dict
else:
attributeMap = OrderedDict
@@ -30,7 +43,7 @@ class HTMLTokenizer(object):
* self.currentToken
Holds the token that is currently being processed.
- * self.state
+ * self._state
Holds a reference to the method to be invoked... XXX
* self.stream
@@ -43,13 +56,11 @@ def __init__(self, stream, parser=None, **kwargs):
self.parser = parser
# Setup the initial tokenizer state
- self.escapeFlag = False
- self.lastFourChars = []
- self.state = self.dataState
- self.escape = False
+ self._state = self.dataState
# The current token being created
self.currentToken = None
+ self.currentAttribute = None
super(HTMLTokenizer, self).__init__()
def __iter__(self):
@@ -60,13 +71,180 @@ def __iter__(self):
is requested.
"""
self.tokenQueue = deque([])
- # Start processing. When EOF is reached self.state will return False
+ # Start processing. When EOF is reached self._state will return False
# instead of True and the loop will terminate.
- while self.state():
- while self.stream.errors:
- yield {"type": tokenTypes["ParseError"], "data": self.stream.errors.pop(0)}
- while self.tokenQueue:
- yield self.tokenQueue.popleft()
+ if cython.compiled:
+ with cython.binding(False):
+ while self._state(self):
+ while self.stream.errors:
+ yield {"type": tokenTypes["ParseError"], "data": self.stream.errors.pop(0)}
+ while self.tokenQueue:
+ yield self.tokenQueue.popleft()
+ else:
+ while self._state():
+ while self.stream.errors:
+ yield {"type": tokenTypes["ParseError"], "data": self.stream.errors.pop(0)}
+ while self.tokenQueue:
+ yield self.tokenQueue.popleft()
+
+ @property
+ def state(self):
+ if cython.compiled:
+ if self._state == self.dataState:
+ return "dataState"
+ elif self._state == self.entityDataState:
+ return "entityDataState"
+ elif self._state == self.rcdataState:
+ return "rcdataState"
+ elif self._state == self.characterReferenceInRcdata:
+ return "characterReferenceInRcdata"
+ elif self._state == self.rawtextState:
+ return "rawtextState"
+ elif self._state == self.scriptDataState:
+ return "scriptDataState"
+ elif self._state == self.plaintextState:
+ return "plaintextState"
+ elif self._state == self.tagOpenState:
+ return "tagOpenState"
+ elif self._state == self.closeTagOpenState:
+ return "closeTagOpenState"
+ elif self._state == self.tagNameState:
+ return "tagNameState"
+ elif self._state == self.rcdataLessThanSignState:
+ return "rcdataLessThanSignState"
+ elif self._state == self.rcdataEndTagOpenState:
+ return "rcdataEndTagOpenState"
+ elif self._state == self.rcdataEndTagNameState:
+ return "rcdataEndTagNameState"
+ elif self._state == self.rawtextLessThanSignState:
+ return "rawtextLessThanSignState"
+ elif self._state == self.rawtextEndTagOpenState:
+ return "rawtextEndTagOpenState"
+ elif self._state == self.rawtextEndTagNameState:
+ return "rawtextEndTagNameState"
+ elif self._state == self.scriptDataLessThanSignState:
+ return "scriptDataLessThanSignState"
+ elif self._state == self.scriptDataEndTagOpenState:
+ return "scriptDataEndTagOpenState"
+ elif self._state == self.scriptDataEndTagNameState:
+ return "scriptDataEndTagNameState"
+ elif self._state == self.scriptDataEscapeStartState:
+ return "scriptDataEscapeStartState"
+ elif self._state == self.scriptDataEscapeStartDashState:
+ return "scriptDataEscapeStartDashState"
+ elif self._state == self.scriptDataEscapedState:
+ return "scriptDataEscapedState"
+ elif self._state == self.scriptDataEscapedDashState:
+ return "scriptDataEscapedDashState"
+ elif self._state == self.scriptDataEscapedDashDashState:
+ return "scriptDataEscapedDashDashState"
+ elif self._state == self.scriptDataEscapedLessThanSignState:
+ return "scriptDataEscapedLessThanSignState"
+ elif self._state == self.scriptDataEscapedEndTagOpenState:
+ return "scriptDataEscapedEndTagOpenState"
+ elif self._state == self.scriptDataEscapedEndTagNameState:
+ return "scriptDataEscapedEndTagNameState"
+ elif self._state == self.scriptDataDoubleEscapeStartState:
+ return "scriptDataDoubleEscapeStartState"
+ elif self._state == self.scriptDataDoubleEscapedState:
+ return "scriptDataDoubleEscapedState"
+ elif self._state == self.scriptDataDoubleEscapedDashState:
+ return "scriptDataDoubleEscapedDashState"
+ elif self._state == self.scriptDataDoubleEscapedDashDashState:
+ return "scriptDataDoubleEscapedDashDashState"
+ elif self._state == self.scriptDataDoubleEscapedLessThanSignState:
+ return "scriptDataDoubleEscapedLessThanSignState"
+ elif self._state == self.scriptDataDoubleEscapeEndState:
+ return "scriptDataDoubleEscapeEndState"
+ elif self._state == self.beforeAttributeNameState:
+ return "beforeAttributeNameState"
+ elif self._state == self.attributeNameState:
+ return "attributeNameState"
+ elif self._state == self.afterAttributeNameState:
+ return "afterAttributeNameState"
+ elif self._state == self.beforeAttributeValueState:
+ return "beforeAttributeValueState"
+ elif self._state == self.attributeValueDoubleQuotedState:
+ return "attributeValueDoubleQuotedState"
+ elif self._state == self.attributeValueSingleQuotedState:
+ return "attributeValueSingleQuotedState"
+ elif self._state == self.attributeValueUnQuotedState:
+ return "attributeValueUnQuotedState"
+ elif self._state == self.afterAttributeValueState:
+ return "afterAttributeValueState"
+ elif self._state == self.selfClosingStartTagState:
+ return "selfClosingStartTagState"
+ elif self._state == self.bogusCommentState:
+ return "bogusCommentState"
+ elif self._state == self.markupDeclarationOpenState:
+ return "markupDeclarationOpenState"
+ elif self._state == self.commentStartState:
+ return "commentStartState"
+ elif self._state == self.commentStartDashState:
+ return "commentStartDashState"
+ elif self._state == self.commentState:
+ return "commentState"
+ elif self._state == self.commentEndDashState:
+ return "commentEndDashState"
+ elif self._state == self.commentEndState:
+ return "commentEndState"
+ elif self._state == self.commentEndBangState:
+ return "commentEndBangState"
+ elif self._state == self.doctypeState:
+ return "doctypeState"
+ elif self._state == self.beforeDoctypeNameState:
+ return "beforeDoctypeNameState"
+ elif self._state == self.doctypeNameState:
+ return "doctypeNameState"
+ elif self._state == self.afterDoctypeNameState:
+ return "afterDoctypeNameState"
+ elif self._state == self.afterDoctypePublicKeywordState:
+ return "afterDoctypePublicKeywordState"
+ elif self._state == self.beforeDoctypePublicIdentifierState:
+ return "beforeDoctypePublicIdentifierState"
+ elif self._state == self.doctypePublicIdentifierDoubleQuotedState:
+ return "doctypePublicIdentifierDoubleQuotedState"
+ elif self._state == self.doctypePublicIdentifierSingleQuotedState:
+ return "doctypePublicIdentifierSingleQuotedState"
+ elif self._state == self.afterDoctypePublicIdentifierState:
+ return "afterDoctypePublicIdentifierState"
+ elif self._state == self.betweenDoctypePublicAndSystemIdentifiersState:
+ return "betweenDoctypePublicAndSystemIdentifiersState"
+ elif self._state == self.afterDoctypeSystemKeywordState:
+ return "afterDoctypeSystemKeywordState"
+ elif self._state == self.beforeDoctypeSystemIdentifierState:
+ return "beforeDoctypeSystemIdentifierState"
+ elif self._state == self.doctypeSystemIdentifierDoubleQuotedState:
+ return "doctypeSystemIdentifierDoubleQuotedState"
+ elif self._state == self.doctypeSystemIdentifierSingleQuotedState:
+ return "doctypeSystemIdentifierSingleQuotedState"
+ elif self._state == self.afterDoctypeSystemIdentifierState:
+ return "afterDoctypeSystemIdentifierState"
+ elif self._state == self.bogusDoctypeState:
+ return "bogusDoctypeState"
+ elif self._state == self.cdataSectionState:
+ return "cdataSectionState"
+ else:
+ raise ValueError("unreachable")
+ else:
+ return self._state.__name__
+
+ @state.setter
+ def state(self, newState):
+ if newState == "dataState":
+ self._state = self.dataState
+ elif newState == "cdataSectionState":
+ self._state = self.cdataSectionState
+ elif newState == "rawtextState":
+ self._state = self.rawtextState
+ elif newState == "rcdataState":
+ self._state = self.rcdataState
+ elif newState == "plaintextState":
+ self._state = self.plaintextState
+ elif newState == "scriptDataState":
+ self._state = self.scriptDataState
+ else:
+ raise ValueError(newState)
def consumeNumberEntity(self, isHex):
"""This function returns either U+FFFD or the character based on the
@@ -90,7 +268,14 @@ def consumeNumberEntity(self, isHex):
c = self.stream.char()
# Convert the set of characters consumed to an int.
- charAsInt = int("".join(charStack), radix)
+ if cython.compiled:
+ with cython.overflowcheck(True):
+ try:
+ charAsInt = int("".join(charStack), radix)
+ except OverflowError:
+ charAsInt = 0x7FFFFFFF
+ else:
+ charAsInt = int("".join(charStack), radix)
# Certain characters get replaced with others
if charAsInt in replacementCharacters:
@@ -105,20 +290,14 @@ def consumeNumberEntity(self, isHex):
"illegal-codepoint-for-numeric-entity",
"datavars": {"charAsInt": charAsInt}})
else:
- # Should speed up this check somehow (e.g. move the set to a constant)
- if ((0x0001 <= charAsInt <= 0x0008) or
+ if (
+ (0x0001 <= charAsInt <= 0x0008) or
(0x000E <= charAsInt <= 0x001F) or
(0x007F <= charAsInt <= 0x009F) or
(0xFDD0 <= charAsInt <= 0xFDEF) or
- charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE,
- 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
- 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE,
- 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE,
- 0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE,
- 0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE,
- 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
- 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE,
- 0xFFFFF, 0x10FFFE, 0x10FFFF])):
+ charAsInt == 0x000B or
+ (charAsInt & 0xFFFE) == 0xFFFE
+ ):
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data":
"illegal-codepoint-for-numeric-entity",
@@ -212,7 +391,7 @@ def consumeEntity(self, allowedChar=None, fromAttribute=False):
output = "&" + "".join(charStack)
if fromAttribute:
- self.currentToken["data"][-1][1] += output
+ self.currentToken["data"][self.currentAttribute][-1] += output
else:
if output in spaceCharacters:
tokenType = "SpaceCharacters"
@@ -233,14 +412,11 @@ def emitCurrentToken(self):
token = self.currentToken
# Add token to the queue to be yielded
if (token["type"] in tagTokenTypes):
- token["name"] = token["name"].translate(asciiUpper2Lower)
+ token["name"] = ascii_lower(token["name"])
if token["type"] == tokenTypes["StartTag"]:
- raw = token["data"]
- data = attributeMap(raw)
- if len(raw) > len(data):
- # we had some duplicated attribute, fix so first wins
- data.update(raw[::-1])
- token["data"] = data
+ data = token["data"]
+ for k, v in data.items():
+ data[k] = v[0]
if token["type"] == tokenTypes["EndTag"]:
if token["data"]:
@@ -250,15 +426,15 @@ def emitCurrentToken(self):
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "self-closing-flag-on-end-tag"})
self.tokenQueue.append(token)
- self.state = self.dataState
+ self._state = self.dataState
# Below are the various tokenizer states worked out.
def dataState(self):
data = self.stream.char()
if data == "&":
- self.state = self.entityDataState
+ self._state = self.entityDataState
elif data == "<":
- self.state = self.tagOpenState
+ self._state = self.tagOpenState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -284,15 +460,15 @@ def dataState(self):
def entityDataState(self):
self.consumeEntity()
- self.state = self.dataState
+ self._state = self.dataState
return True
def rcdataState(self):
data = self.stream.char()
if data == "&":
- self.state = self.characterReferenceInRcdata
+ self._state = self.characterReferenceInRcdata
elif data == "<":
- self.state = self.rcdataLessThanSignState
+ self._state = self.rcdataLessThanSignState
elif data == EOF:
# Tokenization ends.
return False
@@ -318,13 +494,13 @@ def rcdataState(self):
def characterReferenceInRcdata(self):
self.consumeEntity()
- self.state = self.rcdataState
+ self._state = self.rcdataState
return True
def rawtextState(self):
data = self.stream.char()
if data == "<":
- self.state = self.rawtextLessThanSignState
+ self._state = self.rawtextLessThanSignState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -342,7 +518,7 @@ def rawtextState(self):
def scriptDataState(self):
data = self.stream.char()
if data == "<":
- self.state = self.scriptDataLessThanSignState
+ self._state = self.scriptDataLessThanSignState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -375,74 +551,74 @@ def plaintextState(self):
def tagOpenState(self):
data = self.stream.char()
if data == "!":
- self.state = self.markupDeclarationOpenState
+ self._state = self.markupDeclarationOpenState
elif data == "/":
- self.state = self.closeTagOpenState
+ self._state = self.closeTagOpenState
elif data in asciiLetters:
self.currentToken = {"type": tokenTypes["StartTag"],
- "name": data, "data": [],
+ "name": data, "data": attributeMap(),
"selfClosing": False,
"selfClosingAcknowledged": False}
- self.state = self.tagNameState
+ self._state = self.tagNameState
elif data == ">":
# XXX In theory it could be something besides a tag name. But
# do we really care?
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-tag-name-but-got-right-bracket"})
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<>"})
- self.state = self.dataState
+ self._state = self.dataState
elif data == "?":
# XXX In theory it could be something besides a tag name. But
# do we really care?
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-tag-name-but-got-question-mark"})
self.stream.unget(data)
- self.state = self.bogusCommentState
+ self._state = self.bogusCommentState
else:
# XXX
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-tag-name"})
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"})
self.stream.unget(data)
- self.state = self.dataState
+ self._state = self.dataState
return True
def closeTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.currentToken = {"type": tokenTypes["EndTag"], "name": data,
- "data": [], "selfClosing": False}
- self.state = self.tagNameState
+ "data": attributeMap(), "selfClosing": False}
+ self._state = self.tagNameState
elif data == ">":
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-closing-tag-but-got-right-bracket"})
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-closing-tag-but-got-eof"})
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "":
self.emitCurrentToken()
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-tag-name"})
- self.state = self.dataState
+ self._state = self.dataState
elif data == "/":
- self.state = self.selfClosingStartTagState
+ self._state = self.selfClosingStartTagState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -457,22 +633,22 @@ def rcdataLessThanSignState(self):
data = self.stream.char()
if data == "/":
self.temporaryBuffer = ""
- self.state = self.rcdataEndTagOpenState
+ self._state = self.rcdataEndTagOpenState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"})
self.stream.unget(data)
- self.state = self.rcdataState
+ self._state = self.rcdataState
return True
def rcdataEndTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.temporaryBuffer += data
- self.state = self.rcdataEndTagNameState
+ self._state = self.rcdataEndTagNameState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "" and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
"name": self.temporaryBuffer,
- "data": [], "selfClosing": False}
+ "data": attributeMap(), "selfClosing": False}
self.emitCurrentToken()
- self.state = self.dataState
+ self._state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "" and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
"name": self.temporaryBuffer,
- "data": [], "selfClosing": False}
+ "data": attributeMap(), "selfClosing": False}
self.emitCurrentToken()
- self.state = self.dataState
+ self._state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "" and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
"name": self.temporaryBuffer,
- "data": [], "selfClosing": False}
+ "data": attributeMap(), "selfClosing": False}
self.emitCurrentToken()
- self.state = self.dataState
+ self._state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"})
- self.state = self.scriptDataState
+ self._state = self.scriptDataState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "\uFFFD"})
- self.state = self.scriptDataEscapedState
+ self._state = self.scriptDataEscapedState
elif data == EOF:
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
- self.state = self.scriptDataEscapedState
+ self._state = self.scriptDataEscapedState
return True
def scriptDataEscapedLessThanSignState(self):
data = self.stream.char()
if data == "/":
self.temporaryBuffer = ""
- self.state = self.scriptDataEscapedEndTagOpenState
+ self._state = self.scriptDataEscapedEndTagOpenState
elif data in asciiLetters:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<" + data})
self.temporaryBuffer = data
- self.state = self.scriptDataDoubleEscapeStartState
+ self._state = self.scriptDataDoubleEscapeStartState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"})
self.stream.unget(data)
- self.state = self.scriptDataEscapedState
+ self._state = self.scriptDataEscapedState
return True
def scriptDataEscapedEndTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.temporaryBuffer = data
- self.state = self.scriptDataEscapedEndTagNameState
+ self._state = self.scriptDataEscapedEndTagNameState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "" and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
"name": self.temporaryBuffer,
- "data": [], "selfClosing": False}
+ "data": attributeMap(), "selfClosing": False}
self.emitCurrentToken()
- self.state = self.dataState
+ self._state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": ""))):
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
if self.temporaryBuffer.lower() == "script":
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
else:
- self.state = self.scriptDataEscapedState
+ self._state = self.scriptDataEscapedState
elif data in asciiLetters:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
self.temporaryBuffer += data
else:
self.stream.unget(data)
- self.state = self.scriptDataEscapedState
+ self._state = self.scriptDataEscapedState
return True
def scriptDataDoubleEscapedState(self):
data = self.stream.char()
if data == "-":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"})
- self.state = self.scriptDataDoubleEscapedDashState
+ self._state = self.scriptDataDoubleEscapedDashState
elif data == "<":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"})
- self.state = self.scriptDataDoubleEscapedLessThanSignState
+ self._state = self.scriptDataDoubleEscapedLessThanSignState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -774,7 +950,7 @@ def scriptDataDoubleEscapedState(self):
elif data == EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-script-in-script"})
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
return True
@@ -783,23 +959,23 @@ def scriptDataDoubleEscapedDashState(self):
data = self.stream.char()
if data == "-":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"})
- self.state = self.scriptDataDoubleEscapedDashDashState
+ self._state = self.scriptDataDoubleEscapedDashDashState
elif data == "<":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"})
- self.state = self.scriptDataDoubleEscapedLessThanSignState
+ self._state = self.scriptDataDoubleEscapedLessThanSignState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "\uFFFD"})
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
elif data == EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-script-in-script"})
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
return True
def scriptDataDoubleEscapedDashDashState(self):
@@ -808,23 +984,23 @@ def scriptDataDoubleEscapedDashDashState(self):
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"})
elif data == "<":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"})
- self.state = self.scriptDataDoubleEscapedLessThanSignState
+ self._state = self.scriptDataDoubleEscapedLessThanSignState
elif data == ">":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"})
- self.state = self.scriptDataState
+ self._state = self.scriptDataState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "\uFFFD"})
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
elif data == EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-script-in-script"})
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
return True
def scriptDataDoubleEscapedLessThanSignState(self):
@@ -832,10 +1008,10 @@ def scriptDataDoubleEscapedLessThanSignState(self):
if data == "/":
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "/"})
self.temporaryBuffer = ""
- self.state = self.scriptDataDoubleEscapeEndState
+ self._state = self.scriptDataDoubleEscapeEndState
else:
self.stream.unget(data)
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
return True
def scriptDataDoubleEscapeEndState(self):
@@ -843,15 +1019,15 @@ def scriptDataDoubleEscapeEndState(self):
if data in (spaceCharacters | frozenset(("/", ">"))):
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
if self.temporaryBuffer.lower() == "script":
- self.state = self.scriptDataEscapedState
+ self._state = self.scriptDataEscapedState
else:
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
elif data in asciiLetters:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
self.temporaryBuffer += data
else:
self.stream.unget(data)
- self.state = self.scriptDataDoubleEscapedState
+ self._state = self.scriptDataDoubleEscapedState
return True
def beforeAttributeNameState(self):
@@ -859,29 +1035,29 @@ def beforeAttributeNameState(self):
if data in spaceCharacters:
self.stream.charsUntil(spaceCharacters, True)
elif data in asciiLetters:
- self.currentToken["data"].append([data, ""])
- self.state = self.attributeNameState
+ self.currentAttribute = data
+ self._state = self.attributeNameState
elif data == ">":
self.emitCurrentToken()
elif data == "/":
- self.state = self.selfClosingStartTagState
+ self._state = self.selfClosingStartTagState
elif data in ("'", '"', "=", "<"):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"invalid-character-in-attribute-name"})
- self.currentToken["data"].append([data, ""])
- self.state = self.attributeNameState
+ self.currentAttribute = data
+ self._state = self.attributeNameState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
- self.currentToken["data"].append(["\uFFFD", ""])
- self.state = self.attributeNameState
+ self.currentAttribute = "\uFFFD"
+ self._state = self.attributeNameState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-attribute-name-but-got-eof"})
- self.state = self.dataState
+ self._state = self.dataState
else:
- self.currentToken["data"].append([data, ""])
- self.state = self.attributeNameState
+ self.currentAttribute = data
+ self._state = self.attributeNameState
return True
def attributeNameState(self):
@@ -889,9 +1065,9 @@ def attributeNameState(self):
leavingThisState = True
emitToken = False
if data == "=":
- self.state = self.beforeAttributeValueState
+ self._state = self.beforeAttributeValueState
elif data in asciiLetters:
- self.currentToken["data"][-1][0] += data +\
+ self.currentAttribute += data +\
self.stream.charsUntil(asciiLetters, True)
leavingThisState = False
elif data == ">":
@@ -900,39 +1076,40 @@ def attributeNameState(self):
# because data is a dict not a list
emitToken = True
elif data in spaceCharacters:
- self.state = self.afterAttributeNameState
+ self._state = self.afterAttributeNameState
elif data == "/":
- self.state = self.selfClosingStartTagState
+ self._state = self.selfClosingStartTagState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
- self.currentToken["data"][-1][0] += "\uFFFD"
+ self.currentAttribute += "\uFFFD"
leavingThisState = False
elif data in ("'", '"', "<"):
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data":
"invalid-character-in-attribute-name"})
- self.currentToken["data"][-1][0] += data
+ self.currentAttribute += data
leavingThisState = False
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "eof-in-attribute-name"})
- self.state = self.dataState
+ self._state = self.dataState
else:
- self.currentToken["data"][-1][0] += data
+ self.currentAttribute += data
leavingThisState = False
+ assert leavingThisState == ((self._state != self.attributeNameState) or emitToken)
if leavingThisState:
# Attributes are not dropped at this stage. That happens when the
# start tag token is emitted so values can still be safely appended
# to attributes, but we do want to report the parse error in time.
- self.currentToken["data"][-1][0] = (
- self.currentToken["data"][-1][0].translate(asciiUpper2Lower))
- for name, _ in self.currentToken["data"][:-1]:
- if self.currentToken["data"][-1][0] == name:
- self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
- "duplicate-attribute"})
- break
+ self.currentAttribute = ascii_lower(self.currentAttribute)
+ if self.currentAttribute in self.currentToken["data"]:
+ self.tokenQueue.append({"type": tokenTypes["ParseError"],
+ "data": "duplicate-attribute"})
+ self.currentToken["data"][self.currentAttribute].append("")
+ else:
+ self.currentToken["data"][self.currentAttribute] = [""]
# XXX Fix for above XXX
if emitToken:
self.emitCurrentToken()
@@ -943,31 +1120,31 @@ def afterAttributeNameState(self):
if data in spaceCharacters:
self.stream.charsUntil(spaceCharacters, True)
elif data == "=":
- self.state = self.beforeAttributeValueState
+ self._state = self.beforeAttributeValueState
elif data == ">":
self.emitCurrentToken()
elif data in asciiLetters:
- self.currentToken["data"].append([data, ""])
- self.state = self.attributeNameState
+ self.currentAttribute = data
+ self._state = self.attributeNameState
elif data == "/":
- self.state = self.selfClosingStartTagState
+ self._state = self.selfClosingStartTagState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
- self.currentToken["data"].append(["\uFFFD", ""])
- self.state = self.attributeNameState
+ self.currentAttribute = "\uFFFD"
+ self._state = self.attributeNameState
elif data in ("'", '"', "<"):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"invalid-character-after-attribute-name"})
- self.currentToken["data"].append([data, ""])
- self.state = self.attributeNameState
+ self.currentAttribute = data
+ self._state = self.attributeNameState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-end-of-tag-but-got-eof"})
- self.state = self.dataState
+ self._state = self.dataState
else:
- self.currentToken["data"].append([data, ""])
- self.state = self.attributeNameState
+ self.currentAttribute = data
+ self._state = self.attributeNameState
return True
def beforeAttributeValueState(self):
@@ -975,12 +1152,12 @@ def beforeAttributeValueState(self):
if data in spaceCharacters:
self.stream.charsUntil(spaceCharacters, True)
elif data == "\"":
- self.state = self.attributeValueDoubleQuotedState
+ self._state = self.attributeValueDoubleQuotedState
elif data == "&":
- self.state = self.attributeValueUnQuotedState
+ self._state = self.attributeValueUnQuotedState
self.stream.unget(data)
elif data == "'":
- self.state = self.attributeValueSingleQuotedState
+ self._state = self.attributeValueSingleQuotedState
elif data == ">":
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-attribute-value-but-got-right-bracket"})
@@ -988,64 +1165,64 @@ def beforeAttributeValueState(self):
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
- self.currentToken["data"][-1][1] += "\uFFFD"
- self.state = self.attributeValueUnQuotedState
+ self.currentToken["data"][self.currentAttribute][-1] += "\uFFFD"
+ self._state = self.attributeValueUnQuotedState
elif data in ("=", "<", "`"):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"equals-in-unquoted-attribute-value"})
- self.currentToken["data"][-1][1] += data
- self.state = self.attributeValueUnQuotedState
+ self.currentToken["data"][self.currentAttribute][-1] += data
+ self._state = self.attributeValueUnQuotedState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-attribute-value-but-got-eof"})
- self.state = self.dataState
+ self._state = self.dataState
else:
- self.currentToken["data"][-1][1] += data
- self.state = self.attributeValueUnQuotedState
+ self.currentToken["data"][self.currentAttribute][-1] += data
+ self._state = self.attributeValueUnQuotedState
return True
def attributeValueDoubleQuotedState(self):
data = self.stream.char()
if data == "\"":
- self.state = self.afterAttributeValueState
+ self._state = self.afterAttributeValueState
elif data == "&":
self.processEntityInAttribute('"')
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
- self.currentToken["data"][-1][1] += "\uFFFD"
+ self.currentToken["data"][self.currentAttribute][-1] += "\uFFFD"
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-attribute-value-double-quote"})
- self.state = self.dataState
+ self._state = self.dataState
else:
- self.currentToken["data"][-1][1] += data +\
+ self.currentToken["data"][self.currentAttribute][-1] += data +\
self.stream.charsUntil(("\"", "&", "\u0000"))
return True
def attributeValueSingleQuotedState(self):
data = self.stream.char()
if data == "'":
- self.state = self.afterAttributeValueState
+ self._state = self.afterAttributeValueState
elif data == "&":
self.processEntityInAttribute("'")
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
- self.currentToken["data"][-1][1] += "\uFFFD"
+ self.currentToken["data"][self.currentAttribute][-1] += "\uFFFD"
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-attribute-value-single-quote"})
- self.state = self.dataState
+ self._state = self.dataState
else:
- self.currentToken["data"][-1][1] += data +\
+ self.currentToken["data"][self.currentAttribute][-1] += data +\
self.stream.charsUntil(("'", "&", "\u0000"))
return True
def attributeValueUnQuotedState(self):
data = self.stream.char()
if data in spaceCharacters:
- self.state = self.beforeAttributeNameState
+ self._state = self.beforeAttributeNameState
elif data == "&":
self.processEntityInAttribute(">")
elif data == ">":
@@ -1053,38 +1230,38 @@ def attributeValueUnQuotedState(self):
elif data in ('"', "'", "=", "<", "`"):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-character-in-unquoted-attribute-value"})
- self.currentToken["data"][-1][1] += data
+ self.currentToken["data"][self.currentAttribute][-1] += data
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
- self.currentToken["data"][-1][1] += "\uFFFD"
+ self.currentToken["data"][self.currentAttribute][-1] += "\uFFFD"
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-attribute-value-no-quotes"})
- self.state = self.dataState
+ self._state = self.dataState
else:
- self.currentToken["data"][-1][1] += data + self.stream.charsUntil(
+ self.currentToken["data"][self.currentAttribute][-1] += data + self.stream.charsUntil(
frozenset(("&", ">", '"', "'", "=", "<", "`", "\u0000")) | spaceCharacters)
return True
def afterAttributeValueState(self):
data = self.stream.char()
if data in spaceCharacters:
- self.state = self.beforeAttributeNameState
+ self._state = self.beforeAttributeNameState
elif data == ">":
self.emitCurrentToken()
elif data == "/":
- self.state = self.selfClosingStartTagState
+ self._state = self.selfClosingStartTagState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-EOF-after-attribute-value"})
self.stream.unget(data)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-character-after-attribute-value"})
self.stream.unget(data)
- self.state = self.beforeAttributeNameState
+ self._state = self.beforeAttributeNameState
return True
def selfClosingStartTagState(self):
@@ -1097,12 +1274,12 @@ def selfClosingStartTagState(self):
"data":
"unexpected-EOF-after-solidus-in-tag"})
self.stream.unget(data)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-character-after-solidus-in-tag"})
self.stream.unget(data)
- self.state = self.beforeAttributeNameState
+ self._state = self.beforeAttributeNameState
return True
def bogusCommentState(self):
@@ -1117,7 +1294,7 @@ def bogusCommentState(self):
# Eat the character directly after the bogus comment which is either a
# ">" or an EOF.
self.stream.char()
- self.state = self.dataState
+ self._state = self.dataState
return True
def markupDeclarationOpenState(self):
@@ -1126,7 +1303,7 @@ def markupDeclarationOpenState(self):
charStack.append(self.stream.char())
if charStack[-1] == "-":
self.currentToken = {"type": tokenTypes["Comment"], "data": ""}
- self.state = self.commentStartState
+ self._state = self.commentStartState
return True
elif charStack[-1] in ('d', 'D'):
matched = True
@@ -1141,7 +1318,7 @@ def markupDeclarationOpenState(self):
"name": "",
"publicId": None, "systemId": None,
"correct": True}
- self.state = self.doctypeState
+ self._state = self.doctypeState
return True
elif (charStack[-1] == "[" and
self.parser is not None and
@@ -1154,7 +1331,7 @@ def markupDeclarationOpenState(self):
matched = False
break
if matched:
- self.state = self.cdataSectionState
+ self._state = self.cdataSectionState
return True
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
@@ -1162,13 +1339,13 @@ def markupDeclarationOpenState(self):
while charStack:
self.stream.unget(charStack.pop())
- self.state = self.bogusCommentState
+ self._state = self.bogusCommentState
return True
def commentStartState(self):
data = self.stream.char()
if data == "-":
- self.state = self.commentStartDashState
+ self._state = self.commentStartDashState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -1177,21 +1354,21 @@ def commentStartState(self):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"incorrect-comment"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-comment"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["data"] += data
- self.state = self.commentState
+ self._state = self.commentState
return True
def commentStartDashState(self):
data = self.stream.char()
if data == "-":
- self.state = self.commentEndState
+ self._state = self.commentEndState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -1200,21 +1377,21 @@ def commentStartDashState(self):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"incorrect-comment"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-comment"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["data"] += "-" + data
- self.state = self.commentState
+ self._state = self.commentState
return True
def commentState(self):
data = self.stream.char()
if data == "-":
- self.state = self.commentEndDashState
+ self._state = self.commentEndDashState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -1223,7 +1400,7 @@ def commentState(self):
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "eof-in-comment"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["data"] += data + \
self.stream.charsUntil(("-", "\u0000"))
@@ -1232,36 +1409,36 @@ def commentState(self):
def commentEndDashState(self):
data = self.stream.char()
if data == "-":
- self.state = self.commentEndState
+ self._state = self.commentEndState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.currentToken["data"] += "-\uFFFD"
- self.state = self.commentState
+ self._state = self.commentState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-comment-end-dash"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["data"] += "-" + data
- self.state = self.commentState
+ self._state = self.commentState
return True
def commentEndState(self):
data = self.stream.char()
if data == ">":
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.currentToken["data"] += "--\uFFFD"
- self.state = self.commentState
+ self._state = self.commentState
elif data == "!":
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-bang-after-double-dash-in-comment"})
- self.state = self.commentEndBangState
+ self._state = self.commentEndBangState
elif data == "-":
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-dash-after-double-dash-in-comment"})
@@ -1270,53 +1447,53 @@ def commentEndState(self):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-comment-double-dash"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
# XXX
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-comment"})
self.currentToken["data"] += "--" + data
- self.state = self.commentState
+ self._state = self.commentState
return True
def commentEndBangState(self):
data = self.stream.char()
if data == ">":
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data == "-":
self.currentToken["data"] += "--!"
- self.state = self.commentEndDashState
+ self._state = self.commentEndDashState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.currentToken["data"] += "--!\uFFFD"
- self.state = self.commentState
+ self._state = self.commentState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-comment-end-bang-state"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["data"] += "--!" + data
- self.state = self.commentState
+ self._state = self.commentState
return True
def doctypeState(self):
data = self.stream.char()
if data in spaceCharacters:
- self.state = self.beforeDoctypeNameState
+ self._state = self.beforeDoctypeNameState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-doctype-name-but-got-eof"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"need-space-after-doctype"})
self.stream.unget(data)
- self.state = self.beforeDoctypeNameState
+ self._state = self.beforeDoctypeNameState
return True
def beforeDoctypeNameState(self):
@@ -1328,44 +1505,44 @@ def beforeDoctypeNameState(self):
"expected-doctype-name-but-got-right-bracket"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.currentToken["name"] = "\uFFFD"
- self.state = self.doctypeNameState
+ self._state = self.doctypeNameState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"expected-doctype-name-but-got-eof"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["name"] = data
- self.state = self.doctypeNameState
+ self._state = self.doctypeNameState
return True
def doctypeNameState(self):
data = self.stream.char()
if data in spaceCharacters:
- self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
- self.state = self.afterDoctypeNameState
+ self.currentToken["name"] = ascii_lower(self.currentToken["name"])
+ self._state = self.afterDoctypeNameState
elif data == ">":
- self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
+ self.currentToken["name"] = ascii_lower(self.currentToken["name"])
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
self.currentToken["name"] += "\uFFFD"
- self.state = self.doctypeNameState
+ self._state = self.doctypeNameState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype-name"})
self.currentToken["correct"] = False
- self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
+ self.currentToken["name"] = ascii_lower(self.currentToken["name"])
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["name"] += data
return True
@@ -1376,14 +1553,14 @@ def afterDoctypeNameState(self):
pass
elif data == ">":
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.currentToken["correct"] = False
self.stream.unget(data)
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
if data in ("p", "P"):
matched = True
@@ -1394,7 +1571,7 @@ def afterDoctypeNameState(self):
matched = False
break
if matched:
- self.state = self.afterDoctypePublicKeywordState
+ self._state = self.afterDoctypePublicKeywordState
return True
elif data in ("s", "S"):
matched = True
@@ -1405,7 +1582,7 @@ def afterDoctypeNameState(self):
matched = False
break
if matched:
- self.state = self.afterDoctypeSystemKeywordState
+ self._state = self.afterDoctypeSystemKeywordState
return True
# All the characters read before the current 'data' will be
@@ -1417,28 +1594,28 @@ def afterDoctypeNameState(self):
"expected-space-or-right-bracket-in-doctype", "datavars":
{"data": data}})
self.currentToken["correct"] = False
- self.state = self.bogusDoctypeState
+ self._state = self.bogusDoctypeState
return True
def afterDoctypePublicKeywordState(self):
data = self.stream.char()
if data in spaceCharacters:
- self.state = self.beforeDoctypePublicIdentifierState
+ self._state = self.beforeDoctypePublicIdentifierState
elif data in ("'", '"'):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.stream.unget(data)
- self.state = self.beforeDoctypePublicIdentifierState
+ self._state = self.beforeDoctypePublicIdentifierState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.stream.unget(data)
- self.state = self.beforeDoctypePublicIdentifierState
+ self._state = self.beforeDoctypePublicIdentifierState
return True
def beforeDoctypePublicIdentifierState(self):
@@ -1447,33 +1624,33 @@ def beforeDoctypePublicIdentifierState(self):
pass
elif data == "\"":
self.currentToken["publicId"] = ""
- self.state = self.doctypePublicIdentifierDoubleQuotedState
+ self._state = self.doctypePublicIdentifierDoubleQuotedState
elif data == "'":
self.currentToken["publicId"] = ""
- self.state = self.doctypePublicIdentifierSingleQuotedState
+ self._state = self.doctypePublicIdentifierSingleQuotedState
elif data == ">":
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-end-of-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.currentToken["correct"] = False
- self.state = self.bogusDoctypeState
+ self._state = self.bogusDoctypeState
return True
def doctypePublicIdentifierDoubleQuotedState(self):
data = self.stream.char()
if data == "\"":
- self.state = self.afterDoctypePublicIdentifierState
+ self._state = self.afterDoctypePublicIdentifierState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -1483,13 +1660,13 @@ def doctypePublicIdentifierDoubleQuotedState(self):
"unexpected-end-of-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["publicId"] += data
return True
@@ -1497,7 +1674,7 @@ def doctypePublicIdentifierDoubleQuotedState(self):
def doctypePublicIdentifierSingleQuotedState(self):
data = self.stream.char()
if data == "'":
- self.state = self.afterDoctypePublicIdentifierState
+ self._state = self.afterDoctypePublicIdentifierState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -1507,13 +1684,13 @@ def doctypePublicIdentifierSingleQuotedState(self):
"unexpected-end-of-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["publicId"] += data
return True
@@ -1521,31 +1698,31 @@ def doctypePublicIdentifierSingleQuotedState(self):
def afterDoctypePublicIdentifierState(self):
data = self.stream.char()
if data in spaceCharacters:
- self.state = self.betweenDoctypePublicAndSystemIdentifiersState
+ self._state = self.betweenDoctypePublicAndSystemIdentifiersState
elif data == ">":
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data == '"':
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.currentToken["systemId"] = ""
- self.state = self.doctypeSystemIdentifierDoubleQuotedState
+ self._state = self.doctypeSystemIdentifierDoubleQuotedState
elif data == "'":
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.currentToken["systemId"] = ""
- self.state = self.doctypeSystemIdentifierSingleQuotedState
+ self._state = self.doctypeSystemIdentifierSingleQuotedState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.currentToken["correct"] = False
- self.state = self.bogusDoctypeState
+ self._state = self.bogusDoctypeState
return True
def betweenDoctypePublicAndSystemIdentifiersState(self):
@@ -1554,44 +1731,44 @@ def betweenDoctypePublicAndSystemIdentifiersState(self):
pass
elif data == ">":
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data == '"':
self.currentToken["systemId"] = ""
- self.state = self.doctypeSystemIdentifierDoubleQuotedState
+ self._state = self.doctypeSystemIdentifierDoubleQuotedState
elif data == "'":
self.currentToken["systemId"] = ""
- self.state = self.doctypeSystemIdentifierSingleQuotedState
+ self._state = self.doctypeSystemIdentifierSingleQuotedState
elif data == EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.currentToken["correct"] = False
- self.state = self.bogusDoctypeState
+ self._state = self.bogusDoctypeState
return True
def afterDoctypeSystemKeywordState(self):
data = self.stream.char()
if data in spaceCharacters:
- self.state = self.beforeDoctypeSystemIdentifierState
+ self._state = self.beforeDoctypeSystemIdentifierState
elif data in ("'", '"'):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.stream.unget(data)
- self.state = self.beforeDoctypeSystemIdentifierState
+ self._state = self.beforeDoctypeSystemIdentifierState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.stream.unget(data)
- self.state = self.beforeDoctypeSystemIdentifierState
+ self._state = self.beforeDoctypeSystemIdentifierState
return True
def beforeDoctypeSystemIdentifierState(self):
@@ -1600,33 +1777,33 @@ def beforeDoctypeSystemIdentifierState(self):
pass
elif data == "\"":
self.currentToken["systemId"] = ""
- self.state = self.doctypeSystemIdentifierDoubleQuotedState
+ self._state = self.doctypeSystemIdentifierDoubleQuotedState
elif data == "'":
self.currentToken["systemId"] = ""
- self.state = self.doctypeSystemIdentifierSingleQuotedState
+ self._state = self.doctypeSystemIdentifierSingleQuotedState
elif data == ">":
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
self.currentToken["correct"] = False
- self.state = self.bogusDoctypeState
+ self._state = self.bogusDoctypeState
return True
def doctypeSystemIdentifierDoubleQuotedState(self):
data = self.stream.char()
if data == "\"":
- self.state = self.afterDoctypeSystemIdentifierState
+ self._state = self.afterDoctypeSystemIdentifierState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -1636,13 +1813,13 @@ def doctypeSystemIdentifierDoubleQuotedState(self):
"unexpected-end-of-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["systemId"] += data
return True
@@ -1650,7 +1827,7 @@ def doctypeSystemIdentifierDoubleQuotedState(self):
def doctypeSystemIdentifierSingleQuotedState(self):
data = self.stream.char()
if data == "'":
- self.state = self.afterDoctypeSystemIdentifierState
+ self._state = self.afterDoctypeSystemIdentifierState
elif data == "\u0000":
self.tokenQueue.append({"type": tokenTypes["ParseError"],
"data": "invalid-codepoint"})
@@ -1660,13 +1837,13 @@ def doctypeSystemIdentifierSingleQuotedState(self):
"unexpected-end-of-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.currentToken["systemId"] += data
return True
@@ -1677,29 +1854,29 @@ def afterDoctypeSystemIdentifierState(self):
pass
elif data == ">":
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype"})
self.currentToken["correct"] = False
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"unexpected-char-in-doctype"})
- self.state = self.bogusDoctypeState
+ self._state = self.bogusDoctypeState
return True
def bogusDoctypeState(self):
data = self.stream.char()
if data == ">":
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
elif data is EOF:
# XXX EMIT
self.stream.unget(data)
self.tokenQueue.append(self.currentToken)
- self.state = self.dataState
+ self._state = self.dataState
else:
pass
return True
@@ -1731,5 +1908,5 @@ def cdataSectionState(self):
if data:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": data})
- self.state = self.dataState
+ self._state = self.dataState
return True
diff --git a/html5lib/_utils.py b/html5lib/_utils.py
index 9ea57942..7e23ee57 100644
--- a/html5lib/_utils.py
+++ b/html5lib/_utils.py
@@ -145,15 +145,3 @@ def moduleFactory(baseModule, *args, **kwargs):
return mod
return moduleFactory
-
-
-def memoize(func):
- cache = {}
-
- def wrapped(*args, **kwargs):
- key = (tuple(args), tuple(kwargs.items()))
- if key not in cache:
- cache[key] = func(*args, **kwargs)
- return cache[key]
-
- return wrapped
diff --git a/html5lib/html5parser.pxd b/html5lib/html5parser.pxd
new file mode 100644
index 00000000..4b1b62c8
--- /dev/null
+++ b/html5lib/html5parser.pxd
@@ -0,0 +1,379 @@
+cimport cython
+
+from . cimport _tokenizer
+
+# # def parse(doc, treebuilder="etree", namespaceHTMLElements=True, **kwargs)
+
+# # def parseFragment(doc, container="div", treebuilder="etree", namespaceHTMLElements=True, **kwargs)
+
+cdef class HTMLParser(object):
+ cdef bint strict
+ cdef bint debug
+ cdef readonly object tree
+ cdef readonly list errors
+ cdef readonly dict phases
+ cdef bint innerHTMLMode
+ cdef unicode container
+ cdef readonly bint scripting
+ cdef readonly _tokenizer.HTMLTokenizer tokenizer
+ cdef public bint firstStartTag
+ cdef readonly list log
+ cdef public unicode compatMode
+ cdef readonly unicode innerHTML
+ cdef public object phase
+ cdef public bint framesetOK
+ cdef public object originalPhase
+
+
+ #def _parse(self, stream, innerHTML=False, container="div", scripting=False, **kwargs)
+ cdef reset(self)
+ #cdef documentEncoding(self)
+ cpdef bint isHTMLIntegrationPoint(self, object element) except? -1
+ cpdef bint isMathMLTextIntegrationPoint(self, object element) except? -1
+ @cython.locals(
+ CharactersToken=int,
+ SpaceCharactersToken=int,
+ StartTagToken=int,
+ EndTagToken=int,
+ CommentToken=int,
+ DoctypeToken=int,
+ ParseErrorToken=int,
+ defaultNamespace=unicode,
+ token=dict,
+ prev_token=dict,
+ new_token=dict,
+ openElements=list,
+ type=int
+ )
+ cdef mainLoop(self)
+ #def parse(self, stream, *args, **kwargs)
+ #def parseFragment(self, stream, *args, **kwargs)
+ cpdef void parseError(self, errorcode=?, datavars=?) except *
+ cpdef void adjustMathMLAttributes(self, dict token) except *
+ cpdef void adjustSVGAttributes(self, dict token) except *
+ cpdef void adjustForeignAttributes(self, dict token) except *
+ cdef void reparseTokenNormal(self, dict token) except *
+ cpdef void resetInsertionMode(self) except *
+ cpdef void parseRCDataRawtext(self, dict token, unicode contentType) except *
+
+# cdef class Phase(object):
+# cdef readonly object parser
+# cdef readonly object tree
+# cdef dict __startTagCache
+# cdef dict __endTagCache
+
+# cpdef processEOF(self)
+# cpdef processComment(self, dict token)
+# cpdef processDoctype(self, dict token)
+# cpdef processCharacters(self, dict token)
+# cpdef processSpaceCharacters(self, dict token)
+# cpdef processStartTag(self, dict token)
+# cpdef startTagHtml(self, dict token)
+# cpdef processEndTag(self, dict token)
+
+# # cdef class InitialPhase(Phase):
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processComment(self, dict token)
+# # cdef processDoctype(self, dict token)
+# # cdef anythingElse(self)
+# # cdef processCharacters(self, dict token)
+# # cdef processStartTag(self, dict token)
+# # cdef processEndTag(self, dict token)
+# # cdef processEOF(self)
+
+# # cdef class BeforeHtmlPhase(Phase):
+# # cdef insertHtmlElement(self)
+# # cdef processEOF(self)
+# # cdef processComment(self, dict token)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef processStartTag(self, dict token)
+# # cdef processEndTag(self, dict token)
+
+# # cdef class BeforeHeadPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagHtml(self, dict token)
+# # cdef startTagHead(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagImplyHead(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InHeadPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagHtml(self, dict token)
+# # cdef startTagHead(self, dict token)
+# # cdef startTagBaseLinkCommand(self, dict token)
+# # cdef startTagMeta(self, dict token)
+# # cdef startTagTitle(self, dict token)
+# # cdef startTagNoFramesStyle(self, dict token)
+# # cdef startTagNoscript(self, dict token)
+# # cdef startTagScript(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagHead(self, dict token)
+# # cdef endTagHtmlBodyBr(self, dict token)
+# # cdef endTagOther(self, dict token)
+# # cdef anythingElse(self)
+
+# # cdef class InHeadNoscriptPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processComment(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef startTagHtml(self, dict token)
+# # cdef startTagBaseLinkCommand(self, dict token)
+# # cdef startTagHeadNoscript(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagNoscript(self, dict token)
+# # cdef endTagBr(self, dict token)
+# # cdef endTagOther(self, dict token)
+# # cdef anythingElse(self)
+
+# # cdef class AfterHeadPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagHtml(self, dict token)
+# # cdef startTagBody(self, dict token)
+# # cdef startTagFrameset(self, dict token)
+# # cdef startTagFromHead(self, dict token)
+# # cdef startTagHead(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagHtmlBodyBr(self, dict token)
+# # cdef endTagOther(self, dict token)
+# # cdef anythingElse(self)
+
+# # cdef class InBodyPhase(Phase):
+# # cdef bint dropNewline
+# # cdef isMatchingFormattingElement(self, node1, node2)
+# # cdef addFormattingElement(self, dict token)
+# # cdef processEOF(self)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processSpaceCharactersDropNewline(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef processSpaceCharactersNonPre(self, dict token)
+# # cdef startTagProcessInHead(self, dict token)
+# # cdef startTagBody(self, dict token)
+# # cdef startTagFrameset(self, dict token)
+# # cdef startTagCloseP(self, dict token)
+# # cdef startTagPreListing(self, dict token)
+# # cdef startTagForm(self, dict token)
+# # cdef startTagListItem(self, dict token)
+# # cdef startTagPlaintext(self, dict token)
+# # cdef startTagHeading(self, dict token)
+# # cdef startTagA(self, dict token)
+# # cdef startTagFormatting(self, dict token)
+# # cdef startTagNobr(self, dict token)
+# # cdef startTagButton(self, dict token)
+# # cdef startTagAppletMarqueeObject(self, dict token)
+# # cdef startTagXmp(self, dict token)
+# # cdef startTagTable(self, dict token)
+# # cdef startTagVoidFormatting(self, dict token)
+# # cdef startTagInput(self, dict token)
+# # cdef startTagParamSource(self, dict token)
+# # cdef startTagHr(self, dict token)
+# # cdef startTagImage(self, dict token)
+# # cdef startTagIsIndex(self, dict token)
+# # cdef startTagTextarea(self, dict token)
+# # cdef startTagIFrame(self, dict token)
+# # cdef startTagNoscript(self, dict token)
+# # cdef startTagRawtext(self, dict token)
+# # cdef startTagOpt(self, dict token)
+# # cdef startTagSelect(self, dict token)
+# # cdef startTagRpRt(self, dict token)
+# # cdef startTagMath(self, dict token)
+# # cdef startTagSvg(self, dict token)
+# # cdef startTagMisplaced(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagP(self, dict token)
+# # cdef endTagBody(self, dict token)
+# # cdef endTagHtml(self, dict token)
+# # cdef endTagBlock(self, dict token)
+# # cdef endTagForm(self, dict token)
+# # cdef endTagListItem(self, dict token)
+# # cdef endTagHeading(self, dict token)
+# # cdef endTagFormatting(self, dict token)
+# # cdef endTagAppletMarqueeObject(self, dict token)
+# # cdef endTagBr(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class TextPhase(Phase):
+# # cdef processCharacters(self, dict token)
+# # cdef processEOF(self)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagScript(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InTablePhase(Phase):
+# # cdef object originalPhase
+# # cdef object characterTokens
+
+# # cdef clearStackToTableContext(self)
+# # cdef processEOF(self)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef insertText(self, dict token)
+# # cdef startTagCaption(self, dict token)
+# # cdef startTagColgroup(self, dict token)
+# # cdef startTagCol(self, dict token)
+# # cdef startTagRowGroup(self, dict token)
+# # cdef startTagImplyTbody(self, dict token)
+# # cdef startTagTable(self, dict token)
+# # cdef startTagStyleScript(self, dict token)
+# # cdef startTagInput(self, dict token)
+# # cdef startTagForm(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagTable(self, dict token)
+# # cdef endTagIgnore(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InTableTextPhase(Phase):
+# # cdef flushCharacters(self)
+# # cdef processComment(self, dict token)
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processStartTag(self, dict token)
+# # cdef processEndTag(self, dict token)
+
+# # cdef class InCaptionPhase(Phase):
+# # cdef ignoreEndTagCaption(self)
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagTableElement(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagCaption(self, dict token)
+# # cdef endTagTable(self, dict token)
+# # cdef endTagIgnore(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InColumnGroupPhase(Phase):
+# # cdef ignoreEndTagColgroup(self)
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagCol(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagColgroup(self, dict token)
+# # cdef endTagCol(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InTableBodyPhase(Phase):
+# # cdef clearStackToTableBodyContext(self)
+# # cdef processEOF(self)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagTr(self, dict token)
+# # cdef startTagTableCell(self, dict token)
+# # cdef startTagTableOther(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagTableRowGroup(self, dict token)
+# # cdef endTagTable(self, dict token)
+# # cdef endTagIgnore(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InRowPhase(Phase):
+# # cdef clearStackToTableRowContext(self)
+# # cdef ignoreEndTagTr(self)
+# # cdef processEOF(self)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagTableCell(self, dict token)
+# # cdef startTagTableOther(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagTr(self, dict token)
+# # cdef endTagTable(self, dict token)
+# # cdef endTagTableRowGroup(self, dict token)
+# # cdef endTagIgnore(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InCellPhase(Phase):
+# # cdef closeCell(self)
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagTableOther(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagTableCell(self, dict token)
+# # cdef endTagIgnore(self, dict token)
+# # cdef endTagImply(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InSelectPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagOption(self, dict token)
+# # cdef startTagOptgroup(self, dict token)
+# # cdef startTagSelect(self, dict token)
+# # cdef startTagInput(self, dict token)
+# # cdef startTagScript(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagOption(self, dict token)
+# # cdef endTagOptgroup(self, dict token)
+# # cdef endTagSelect(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InSelectInTablePhase(Phase):
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagTable(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagTable(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InForeignContentPhase(Phase):
+# # cdef adjustSVGTagNames(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef processStartTag(self, dict token)
+# # cdef processEndTag(self, dict token)
+
+# # cdef class AfterBodyPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processComment(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagHtml(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagHtml(self, name)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class InFramesetPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagFrameset(self, dict token)
+# # cdef startTagFrame(self, dict token)
+# # cdef startTagNoframes(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagFrameset(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class AfterFramesetPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagNoframes(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef endTagHtml(self, dict token)
+# # cdef endTagOther(self, dict token)
+
+# # cdef class AfterAfterBodyPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processComment(self, dict token)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagHtml(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef processEndTag(self, dict token)
+
+# # cdef class AfterAfterFramesetPhase(Phase):
+# # cdef processEOF(self)
+# # cdef processComment(self, dict token)
+# # cdef processSpaceCharacters(self, dict token)
+# # cdef processCharacters(self, dict token)
+# # cdef startTagHtml(self, dict token)
+# # cdef startTagNoFrames(self, dict token)
+# # cdef startTagOther(self, dict token)
+# # cdef processEndTag(self, dict token)
+
+cdef inline void adjust_attributes(dict token, dict replacements) except *
+cdef inline dict impliedTagToken(unicode name, unicode type=?, dict attributes=?, bint selfClosing=?)
+
+# cdef class ParseError(Exception):
+# pass
diff --git a/html5lib/html5parser.py b/html5lib/html5parser.py
index 74d829d9..b2cce0eb 100644
--- a/html5lib/html5parser.py
+++ b/html5lib/html5parser.py
@@ -1,7 +1,6 @@
from __future__ import absolute_import, division, unicode_literals
-from six import with_metaclass, viewkeys
-
-import types
+import cython
+from six import viewkeys
from . import _inputstream
from . import _tokenizer
@@ -13,7 +12,7 @@
from .constants import (
spaceCharacters, asciiUpper2Lower,
specialElements, headingElements, cdataElements, rcdataElements,
- tokenTypes, tagTokenTypes,
+ tokenTypes,
namespaces,
htmlIntegrationPointElements, mathmlTextIntegrationPointElements,
adjustForeignAttributes as adjustForeignAttributesMap,
@@ -71,18 +70,6 @@ def parseFragment(doc, container="div", treebuilder="etree", namespaceHTMLElemen
return p.parseFragment(doc, container=container, **kwargs)
-def method_decorator_metaclass(function):
- class Decorated(type):
- def __new__(meta, classname, bases, classDict):
- for attributeName, attribute in classDict.items():
- if isinstance(attribute, types.FunctionType):
- attribute = function(attribute)
-
- classDict[attributeName] = attribute
- return type.__new__(meta, classname, bases, classDict)
- return Decorated
-
-
class HTMLParser(object):
"""HTML parser
@@ -112,6 +99,7 @@ def __init__(self, tree=None, strict=False, namespaceHTMLElements=True, debug=Fa
# Raise an exception on the first error encountered
self.strict = strict
+ self.debug = debug
if tree is None:
tree = treebuilders.getTreeBuilder("etree")
@@ -119,7 +107,8 @@ def __init__(self, tree=None, strict=False, namespaceHTMLElements=True, debug=Fa
self.errors = []
self.phases = {name: cls(self, self.tree) for name, cls in
- getPhases(debug).items()}
+ _phases.items()}
+ self.tokenizer = None
def _parse(self, stream, innerHTML=False, container="div", scripting=False, **kwargs):
@@ -147,14 +136,14 @@ def reset(self):
self.innerHTML = self.container.lower()
if self.innerHTML in cdataElements:
- self.tokenizer.state = self.tokenizer.rcdataState
+ self.tokenizer.state = "rcdataState"
elif self.innerHTML in rcdataElements:
- self.tokenizer.state = self.tokenizer.rawtextState
+ self.tokenizer.state = "rawtextState"
elif self.innerHTML == 'plaintext':
- self.tokenizer.state = self.tokenizer.plaintextState
+ self.tokenizer.state = "plaintextState"
else:
# state already is data state
- # self.tokenizer.state = self.tokenizer.dataState
+ # self.tokenizer.state = "dataState"
pass
self.phase = self.phases["beforeHtml"]
self.phase.insertHtmlElement()
@@ -163,10 +152,6 @@ def reset(self):
self.innerHTML = False # pylint:disable=redefined-variable-type
self.phase = self.phases["initial"]
- self.lastPhase = None
-
- self.beforeRCDataPhase = None
-
self.framesetOK = True
@property
@@ -175,7 +160,7 @@ def documentEncoding(self):
:obj:`None` if that is not determined yet
"""
- if not hasattr(self, 'tokenizer'):
+ if self.tokenizer is None:
return None
return self.tokenizer.stream.charEncoding[0].name
@@ -201,14 +186,19 @@ def mainLoop(self):
DoctypeToken = tokenTypes["Doctype"]
ParseErrorToken = tokenTypes["ParseError"]
+ type_names = {value: key for key, value in tokenTypes.items()}
+ debug = self.debug
+ defaultNamespace = self.tree.defaultNamespace
+
for token in self.tokenizer:
prev_token = None
new_token = token
while new_token is not None:
prev_token = new_token
- currentNode = self.tree.openElements[-1] if self.tree.openElements else None
- currentNodeNamespace = currentNode.namespace if currentNode else None
- currentNodeName = currentNode.name if currentNode else None
+ openElements = self.tree.openElements
+ currentNode = openElements[-1] if openElements else None
+ currentNodeNamespace = currentNode.namespace if currentNode is not None else None
+ currentNodeName = currentNode.name if currentNode is not None else None
type = new_token["type"]
@@ -216,11 +206,11 @@ def mainLoop(self):
self.parseError(new_token["data"], new_token.get("datavars", {}))
new_token = None
else:
- if (len(self.tree.openElements) == 0 or
- currentNodeNamespace == self.tree.defaultNamespace or
+ if (not openElements or
+ currentNodeNamespace == defaultNamespace or
(self.isMathMLTextIntegrationPoint(currentNode) and
((type == StartTagToken and
- token["name"] not in frozenset(["mglyph", "malignmark"])) or
+ token["name"] not in ("mglyph", "malignmark")) or
type in (CharactersToken, SpaceCharactersToken))) or
(currentNodeNamespace == namespaces["mathml"] and
currentNodeName == "annotation-xml" and
@@ -232,6 +222,17 @@ def mainLoop(self):
else:
phase = self.phases["inForeignContent"]
+ if debug:
+ info = {"type": type_names[type]}
+ if type in (StartTagToken, EndTagToken):
+ info["name"] = new_token['name']
+
+ self.log.append((self.tokenizer.state,
+ self.phase.__class__.__name__,
+ phase.__class__.__name__,
+ "process" + info["type"],
+ info))
+
if type == CharactersToken:
new_token = phase.processCharacters(new_token)
elif type == SpaceCharactersToken:
@@ -384,2402 +385,2410 @@ def parseRCDataRawtext(self, token, contentType):
self.tree.insertElement(token)
if contentType == "RAWTEXT":
- self.tokenizer.state = self.tokenizer.rawtextState
+ self.tokenizer.state = "rawtextState"
else:
- self.tokenizer.state = self.tokenizer.rcdataState
+ self.tokenizer.state = "rcdataState"
self.originalPhase = self.phase
self.phase = self.phases["text"]
-@_utils.memoize
-def getPhases(debug):
- def log(function):
- """Logger that records which phase processes each token"""
- type_names = {value: key for key, value in tokenTypes.items()}
-
- def wrapped(self, *args, **kwargs):
- if function.__name__.startswith("process") and len(args) > 0:
- token = args[0]
- info = {"type": type_names[token['type']]}
- if token['type'] in tagTokenTypes:
- info["name"] = token['name']
-
- self.parser.log.append((self.parser.tokenizer.state.__name__,
- self.parser.phase.__class__.__name__,
- self.__class__.__name__,
- function.__name__,
- info))
- return function(self, *args, **kwargs)
- else:
- return function(self, *args, **kwargs)
- return wrapped
-
- def getMetaclass(use_metaclass, metaclass_func):
- if use_metaclass:
- return method_decorator_metaclass(metaclass_func)
+class Phase(object):
+ """Base class for helper object that implements each phase of processing
+ """
+ __slots__ = ("parser", "tree", "__startTagCache", "__endTagCache")
+
+ def __init__(self, parser, tree):
+ self.parser = parser
+ self.tree = tree
+ self.__startTagCache = {}
+ self.__endTagCache = {}
+
+ def processEOF(self):
+ raise NotImplementedError
+
+ def processComment(self, token):
+ # For most phases the following is correct. Where it's not it will be
+ # overridden.
+ self.tree.insertComment(token, self.tree.openElements[-1])
+
+ def processDoctype(self, token):
+ self.parser.parseError("unexpected-doctype")
+
+ def processCharacters(self, token):
+ self.tree.insertText(token["data"])
+
+ def processSpaceCharacters(self, token):
+ self.tree.insertText(token["data"])
+
+ def processStartTag(self, token):
+ # Note the caching is done here rather than BoundMethodDispatcher as doing it there
+ # requires a circular reference to the Phase, and this ends up with a significant
+ # (CPython 2.7, 3.8) GC cost when parsing many short inputs
+ name = token["name"]
+ # In Py2, using `in` is quicker in general than try/except KeyError
+ # In Py3, `in` is quicker when there are few cache hits (typically short inputs)
+ if name in self.__startTagCache:
+ func = self.__startTagCache[name]
else:
- return type
-
- # pylint:disable=unused-argument
- class Phase(with_metaclass(getMetaclass(debug, log))):
- """Base class for helper object that implements each phase of processing
- """
- __slots__ = ("parser", "tree", "__startTagCache", "__endTagCache")
-
- def __init__(self, parser, tree):
- self.parser = parser
- self.tree = tree
- self.__startTagCache = {}
- self.__endTagCache = {}
-
- def processEOF(self):
- raise NotImplementedError
-
- def processComment(self, token):
- # For most phases the following is correct. Where it's not it will be
- # overridden.
- self.tree.insertComment(token, self.tree.openElements[-1])
-
- def processDoctype(self, token):
- self.parser.parseError("unexpected-doctype")
-
- def processCharacters(self, token):
- self.tree.insertText(token["data"])
-
- def processSpaceCharacters(self, token):
- self.tree.insertText(token["data"])
-
- def processStartTag(self, token):
- # Note the caching is done here rather than BoundMethodDispatcher as doing it there
- # requires a circular reference to the Phase, and this ends up with a significant
- # (CPython 2.7, 3.8) GC cost when parsing many short inputs
- name = token["name"]
- # In Py2, using `in` is quicker in general than try/except KeyError
- # In Py3, `in` is quicker when there are few cache hits (typically short inputs)
- if name in self.__startTagCache:
- func = self.__startTagCache[name]
- else:
- func = self.__startTagCache[name] = self.startTagHandler[name]
- # bound the cache size in case we get loads of unknown tags
- while len(self.__startTagCache) > len(self.startTagHandler) * 1.1:
- # this makes the eviction policy random on Py < 3.7 and FIFO >= 3.7
- self.__startTagCache.pop(next(iter(self.__startTagCache)))
- return func(token)
-
- def startTagHtml(self, token):
- if not self.parser.firstStartTag and token["name"] == "html":
- self.parser.parseError("non-html-root")
- # XXX Need a check here to see if the first start tag token emitted is
- # this token... If it's not, invoke self.parser.parseError().
- for attr, value in token["data"].items():
- if attr not in self.tree.openElements[0].attributes:
- self.tree.openElements[0].attributes[attr] = value
- self.parser.firstStartTag = False
-
- def processEndTag(self, token):
- # Note the caching is done here rather than BoundMethodDispatcher as doing it there
- # requires a circular reference to the Phase, and this ends up with a significant
- # (CPython 2.7, 3.8) GC cost when parsing many short inputs
- name = token["name"]
- # In Py2, using `in` is quicker in general than try/except KeyError
- # In Py3, `in` is quicker when there are few cache hits (typically short inputs)
- if name in self.__endTagCache:
- func = self.__endTagCache[name]
- else:
- func = self.__endTagCache[name] = self.endTagHandler[name]
- # bound the cache size in case we get loads of unknown tags
- while len(self.__endTagCache) > len(self.endTagHandler) * 1.1:
- # this makes the eviction policy random on Py < 3.7 and FIFO >= 3.7
- self.__endTagCache.pop(next(iter(self.__endTagCache)))
- return func(token)
-
- class InitialPhase(Phase):
- __slots__ = tuple()
-
- def processSpaceCharacters(self, token):
- pass
-
- def processComment(self, token):
- self.tree.insertComment(token, self.tree.document)
-
- def processDoctype(self, token):
- name = token["name"]
- publicId = token["publicId"]
- systemId = token["systemId"]
- correct = token["correct"]
-
- if (name != "html" or publicId is not None or
- systemId is not None and systemId != "about:legacy-compat"):
- self.parser.parseError("unknown-doctype")
-
- if publicId is None:
- publicId = ""
-
- self.tree.insertDoctype(token)
-
- if publicId != "":
- publicId = publicId.translate(asciiUpper2Lower)
-
- if (not correct or token["name"] != "html" or
- publicId.startswith(
- ("+//silmaril//dtd html pro v0r11 19970101//",
- "-//advasoft ltd//dtd html 3.0 aswedit + extensions//",
- "-//as//dtd html 3.0 aswedit + extensions//",
- "-//ietf//dtd html 2.0 level 1//",
- "-//ietf//dtd html 2.0 level 2//",
- "-//ietf//dtd html 2.0 strict level 1//",
- "-//ietf//dtd html 2.0 strict level 2//",
- "-//ietf//dtd html 2.0 strict//",
- "-//ietf//dtd html 2.0//",
- "-//ietf//dtd html 2.1e//",
- "-//ietf//dtd html 3.0//",
- "-//ietf//dtd html 3.2 final//",
- "-//ietf//dtd html 3.2//",
- "-//ietf//dtd html 3//",
- "-//ietf//dtd html level 0//",
- "-//ietf//dtd html level 1//",
- "-//ietf//dtd html level 2//",
- "-//ietf//dtd html level 3//",
- "-//ietf//dtd html strict level 0//",
- "-//ietf//dtd html strict level 1//",
- "-//ietf//dtd html strict level 2//",
- "-//ietf//dtd html strict level 3//",
- "-//ietf//dtd html strict//",
- "-//ietf//dtd html//",
- "-//metrius//dtd metrius presentational//",
- "-//microsoft//dtd internet explorer 2.0 html strict//",
- "-//microsoft//dtd internet explorer 2.0 html//",
- "-//microsoft//dtd internet explorer 2.0 tables//",
- "-//microsoft//dtd internet explorer 3.0 html strict//",
- "-//microsoft//dtd internet explorer 3.0 html//",
- "-//microsoft//dtd internet explorer 3.0 tables//",
- "-//netscape comm. corp.//dtd html//",
- "-//netscape comm. corp.//dtd strict html//",
- "-//o'reilly and associates//dtd html 2.0//",
- "-//o'reilly and associates//dtd html extended 1.0//",
- "-//o'reilly and associates//dtd html extended relaxed 1.0//",
- "-//softquad software//dtd hotmetal pro 6.0::19990601::extensions to html 4.0//",
- "-//softquad//dtd hotmetal pro 4.0::19971010::extensions to html 4.0//",
- "-//spyglass//dtd html 2.0 extended//",
- "-//sq//dtd html 2.0 hotmetal + extensions//",
- "-//sun microsystems corp.//dtd hotjava html//",
- "-//sun microsystems corp.//dtd hotjava strict html//",
- "-//w3c//dtd html 3 1995-03-24//",
- "-//w3c//dtd html 3.2 draft//",
- "-//w3c//dtd html 3.2 final//",
- "-//w3c//dtd html 3.2//",
- "-//w3c//dtd html 3.2s draft//",
- "-//w3c//dtd html 4.0 frameset//",
- "-//w3c//dtd html 4.0 transitional//",
- "-//w3c//dtd html experimental 19960712//",
- "-//w3c//dtd html experimental 970421//",
- "-//w3c//dtd w3 html//",
- "-//w3o//dtd w3 html 3.0//",
- "-//webtechs//dtd mozilla html 2.0//",
- "-//webtechs//dtd mozilla html//")) or
- publicId in ("-//w3o//dtd w3 html strict 3.0//en//",
- "-/w3c/dtd html 4.0 transitional/en",
- "html") or
- publicId.startswith(
- ("-//w3c//dtd html 4.01 frameset//",
- "-//w3c//dtd html 4.01 transitional//")) and
- systemId is None or
- systemId and systemId.lower() == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd"):
- self.parser.compatMode = "quirks"
- elif (publicId.startswith(
- ("-//w3c//dtd xhtml 1.0 frameset//",
- "-//w3c//dtd xhtml 1.0 transitional//")) or
- publicId.startswith(
- ("-//w3c//dtd html 4.01 frameset//",
- "-//w3c//dtd html 4.01 transitional//")) and
- systemId is not None):
- self.parser.compatMode = "limited quirks"
-
- self.parser.phase = self.parser.phases["beforeHtml"]
-
- def anythingElse(self):
+ func = self.__startTagCache[name] = self.startTagHandler[name]
+ # bound the cache size in case we get loads of unknown tags
+ while len(self.__startTagCache) > len(self.startTagHandler) * 1.1:
+ # this makes the eviction policy random on Py < 3.7 and FIFO >= 3.7
+ self.__startTagCache.pop(next(iter(self.__startTagCache)))
+ return func(token)
+
+ def startTagHtml(self, token):
+ if not self.parser.firstStartTag and token["name"] == "html":
+ self.parser.parseError("non-html-root")
+ # XXX Need a check here to see if the first start tag token emitted is
+ # this token... If it's not, invoke self.parser.parseError().
+ for attr, value in token["data"].items():
+ if attr not in self.tree.openElements[0].attributes:
+ self.tree.openElements[0].attributes[attr] = value
+ self.parser.firstStartTag = False
+
+ def processEndTag(self, token):
+ # Note the caching is done here rather than BoundMethodDispatcher as doing it there
+ # requires a circular reference to the Phase, and this ends up with a significant
+ # (CPython 2.7, 3.8) GC cost when parsing many short inputs
+ name = token["name"]
+ # In Py2, using `in` is quicker in general than try/except KeyError
+ # In Py3, `in` is quicker when there are few cache hits (typically short inputs)
+ if name in self.__endTagCache:
+ func = self.__endTagCache[name]
+ else:
+ func = self.__endTagCache[name] = self.endTagHandler[name]
+ # bound the cache size in case we get loads of unknown tags
+ while len(self.__endTagCache) > len(self.endTagHandler) * 1.1:
+ # this makes the eviction policy random on Py < 3.7 and FIFO >= 3.7
+ self.__endTagCache.pop(next(iter(self.__endTagCache)))
+ return func(token)
+
+
+class InitialPhase(Phase):
+ __slots__ = tuple()
+
+ def processSpaceCharacters(self, token):
+ pass
+
+ def processComment(self, token):
+ self.tree.insertComment(token, self.tree.document)
+
+ def processDoctype(self, token):
+ name = token["name"]
+ publicId = token["publicId"]
+ systemId = token["systemId"]
+ correct = token["correct"]
+
+ if (name != "html" or publicId is not None or
+ systemId is not None and systemId != "about:legacy-compat"):
+ self.parser.parseError("unknown-doctype")
+
+ if publicId is None:
+ publicId = ""
+
+ self.tree.insertDoctype(token)
+
+ if publicId != "":
+ publicId = publicId.translate(asciiUpper2Lower)
+
+ if (not correct or token["name"] != "html" or
+ publicId.startswith(
+ ("+//silmaril//dtd html pro v0r11 19970101//",
+ "-//advasoft ltd//dtd html 3.0 aswedit + extensions//",
+ "-//as//dtd html 3.0 aswedit + extensions//",
+ "-//ietf//dtd html 2.0 level 1//",
+ "-//ietf//dtd html 2.0 level 2//",
+ "-//ietf//dtd html 2.0 strict level 1//",
+ "-//ietf//dtd html 2.0 strict level 2//",
+ "-//ietf//dtd html 2.0 strict//",
+ "-//ietf//dtd html 2.0//",
+ "-//ietf//dtd html 2.1e//",
+ "-//ietf//dtd html 3.0//",
+ "-//ietf//dtd html 3.2 final//",
+ "-//ietf//dtd html 3.2//",
+ "-//ietf//dtd html 3//",
+ "-//ietf//dtd html level 0//",
+ "-//ietf//dtd html level 1//",
+ "-//ietf//dtd html level 2//",
+ "-//ietf//dtd html level 3//",
+ "-//ietf//dtd html strict level 0//",
+ "-//ietf//dtd html strict level 1//",
+ "-//ietf//dtd html strict level 2//",
+ "-//ietf//dtd html strict level 3//",
+ "-//ietf//dtd html strict//",
+ "-//ietf//dtd html//",
+ "-//metrius//dtd metrius presentational//",
+ "-//microsoft//dtd internet explorer 2.0 html strict//",
+ "-//microsoft//dtd internet explorer 2.0 html//",
+ "-//microsoft//dtd internet explorer 2.0 tables//",
+ "-//microsoft//dtd internet explorer 3.0 html strict//",
+ "-//microsoft//dtd internet explorer 3.0 html//",
+ "-//microsoft//dtd internet explorer 3.0 tables//",
+ "-//netscape comm. corp.//dtd html//",
+ "-//netscape comm. corp.//dtd strict html//",
+ "-//o'reilly and associates//dtd html 2.0//",
+ "-//o'reilly and associates//dtd html extended 1.0//",
+ "-//o'reilly and associates//dtd html extended relaxed 1.0//",
+ "-//softquad software//dtd hotmetal pro 6.0::19990601::extensions to html 4.0//",
+ "-//softquad//dtd hotmetal pro 4.0::19971010::extensions to html 4.0//",
+ "-//spyglass//dtd html 2.0 extended//",
+ "-//sq//dtd html 2.0 hotmetal + extensions//",
+ "-//sun microsystems corp.//dtd hotjava html//",
+ "-//sun microsystems corp.//dtd hotjava strict html//",
+ "-//w3c//dtd html 3 1995-03-24//",
+ "-//w3c//dtd html 3.2 draft//",
+ "-//w3c//dtd html 3.2 final//",
+ "-//w3c//dtd html 3.2//",
+ "-//w3c//dtd html 3.2s draft//",
+ "-//w3c//dtd html 4.0 frameset//",
+ "-//w3c//dtd html 4.0 transitional//",
+ "-//w3c//dtd html experimental 19960712//",
+ "-//w3c//dtd html experimental 970421//",
+ "-//w3c//dtd w3 html//",
+ "-//w3o//dtd w3 html 3.0//",
+ "-//webtechs//dtd mozilla html 2.0//",
+ "-//webtechs//dtd mozilla html//")) or
+ publicId in ("-//w3o//dtd w3 html strict 3.0//en//",
+ "-/w3c/dtd html 4.0 transitional/en",
+ "html") or
+ publicId.startswith(
+ ("-//w3c//dtd html 4.01 frameset//",
+ "-//w3c//dtd html 4.01 transitional//")) and
+ systemId is None or
+ systemId and systemId.lower() == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd"):
self.parser.compatMode = "quirks"
- self.parser.phase = self.parser.phases["beforeHtml"]
-
- def processCharacters(self, token):
- self.parser.parseError("expected-doctype-but-got-chars")
- self.anythingElse()
- return token
-
- def processStartTag(self, token):
- self.parser.parseError("expected-doctype-but-got-start-tag",
+ elif (publicId.startswith(
+ ("-//w3c//dtd xhtml 1.0 frameset//",
+ "-//w3c//dtd xhtml 1.0 transitional//")) or
+ publicId.startswith(
+ ("-//w3c//dtd html 4.01 frameset//",
+ "-//w3c//dtd html 4.01 transitional//")) and
+ systemId is not None):
+ self.parser.compatMode = "limited quirks"
+
+ self.parser.phase = self.parser.phases["beforeHtml"]
+
+ def anythingElse(self):
+ self.parser.compatMode = "quirks"
+ self.parser.phase = self.parser.phases["beforeHtml"]
+
+ def processCharacters(self, token):
+ self.parser.parseError("expected-doctype-but-got-chars")
+ self.anythingElse()
+ return token
+
+ def processStartTag(self, token):
+ self.parser.parseError("expected-doctype-but-got-start-tag",
+ {"name": token["name"]})
+ self.anythingElse()
+ return token
+
+ def processEndTag(self, token):
+ self.parser.parseError("expected-doctype-but-got-end-tag",
+ {"name": token["name"]})
+ self.anythingElse()
+ return token
+
+ def processEOF(self):
+ self.parser.parseError("expected-doctype-but-got-eof")
+ self.anythingElse()
+ return True
+
+
+class BeforeHtmlPhase(Phase):
+ __slots__ = tuple()
+
+ # helper methods
+ def insertHtmlElement(self):
+ self.tree.insertRoot(impliedTagToken("html", "StartTag"))
+ self.parser.phase = self.parser.phases["beforeHead"]
+
+ # other
+ def processEOF(self):
+ self.insertHtmlElement()
+ return True
+
+ def processComment(self, token):
+ self.tree.insertComment(token, self.tree.document)
+
+ def processSpaceCharacters(self, token):
+ pass
+
+ def processCharacters(self, token):
+ self.insertHtmlElement()
+ return token
+
+ def processStartTag(self, token):
+ if token["name"] == "html":
+ self.parser.firstStartTag = True
+ self.insertHtmlElement()
+ return token
+
+ def processEndTag(self, token):
+ if token["name"] not in ("head", "body", "html", "br"):
+ self.parser.parseError("unexpected-end-tag-before-html",
{"name": token["name"]})
- self.anythingElse()
+ else:
+ self.insertHtmlElement()
return token
- def processEndTag(self, token):
- self.parser.parseError("expected-doctype-but-got-end-tag",
- {"name": token["name"]})
- self.anythingElse()
- return token
- def processEOF(self):
- self.parser.parseError("expected-doctype-but-got-eof")
- self.anythingElse()
- return True
+class BeforeHeadPhase(Phase):
+ __slots__ = tuple()
- class BeforeHtmlPhase(Phase):
- __slots__ = tuple()
+ def processEOF(self):
+ self.startTagHead(impliedTagToken("head", "StartTag"))
+ return True
- # helper methods
- def insertHtmlElement(self):
- self.tree.insertRoot(impliedTagToken("html", "StartTag"))
- self.parser.phase = self.parser.phases["beforeHead"]
+ def processSpaceCharacters(self, token):
+ pass
- # other
- def processEOF(self):
- self.insertHtmlElement()
- return True
+ def processCharacters(self, token):
+ self.startTagHead(impliedTagToken("head", "StartTag"))
+ return token
- def processComment(self, token):
- self.tree.insertComment(token, self.tree.document)
+ def startTagHtml(self, token):
+ return self.parser.phases["inBody"].processStartTag(token)
- def processSpaceCharacters(self, token):
- pass
+ def startTagHead(self, token):
+ self.tree.insertElement(token)
+ self.tree.headPointer = self.tree.openElements[-1]
+ self.parser.phase = self.parser.phases["inHead"]
- def processCharacters(self, token):
- self.insertHtmlElement()
- return token
+ def startTagOther(self, token):
+ self.startTagHead(impliedTagToken("head", "StartTag"))
+ return token
- def processStartTag(self, token):
- if token["name"] == "html":
- self.parser.firstStartTag = True
- self.insertHtmlElement()
- return token
+ def endTagImplyHead(self, token):
+ self.startTagHead(impliedTagToken("head", "StartTag"))
+ return token
- def processEndTag(self, token):
- if token["name"] not in ("head", "body", "html", "br"):
- self.parser.parseError("unexpected-end-tag-before-html",
- {"name": token["name"]})
- else:
- self.insertHtmlElement()
- return token
+ def endTagOther(self, token):
+ self.parser.parseError("end-tag-after-implied-root",
+ {"name": token["name"]})
- class BeforeHeadPhase(Phase):
- __slots__ = tuple()
+ startTagHandler = _utils.MethodDispatcher([
+ ("html", startTagHtml),
+ ("head", startTagHead)
+ ])
+ startTagHandler.default = startTagOther
- def processEOF(self):
- self.startTagHead(impliedTagToken("head", "StartTag"))
- return True
+ endTagHandler = _utils.MethodDispatcher([
+ (("head", "body", "html", "br"), endTagImplyHead)
+ ])
+ endTagHandler.default = endTagOther
- def processSpaceCharacters(self, token):
- pass
- def processCharacters(self, token):
- self.startTagHead(impliedTagToken("head", "StartTag"))
- return token
+class InHeadPhase(Phase):
+ __slots__ = tuple()
- def startTagHtml(self, token):
- return self.parser.phases["inBody"].processStartTag(token)
+ # the real thing
+ def processEOF(self):
+ self.anythingElse()
+ return True
- def startTagHead(self, token):
- self.tree.insertElement(token)
- self.tree.headPointer = self.tree.openElements[-1]
- self.parser.phase = self.parser.phases["inHead"]
+ def processCharacters(self, token):
+ self.anythingElse()
+ return token
- def startTagOther(self, token):
- self.startTagHead(impliedTagToken("head", "StartTag"))
- return token
+ def startTagHtml(self, token):
+ return self.parser.phases["inBody"].processStartTag(token)
- def endTagImplyHead(self, token):
- self.startTagHead(impliedTagToken("head", "StartTag"))
- return token
+ def startTagHead(self, token):
+ self.parser.parseError("two-heads-are-not-better-than-one")
- def endTagOther(self, token):
- self.parser.parseError("end-tag-after-implied-root",
- {"name": token["name"]})
+ def startTagBaseLinkCommand(self, token):
+ self.tree.insertElement(token)
+ self.tree.openElements.pop()
+ token["selfClosingAcknowledged"] = True
- startTagHandler = _utils.MethodDispatcher([
- ("html", startTagHtml),
- ("head", startTagHead)
- ])
- startTagHandler.default = startTagOther
+ def startTagMeta(self, token):
+ self.tree.insertElement(token)
+ self.tree.openElements.pop()
+ token["selfClosingAcknowledged"] = True
+
+ attributes = token["data"]
+ if self.parser.tokenizer.stream.charEncoding[1] == "tentative":
+ if "charset" in attributes:
+ self.parser.tokenizer.stream.changeEncoding(attributes["charset"])
+ elif ("content" in attributes and
+ "http-equiv" in attributes and
+ attributes["http-equiv"].lower() == "content-type"):
+ # Encoding it as UTF-8 here is a hack, as really we should pass
+ # the abstract Unicode string, and just use the
+ # ContentAttrParser on that, but using UTF-8 allows all chars
+ # to be encoded and as a ASCII-superset works.
+ data = _inputstream.EncodingBytes(attributes["content"].encode("utf-8"))
+ parser = _inputstream.ContentAttrParser(data)
+ codec = parser.parse()
+ self.parser.tokenizer.stream.changeEncoding(codec)
+
+ def startTagTitle(self, token):
+ self.parser.parseRCDataRawtext(token, "RCDATA")
+
+ def startTagNoFramesStyle(self, token):
+ # Need to decide whether to implement the scripting-disabled case
+ self.parser.parseRCDataRawtext(token, "RAWTEXT")
+
+ def startTagNoscript(self, token):
+ if self.parser.scripting:
+ self.parser.parseRCDataRawtext(token, "RAWTEXT")
+ else:
+ self.tree.insertElement(token)
+ self.parser.phase = self.parser.phases["inHeadNoscript"]
- endTagHandler = _utils.MethodDispatcher([
- (("head", "body", "html", "br"), endTagImplyHead)
- ])
- endTagHandler.default = endTagOther
+ def startTagScript(self, token):
+ self.tree.insertElement(token)
+ self.parser.tokenizer.state = "scriptDataState"
+ self.parser.originalPhase = self.parser.phase
+ self.parser.phase = self.parser.phases["text"]
+
+ def startTagOther(self, token):
+ self.anythingElse()
+ return token
+
+ def endTagHead(self, token):
+ node = self.parser.tree.openElements.pop()
+ assert node.name == "head", "Expected head got %s" % node.name
+ self.parser.phase = self.parser.phases["afterHead"]
+
+ def endTagHtmlBodyBr(self, token):
+ self.anythingElse()
+ return token
+
+ def endTagOther(self, token):
+ self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
+
+ def anythingElse(self):
+ self.endTagHead(impliedTagToken("head"))
+
+ startTagHandler = _utils.MethodDispatcher([
+ ("html", startTagHtml),
+ ("title", startTagTitle),
+ (("noframes", "style"), startTagNoFramesStyle),
+ ("noscript", startTagNoscript),
+ ("script", startTagScript),
+ (("base", "basefont", "bgsound", "command", "link"),
+ startTagBaseLinkCommand),
+ ("meta", startTagMeta),
+ ("head", startTagHead)
+ ])
+ startTagHandler.default = startTagOther
+
+ endTagHandler = _utils.MethodDispatcher([
+ ("head", endTagHead),
+ (("br", "html", "body"), endTagHtmlBodyBr)
+ ])
+ endTagHandler.default = endTagOther
+
+
+class InHeadNoscriptPhase(Phase):
+ __slots__ = tuple()
+
+ def processEOF(self):
+ self.parser.parseError("eof-in-head-noscript")
+ self.anythingElse()
+ return True
+
+ def processComment(self, token):
+ return self.parser.phases["inHead"].processComment(token)
+
+ def processCharacters(self, token):
+ self.parser.parseError("char-in-head-noscript")
+ self.anythingElse()
+ return token
+
+ def processSpaceCharacters(self, token):
+ return self.parser.phases["inHead"].processSpaceCharacters(token)
+
+ def startTagHtml(self, token):
+ return self.parser.phases["inBody"].processStartTag(token)
+
+ def startTagBaseLinkCommand(self, token):
+ return self.parser.phases["inHead"].processStartTag(token)
+
+ def startTagHeadNoscript(self, token):
+ self.parser.parseError("unexpected-start-tag", {"name": token["name"]})
+
+ def startTagOther(self, token):
+ self.parser.parseError("unexpected-inhead-noscript-tag", {"name": token["name"]})
+ self.anythingElse()
+ return token
+
+ def endTagNoscript(self, token):
+ node = self.parser.tree.openElements.pop()
+ assert node.name == "noscript", "Expected noscript got %s" % node.name
+ self.parser.phase = self.parser.phases["inHead"]
+
+ def endTagBr(self, token):
+ self.parser.parseError("unexpected-inhead-noscript-tag", {"name": token["name"]})
+ self.anythingElse()
+ return token
+
+ def endTagOther(self, token):
+ self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
+
+ def anythingElse(self):
+ # Caller must raise parse error first!
+ self.endTagNoscript(impliedTagToken("noscript"))
+
+ startTagHandler = _utils.MethodDispatcher([
+ ("html", startTagHtml),
+ (("basefont", "bgsound", "link", "meta", "noframes", "style"), startTagBaseLinkCommand),
+ (("head", "noscript"), startTagHeadNoscript),
+ ])
+ startTagHandler.default = startTagOther
+
+ endTagHandler = _utils.MethodDispatcher([
+ ("noscript", endTagNoscript),
+ ("br", endTagBr),
+ ])
+ endTagHandler.default = endTagOther
+
+
+class AfterHeadPhase(Phase):
+ __slots__ = tuple()
+
+ def processEOF(self):
+ self.anythingElse()
+ return True
+
+ def processCharacters(self, token):
+ self.anythingElse()
+ return token
+
+ def startTagHtml(self, token):
+ return self.parser.phases["inBody"].processStartTag(token)
+
+ def startTagBody(self, token):
+ self.parser.framesetOK = False
+ self.tree.insertElement(token)
+ self.parser.phase = self.parser.phases["inBody"]
- class InHeadPhase(Phase):
- __slots__ = tuple()
+ def startTagFrameset(self, token):
+ self.tree.insertElement(token)
+ self.parser.phase = self.parser.phases["inFrameset"]
- # the real thing
- def processEOF(self):
- self.anythingElse()
- return True
+ def startTagFromHead(self, token):
+ self.parser.parseError("unexpected-start-tag-out-of-my-head",
+ {"name": token["name"]})
+ self.tree.openElements.append(self.tree.headPointer)
+ self.parser.phases["inHead"].processStartTag(token)
+ for node in self.tree.openElements[::-1]:
+ if node.name == "head":
+ self.tree.openElements.remove(node)
+ break
- def processCharacters(self, token):
- self.anythingElse()
- return token
+ def startTagHead(self, token):
+ self.parser.parseError("unexpected-start-tag", {"name": token["name"]})
+
+ def startTagOther(self, token):
+ self.anythingElse()
+ return token
+
+ def endTagHtmlBodyBr(self, token):
+ self.anythingElse()
+ return token
+
+ def endTagOther(self, token):
+ self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
+
+ def anythingElse(self):
+ self.tree.insertElement(impliedTagToken("body", "StartTag"))
+ self.parser.phase = self.parser.phases["inBody"]
+ self.parser.framesetOK = True
+
+ startTagHandler = _utils.MethodDispatcher([
+ ("html", startTagHtml),
+ ("body", startTagBody),
+ ("frameset", startTagFrameset),
+ (("base", "basefont", "bgsound", "link", "meta", "noframes", "script",
+ "style", "title"),
+ startTagFromHead),
+ ("head", startTagHead)
+ ])
+ startTagHandler.default = startTagOther
+ endTagHandler = _utils.MethodDispatcher([(("body", "html", "br"),
+ endTagHtmlBodyBr)])
+ endTagHandler.default = endTagOther
+
+
+class InBodyPhase(Phase):
+ # http://www.whatwg.org/specs/web-apps/current-work/#parsing-main-inbody
+ # the really-really-really-very crazy mode
+ __slots__ = ("processSpaceCharacters",)
+
+ def __init__(self, *args, **kwargs):
+ super(InBodyPhase, self).__init__(*args, **kwargs)
+ # Set this to the default handler
+ self.processSpaceCharacters = self.processSpaceCharactersNonPre
+
+ def isMatchingFormattingElement(self, node1, node2):
+ return (node1.name == node2.name and
+ node1.namespace == node2.namespace and
+ node1.attributes == node2.attributes)
+
+ # helper
+ def addFormattingElement(self, token):
+ self.tree.insertElement(token)
+ element = self.tree.openElements[-1]
- def startTagHtml(self, token):
- return self.parser.phases["inBody"].processStartTag(token)
+ matchingElements = []
+ for node in self.tree.activeFormattingElements[::-1]:
+ if node is Marker:
+ break
+ elif self.isMatchingFormattingElement(node, element):
+ matchingElements.append(node)
+
+ assert len(matchingElements) <= 3
+ if len(matchingElements) == 3:
+ self.tree.activeFormattingElements.remove(matchingElements[-1])
+ self.tree.activeFormattingElements.append(element)
+
+ # the real deal
+ def processEOF(self):
+ allowed_elements = frozenset(("dd", "dt", "li", "p", "tbody", "td",
+ "tfoot", "th", "thead", "tr", "body",
+ "html"))
+ for node in self.tree.openElements[::-1]:
+ if node.name not in allowed_elements:
+ self.parser.parseError("expected-closing-tag-but-got-eof")
+ break
+ # Stop parsing
+
+ def processSpaceCharactersDropNewline(self, token):
+ # Sometimes (start of , , and