Skip to content

Tokenizer: use Python objects to represent tokens #521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 19 commits into from
Closed

Tokenizer: use Python objects to represent tokens #521

Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
183d8a0
Consistency: consume a single character at a time during attribute na…
jayaddison Dec 29, 2020
2e86373
Refactor: pretranslate lowercase element and attribute names
jayaddison Dec 29, 2020
8f96b17
Restore self.currentToken safety check
jayaddison Dec 29, 2020
a912842
Alternate approach: do not pretranslate temporary buffered data
jayaddison Dec 30, 2020
f9f370e
Consistency: character consumption within double-escaped state
jayaddison Dec 30, 2020
bcee8bd
Refactor: use Python objects for tokens within tokenizer
jayaddison Dec 29, 2020
67262f8
Introduce type hierarchy for tag-related tokens
jayaddison Dec 29, 2020
900bdaf
Simplify tag token construction
jayaddison Dec 29, 2020
1f6cae9
Refactor token attribution name/value accumulation
jayaddison Dec 29, 2020
695ac1c
Cleanup: remove leavingThisState / emitToken logic
jayaddison Dec 29, 2020
b1a444b
Remove EmptyTag tokenizer token class
jayaddison Dec 29, 2020
bb7fabc
Refactor: pre-translate strings that are only used in lowercase context
jayaddison Dec 29, 2020
5f4ace9
Cleanup: remove getattr anti-pattern
jayaddison Dec 29, 2020
d744c86
Consistency: use camel-casing to correspond with existing codebase style
jayaddison Dec 29, 2020
1d62e69
Consistency: consume a single character at a time during attribute na…
jayaddison Dec 29, 2020
8772408
Merge branch 'tokenizer/pretranslate-lowercase-names' into tokenizer/…
jayaddison Dec 30, 2020
192cce0
Linting cleanup
jayaddison Dec 30, 2020
e76e0dd
Clarify method name: clearAttribute -> flushAttribute
jayaddison Jan 4, 2021
da37332
Merge branch 'master' into tokenizer/object-tokens
jayaddison Sep 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Refactor: pretranslate lowercase element and attribute names
  • Loading branch information
@jayaddison
jayaddison committed Dec 29, 2020
commit 2e8637367ad001710ac5fb7b41191ace3f43f458
62 changes: 30 additions & 32 deletions html5lib/_tokenizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,6 @@ def emitCurrentToken(self):
token = self.currentToken
# Add token to the queue to be yielded
if (token["type"] in tagTokenTypes):
token["name"] = token["name"].translate(asciiUpper2Lower)
if token["type"] == tokenTypes["StartTag"]:
raw = token["data"]
data = attributeMap(raw)
Expand Down Expand Up @@ -380,7 +379,8 @@ def tagOpenState(self):
self.state = self.closeTagOpenState
elif data in asciiLetters:
self.currentToken = {"type": tokenTypes["StartTag"],
"name": data, "data": [],
"name": data.translate(asciiUpper2Lower),
"data": [],
"selfClosing": False,
"selfClosingAcknowledged": False}
self.state = self.tagNameState
Expand Down Expand Up @@ -410,7 +410,8 @@ def tagOpenState(self):
def closeTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.currentToken = {"type": tokenTypes["EndTag"], "name": data,
self.currentToken = {"type": tokenTypes["EndTag"],
"name": data.translate(asciiUpper2Lower),
"data": [], "selfClosing": False}
self.state = self.tagNameState
elif data == ">":
Expand Down Expand Up @@ -448,7 +449,7 @@ def tagNameState(self):
"data": "invalid-codepoint"})
self.currentToken["name"] += "\uFFFD"
else:
self.currentToken["name"] += data
self.currentToken["name"] += data.translate(asciiUpper2Lower)
# (Don't use charsUntil here, because tag names are
# very short and it's faster to not do anything fancy)
return True
Expand All @@ -467,7 +468,7 @@ def rcdataLessThanSignState(self):
def rcdataEndTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
self.state = self.rcdataEndTagNameState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"})
Expand All @@ -476,7 +477,7 @@ def rcdataEndTagOpenState(self):
return True

def rcdataEndTagNameState(self):
appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower()
appropriate = self.currentToken["name"] == self.temporaryBuffer
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
Expand All @@ -495,7 +496,7 @@ def rcdataEndTagNameState(self):
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "</" + self.temporaryBuffer})
Expand All @@ -517,7 +518,7 @@ def rawtextLessThanSignState(self):
def rawtextEndTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
self.state = self.rawtextEndTagNameState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"})
Expand All @@ -526,7 +527,7 @@ def rawtextEndTagOpenState(self):
return True

def rawtextEndTagNameState(self):
appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower()
appropriate = self.currentToken["name"] == self.temporaryBuffer
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
Expand All @@ -545,7 +546,7 @@ def rawtextEndTagNameState(self):
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "</" + self.temporaryBuffer})
Expand All @@ -570,7 +571,7 @@ def scriptDataLessThanSignState(self):
def scriptDataEndTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
self.state = self.scriptDataEndTagNameState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"})
Expand All @@ -579,7 +580,7 @@ def scriptDataEndTagOpenState(self):
return True

def scriptDataEndTagNameState(self):
appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower()
appropriate = self.currentToken["name"] == self.temporaryBuffer
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
Expand All @@ -598,7 +599,7 @@ def scriptDataEndTagNameState(self):
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "</" + self.temporaryBuffer})
Expand Down Expand Up @@ -695,7 +696,7 @@ def scriptDataEscapedLessThanSignState(self):
self.state = self.scriptDataEscapedEndTagOpenState
elif data in asciiLetters:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<" + data})
self.temporaryBuffer = data
self.temporaryBuffer = data.translate(asciiUpper2Lower)
self.state = self.scriptDataDoubleEscapeStartState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"})
Expand All @@ -706,7 +707,7 @@ def scriptDataEscapedLessThanSignState(self):
def scriptDataEscapedEndTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.temporaryBuffer = data
self.temporaryBuffer = data.translate(asciiUpper2Lower)
self.state = self.scriptDataEscapedEndTagNameState
else:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"})
Expand All @@ -715,7 +716,7 @@ def scriptDataEscapedEndTagOpenState(self):
return True

def scriptDataEscapedEndTagNameState(self):
appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower()
appropriate = self.currentToken["name"] == self.temporaryBuffer
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = {"type": tokenTypes["EndTag"],
Expand All @@ -734,7 +735,7 @@ def scriptDataEscapedEndTagNameState(self):
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
else:
self.tokenQueue.append({"type": tokenTypes["Characters"],
"data": "</" + self.temporaryBuffer})
Expand All @@ -746,13 +747,13 @@ def scriptDataDoubleEscapeStartState(self):
data = self.stream.char()
if data in (spaceCharacters | frozenset(("/", ">"))):
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
if self.temporaryBuffer.lower() == "script":
if self.temporaryBuffer == "script":
self.state = self.scriptDataDoubleEscapedState
else:
self.state = self.scriptDataEscapedState
elif data in asciiLetters:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
else:
self.stream.unget(data)
self.state = self.scriptDataEscapedState
Expand Down Expand Up @@ -842,13 +843,13 @@ def scriptDataDoubleEscapeEndState(self):
data = self.stream.char()
if data in (spaceCharacters | frozenset(("/", ">"))):
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
if self.temporaryBuffer.lower() == "script":
if self.temporaryBuffer == "script":
self.state = self.scriptDataEscapedState
else:
self.state = self.scriptDataDoubleEscapedState
elif data in asciiLetters:
self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data})
self.temporaryBuffer += data
self.temporaryBuffer += data.translate(asciiUpper2Lower)
else:
self.stream.unget(data)
self.state = self.scriptDataDoubleEscapedState
Expand All @@ -859,7 +860,8 @@ def beforeAttributeNameState(self):
if data in spaceCharacters:
self.stream.charsUntil(spaceCharacters, True)
elif data in asciiLetters:
self.currentToken["data"].append([data, ""])
attr_name = data.translate(asciiUpper2Lower)
self.currentToken["data"].append([attr_name, ""])
self.state = self.attributeNameState
elif data == ">":
self.emitCurrentToken()
Expand Down Expand Up @@ -891,7 +893,7 @@ def attributeNameState(self):
if data == "=":
self.state = self.beforeAttributeValueState
elif data in asciiLetters:
self.currentToken["data"][-1][0] += data
self.currentToken["data"][-1][0] += data.translate(asciiUpper2Lower)
leavingThisState = False
elif data == ">":
# XXX If we emit here the attributes are converted to a dict
Expand All @@ -918,15 +920,13 @@ def attributeNameState(self):
"data": "eof-in-attribute-name"})
self.state = self.dataState
else:
self.currentToken["data"][-1][0] += data
self.currentToken["data"][-1][0] += data.translate(asciiUpper2Lower)
leavingThisState = False

if leavingThisState:
# Attributes are not dropped at this stage. That happens when the
# start tag token is emitted so values can still be safely appended
# to attributes, but we do want to report the parse error in time.
self.currentToken["data"][-1][0] = (
self.currentToken["data"][-1][0].translate(asciiUpper2Lower))
for name, _ in self.currentToken["data"][:-1]:
if self.currentToken["data"][-1][0] == name:
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
Expand All @@ -946,7 +946,8 @@ def afterAttributeNameState(self):
elif data == ">":
self.emitCurrentToken()
elif data in asciiLetters:
self.currentToken["data"].append([data, ""])
attr_name = data.translate(asciiUpper2Lower)
self.currentToken["data"].append([attr_name, ""])
self.state = self.attributeNameState
elif data == "/":
self.state = self.selfClosingStartTagState
Expand Down Expand Up @@ -1340,17 +1341,15 @@ def beforeDoctypeNameState(self):
self.tokenQueue.append(self.currentToken)
self.state = self.dataState
else:
self.currentToken["name"] = data
self.currentToken["name"] = data.translate(asciiUpper2Lower)
self.state = self.doctypeNameState
return True

def doctypeNameState(self):
data = self.stream.char()
if data in spaceCharacters:
self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
self.state = self.afterDoctypeNameState
elif data == ">":
self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
self.tokenQueue.append(self.currentToken)
self.state = self.dataState
elif data == "\u0000":
Expand All @@ -1362,11 +1361,10 @@ def doctypeNameState(self):
self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
"eof-in-doctype-name"})
self.currentToken["correct"] = False
self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
self.tokenQueue.append(self.currentToken)
self.state = self.dataState
else:
self.currentToken["name"] += data
self.currentToken["name"] += data.translate(asciiUpper2Lower)
return True

def afterDoctypeNameState(self):
Expand Down