Skip to content

Tokenizer: use Python objects to represent tokens #521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 19 commits into from
Closed

Tokenizer: use Python objects to represent tokens #521

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
183d8a0
Consistency: consume a single character at a time during attribute na…
jayaddison Dec 29, 2020
2e86373
Refactor: pretranslate lowercase element and attribute names
jayaddison Dec 29, 2020
8f96b17
Restore self.currentToken safety check
jayaddison Dec 29, 2020
a912842
Alternate approach: do not pretranslate temporary buffered data
jayaddison Dec 30, 2020
f9f370e
Consistency: character consumption within double-escaped state
jayaddison Dec 30, 2020
bcee8bd
Refactor: use Python objects for tokens within tokenizer
jayaddison Dec 29, 2020
67262f8
Introduce type hierarchy for tag-related tokens
jayaddison Dec 29, 2020
900bdaf
Simplify tag token construction
jayaddison Dec 29, 2020
1f6cae9
Refactor token attribution name/value accumulation
jayaddison Dec 29, 2020
695ac1c
Cleanup: remove leavingThisState / emitToken logic
jayaddison Dec 29, 2020
b1a444b
Remove EmptyTag tokenizer token class
jayaddison Dec 29, 2020
bb7fabc
Refactor: pre-translate strings that are only used in lowercase context
jayaddison Dec 29, 2020
5f4ace9
Cleanup: remove getattr anti-pattern
jayaddison Dec 29, 2020
d744c86
Consistency: use camel-casing to correspond with existing codebase style
jayaddison Dec 29, 2020
1d62e69
Consistency: consume a single character at a time during attribute na…
jayaddison Dec 29, 2020
8772408
Merge branch 'tokenizer/pretranslate-lowercase-names' into tokenizer/…
jayaddison Dec 30, 2020
192cce0
Linting cleanup
jayaddison Dec 30, 2020
e76e0dd
Clarify method name: clearAttribute -> flushAttribute
jayaddison Jan 4, 2021
da37332
Merge branch 'master' into tokenizer/object-tokens
jayaddison Sep 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Simplify tag token construction
  • Loading branch information
@jayaddison
jayaddison committed Dec 30, 2020
commit 900bdafe1bae95aa32834cf45253f99da541b831
50 changes: 26 additions & 24 deletions html5lib/_tokenizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,23 +43,24 @@ class SpaceCharacters(Token):


class Tag(Token):
def __init__(self, name, data, self_closing):
def __init__(self, name, data):
self.name = name
self.data = data
self.self_closing = self_closing
self.data = data or []
self.self_closing = False

class StartTag(Tag):
def __init__(self, name, data, self_closing, self_closing_acknowledged=False):
super(StartTag, self).__init__(name, data, self_closing)
self.self_closing_acknowledged = self_closing_acknowledged
def __init__(self, name, data=None):
super(StartTag, self).__init__(name, data)
self.self_closing_acknowledged = False

class EndTag(Tag):
def __init__(self, name, data, self_closing):
super(EndTag, self).__init__(name, data, self_closing)
def __init__(self, name, data=None):
super(EndTag, self).__init__(name, data)

class EmptyTag(Tag):
def __init__(self, name, data):
super(EmptyTag, self).__init__(name, data, self_closing)
def __init__(self, name, data=None):
super(EmptyTag, self).__init__(name, data)
self.self_closing = True

class Comment(Token):
pass
Expand Down Expand Up @@ -274,6 +275,7 @@ def emitCurrentToken(self):
data = attributeMap(raw)
if len(raw) > len(data):
# we had some duplicated attribute, fix so first wins
was = dict(data)
data.update(raw[::-1])
token.data = data

Expand Down Expand Up @@ -395,7 +397,7 @@ def tagOpenState(self):
elif data == "/":
self.state = self.closeTagOpenState
elif data in asciiLetters:
self.currentToken = StartTag(name=data, data=[], self_closing=False)
self.currentToken = StartTag(name=data)
self.state = self.tagNameState
elif data == ">":
# XXX In theory it could be something besides a tag name. But
Expand All @@ -420,7 +422,7 @@ def tagOpenState(self):
def closeTagOpenState(self):
data = self.stream.char()
if data in asciiLetters:
self.currentToken = EndTag(name=data, data=[], self_closing=False)
self.currentToken = EndTag(name=data)
self.state = self.tagNameState
elif data == ">":
self.tokenQueue.append(ParseError("expected-closing-tag-but-got-right-bracket"))
Expand Down Expand Up @@ -482,13 +484,13 @@ def rcdataEndTagNameState(self):
appropriate = self.currentToken and self.currentToken.name.lower() == self.temporaryBuffer.lower()
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.beforeAttributeNameState
elif data == "/" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.selfClosingStartTagState
elif data == ">" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
Expand Down Expand Up @@ -525,13 +527,13 @@ def rawtextEndTagNameState(self):
appropriate = self.currentToken and self.currentToken.name.lower() == self.temporaryBuffer.lower()
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.beforeAttributeNameState
elif data == "/" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.selfClosingStartTagState
elif data == ">" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
Expand Down Expand Up @@ -571,13 +573,13 @@ def scriptDataEndTagNameState(self):
appropriate = self.currentToken and self.currentToken.name.lower() == self.temporaryBuffer.lower()
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.beforeAttributeNameState
elif data == "/" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.selfClosingStartTagState
elif data == ">" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
Expand Down Expand Up @@ -693,13 +695,13 @@ def scriptDataEscapedEndTagNameState(self):
appropriate = self.currentToken and self.currentToken.name.lower() == self.temporaryBuffer.lower()
data = self.stream.char()
if data in spaceCharacters and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.beforeAttributeNameState
elif data == "/" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.state = self.selfClosingStartTagState
elif data == ">" and appropriate:
self.currentToken = EndTag(name=self.temporaryBuffer, data=[], self_closing=False)
self.currentToken = EndTag(name=self.temporaryBuffer)
self.emitCurrentToken()
self.state = self.dataState
elif data in asciiLetters:
Expand Down
2 changes: 1 addition & 1 deletion html5lib/html5parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2783,7 +2783,7 @@ def impliedTagToken(name, type=EndTag, attributes=None,
selfClosing=False):
if attributes is None:
attributes = {}
return type(name=name, data=attributes, self_closing=selfClosing)
return type(name=name, data=attributes)


class ParseError(Exception):
Expand Down