diff --git a/AUTHORS.rst b/AUTHORS.rst index 90401390..1145dbb0 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -63,4 +63,4 @@ Patches and suggestions - Ville Skyttä - Hugo van Kemenade - Mark Vasilkov - +- Muneeb Sheikh diff --git a/CHANGES.rst b/CHANGES.rst index 47dcda3a..88fea47b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,9 @@ Features: Bug fixes: +* Detect declared encodings after short comments whose terminators overlap + the comment opener. (#427) + * The sanitizer now permits ```` tags. It used to allow ``
`` already. (#423) diff --git a/html5lib/_inputstream.py b/html5lib/_inputstream.py index a93b5a4e..943594c5 100644 --- a/html5lib/_inputstream.py +++ b/html5lib/_inputstream.py @@ -712,7 +712,8 @@ def getEncoding(self): return self.encoding def handleComment(self): - """Skip over comments""" + """Skip over comments, including terminators overlapping the opener.""" + self.data.position -= 2 return self.data.jumpTo(b"-->") def handleMeta(self): diff --git a/html5lib/tests/test_encoding.py b/html5lib/tests/test_encoding.py index 47c4814a..a1bf0bb6 100644 --- a/html5lib/tests/test_encoding.py +++ b/html5lib/tests/test_encoding.py @@ -115,3 +115,19 @@ def test_chardet(): encoding = _inputstream.HTMLInputStream(fp.read()).charEncoding assert encoding[0].name == "big5" # pylint:enable=wrong-import-position + + +@pytest.mark.parametrize("comment", [b"", b"", b"", b""]) +def test_prescan_short_comments(comment): + data = comment + b"" + encoding = _inputstream.EncodingParser(data).getEncoding() + assert encoding is not None + assert encoding.name == "iso-8859-2" + stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False) + assert stream.charEncoding[0].name == "iso-8859-2" + + +def test_prescan_ignores_meta_inside_comment(): + data = b"" + encoding = _inputstream.EncodingParser(data).getEncoding() + assert encoding.name == "iso-8859-2"