From b1454596dda0561610f0d5438962caf501b4cd97 Mon Sep 17 00:00:00 2001 From: Keerthana KT Date: Wed, 14 Oct 2026 20:14:58 +0530 Subject: [PATCH 1/2] fix(commit): parse the `encoding` header when deserializing `Commit._deserialize` compared a 10-byte slice of each stripped header line against the 9-byte literal `b"encoding "`, so the comparison never matched and the branch reading a commit's declared encoding was unreachable. Every commit parsed from the object database was decoded with the default UTF-8: `Commit.encoding` reported `UTF-8` regardless of the header, and the author, committer, and message of a commit written under `i18n.commitencoding` came back with U+FFFD replacement characters while `git log` shows them correctly. Re-serializing such a commit also dropped its `encoding` header and stored the garbled text. Compare the right slice length so the header is read. Adopt the declared encoding only when Python has a text codec for it: the identity lines are decoded with `errors="replace"`, which does not guard against an unknown codec name, so a commit declaring one would otherwise raise `LookupError` on attribute access. An unknown name keeps the previous behavior (the default encoding) and logs the error. Validated against a repository committed under `i18n.commitencoding = ISO-8859-1`: `commit.encoding`, `commit.author.name`, and `commit.message` now match what `git log` shows. Adds regression tests in `test/test_commit.py` that fail before the change. --- git/objects/commit.py | 13 +++++++++++-- test/test_commit.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 690a296a5..0c034d231 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -896,8 +896,17 @@ def _deserialize(self, stream: BytesIO) -> "Commit": enc = next_line buf = enc.strip() while buf: - if buf[0:10] == b"encoding ": - self.encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") + if buf[0:9] == b"encoding ": + encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") + try: + # Probe one byte: decoding b"" skips the codec lookup entirely. + b" ".decode(encoding, "replace") + except (LookupError, ValueError): + # The encoding is not a text encoding Python knows; keep the + # default so the identity and message lines still decode below. + _logger.error("Ignoring unknown commit encoding %r", encoding) + else: + self.encoding = encoding elif buf[0:7] == b"gpgsig ": sig_lines = [buf[buf.find(b" ") + 1 :] + b"\n"] is_next_header = False diff --git a/test/test_commit.py b/test/test_commit.py index 379bc831b..da8b49325 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -458,6 +458,41 @@ def test_invalid_commit(self): self.assertEqual(cmt.author.name, "E.Azer Ko�o�o�oculu", cmt.author.name) self.assertEqual(cmt.author.email, "azer@kodfabrik.com", cmt.author.email) + def test_deserialization_declared_encoding(self): + # The "encoding" header names the encoding of the identity and message bytes. + data = ( + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" + b"author Andr\xe9 1112591163 +0200\n" + b"committer Andr\xe9 1112591163 +0200\n" + b"encoding ISO-8859-1\n" + b"\n" + b"R\xe9sum\xe9\n" + ) + cmt = self.rorepo.commit() + cmt._deserialize(BytesIO(data)) + + self.assertEqual(cmt.encoding, "ISO-8859-1") + self.assertEqual(cmt.author.name, "André") + self.assertEqual(cmt.committer.name, "André") + self.assertEqual(cmt.message, "Résumé\n") + + def test_deserialization_unknown_declared_encoding(self): + # An encoding Python has no text codec for falls back to the default encoding. + data = ( + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" + b"author Andr\xe9 1112591163 +0200\n" + b"committer Andr\xe9 1112591163 +0200\n" + b"encoding pony-speak\n" + b"\n" + b"R\xe9sum\xe9\n" + ) + cmt = self.rorepo.commit() + cmt._deserialize(BytesIO(data)) + + self.assertEqual(cmt.encoding, cmt.default_encoding) + self.assertEqual(cmt.author.name, "Andr�") + self.assertEqual(cmt.message, "R�sum�\n") + @with_rw_directory def test_identity_cannot_alter_headers(self, rw_dir): """A name or email must not add header lines or present another identity.""" From 2c9b9a9e0703dc1ca878f4313df83d9418e108ed Mon Sep 17 00:00:00 2001 From: Byron Date: Thu, 15 Oct 2026 05:01:58 +0200 Subject: [PATCH 2/2] review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I didn't try to fix all the issues with this, let's just make it a little bit better hoping the code won't be around much longer. # Known limitations Commit.encoding still conflates the declared encoding with the codec used for best-effort decoding. Unsupported or malformed declarations leave the default encoding—or an earlier accepted declaration—in place, so the resulting attribute does not faithfully represent the commit’s original metadata. The rejected header bytes appear in the diagnostic but are not retained on the object. Fallback decoding with "replace" can irreversibly alter identity and message content. Reserializing such a commit can therefore change both its encoding declaration and its content; this change does not provide lossless round-tripping. Codec validation also checks only that Python accepts the codec, not that it can decode the actual commit content. Assisted-by: GPT 6.1 Sol Co-authored-by: GPT 6.1 Sol --- git/objects/commit.py | 10 ++++++---- test/test_commit.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 0c034d231..3239079c1 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -897,14 +897,16 @@ def _deserialize(self, stream: BytesIO) -> "Commit": buf = enc.strip() while buf: if buf[0:9] == b"encoding ": - encoding = buf[buf.find(b" ") + 1 :].decode(self.encoding, "ignore") + raw_encoding = buf[9:] try: + # Codec names are ASCII, independent of the content encoding. + encoding = raw_encoding.decode("ascii") # Probe one byte: decoding b"" skips the codec lookup entirely. b" ".decode(encoding, "replace") except (LookupError, ValueError): - # The encoding is not a text encoding Python knows; keep the - # default so the identity and message lines still decode below. - _logger.error("Ignoring unknown commit encoding %r", encoding) + # Reject malformed names and non-text codecs without changing + # the current encoding (the default unless a valid header was read). + _logger.error("Ignoring invalid commit encoding %r", raw_encoding) else: self.encoding = encoding elif buf[0:7] == b"gpgsig ": diff --git a/test/test_commit.py b/test/test_commit.py index da8b49325..a22daa87a 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -493,6 +493,26 @@ def test_deserialization_unknown_declared_encoding(self): self.assertEqual(cmt.author.name, "Andr�") self.assertEqual(cmt.message, "R�sum�\n") + def test_deserialization_multiple_encoding_headers(self): + for headers in ( + b"encoding utf-16-be\nencoding ISO-8859-1\n", + b"encoding ISO-8859-1\nencoding utf-\xff8\n", + ): + with self.subTest(headers=headers): + data = ( + b"tree 3385f7b1e5f91930a3bd3e07105e113fca29981d\n" + b"author Andr\xe9 1112591163 +0200\n" + b"committer Andr\xe9 1112591163 +0200\n" + headers + b"\n" + b"R\xe9sum\xe9\n" + ) + cmt = self.rorepo.commit() + cmt._deserialize(BytesIO(data)) + + self.assertEqual(cmt.encoding, "ISO-8859-1") + self.assertEqual(cmt.author.name, "André") + self.assertEqual(cmt.committer.name, "André") + self.assertEqual(cmt.message, "Résumé\n") + @with_rw_directory def test_identity_cannot_alter_headers(self, rw_dir): """A name or email must not add header lines or present another identity."""