From b437386b7c1332dd842ac7739fdce305e768590b Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Sat, 25 Jul 2026 16:57:20 +0000 Subject: [PATCH 25/25] toml: decode \u/\U escapes to real UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream PR #415 consumes unicode escapes but stores a `?` placeholder. Since minicargo forwards manifest strings into the CARGO_PKG_DESCRIPTION / CARGO_PKG_AUTHORS environment variables, an escaped description would reach env!() with `?` mojibake. Decode the codepoint and re-encode as UTF-8 (restored from the pre-rebase powerpc toml commit, which upstream PR #415 otherwise supersedes). Verified: `café \U0001F600` reaches CARGO_PKG_DESCRIPTION as proper UTF-8 bytes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GNXc7PddzJE4X1swqbe3Xn --- tools/common/toml.cpp | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/tools/common/toml.cpp b/tools/common/toml.cpp index 5a331152..f35ca724 100644 --- a/tools/common/toml.cpp +++ b/tools/common/toml.cpp @@ -11,6 +11,7 @@ #include #include #include +#include /// Representation of a syntatic token in a TOML file struct TomlToken @@ -395,10 +396,37 @@ namespace { case 'r': str += '\r'; break; case 'b': str += '\b'; break; case 'f': str += '\f'; break; - // `\uXXXX` / `\UXXXXXXXX`: consume the hex digits. minicargo never - // needs the exact codepoint of a string value, so store a placeholder. - case 'u': for(int i = 0; i < 4; i ++) (void)is.get(); str += '?'; break; - case 'U': for(int i = 0; i < 8; i ++) (void)is.get(); str += '?'; break; + // `\uXXXX` / `\UXXXXXXXX`: decode and re-encode as UTF-8, so escaped + // text in e.g. `description` survives into CARGO_PKG_DESCRIPTION intact + case 'u': case 'U': { + unsigned n_digits = (c == 'u' ? 4 : 8); + uint32_t v = 0; + for(unsigned i = 0; i < n_digits; i++) { + int h = is.get(); + if('0' <= h && h <= '9') v = v*16 + (h - '0'); + else if('a' <= h && h <= 'f') v = v*16 + (h - 'a' + 10); + else if('A' <= h && h <= 'F') v = v*16 + (h - 'A' + 10); + else throw ::std::runtime_error("toml.cpp handle_escape: Invalid hex digit in unicode escape"); + } + if( v < 0x80 ) { + str += static_cast(v); + } + else if( v < 0x800 ) { + str += static_cast(0xC0 | (v >> 6)); + str += static_cast(0x80 | (v & 0x3F)); + } + else if( v < 0x10000 ) { + str += static_cast(0xE0 | (v >> 12)); + str += static_cast(0x80 | ((v >> 6) & 0x3F)); + str += static_cast(0x80 | (v & 0x3F)); + } + else { + str += static_cast(0xF0 | (v >> 18)); + str += static_cast(0x80 | ((v >> 12) & 0x3F)); + str += static_cast(0x80 | ((v >> 6) & 0x3F)); + str += static_cast(0x80 | (v & 0x3F)); + } + break; } // Line-ending backslash in a multi-line basic string: trim the newline // and all following whitespace up to the next non-whitespace char. case '\n': case '\r': case ' ': case '\t': { -- 2.43.0