From f0ebaabd8efda2a7d8d56f17e89289e0f9fb3cef Mon Sep 17 00:00:00 2001 From: hiros0921 Date: Sat, 22 Aug 2026 19:12:38 +0900 Subject: [PATCH] fix(detect): treat C0 control characters as valid UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `isUTF8()` only accepted TAB, LF, CR and printable ASCII, so a UTF-8 string containing any other C0 control character was not detected as UTF-8. This is inconsistent with `isASCII()`, which accepts every byte in the ASCII range except ESC (0x1B). The same byte was therefore reported as ASCII but not as UTF-8, even though every ASCII byte is valid UTF-8. const utf8 = Buffer.from('UTF8の文字列です', 'utf8'); Encoding.detect(Buffer.from([0x08])); // 'ASCII' Encoding.detect(utf8); // 'UTF8' Encoding.detect(Buffer.concat([utf8, bs])); // 'UNICODE' <- expected 'UTF8' Accept the whole ASCII range except ESC, which is left out to match `isASCII()` and to keep ISO-2022-JP out of `isUTF8()`. This does not widen the result of `detect()`, because `isBINARY` (0x00-0x07, 0xFF) and `isJIS` are both tried before UTF8 in `Encoding.orders`. Only the previously misdetected inputs change: +BS (0x08) UNICODE -> UTF8 +VT (0x0B) UNICODE -> UTF8 +FF (0x0C) UNICODE -> UTF8 +DEL (0x7F) UNICODE -> UTF8 +NUL (0x00) BINARY -> BINARY (unchanged) ISO-2022-JP JIS -> JIS (unchanged) Fixes #49 --- src/encoding-detect.js | 8 ++++++-- tests/test.js | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/encoding-detect.js b/src/encoding-detect.js index 319ffbc..292aff0 100644 --- a/src/encoding-detect.js +++ b/src/encoding-detect.js @@ -201,8 +201,12 @@ function isUTF8(data) { return false; } - if (b === 0x09 || b === 0x0A || b === 0x0D || - (b >= 0x20 && b <= 0x7E)) { + // Every byte in the ASCII range is valid UTF-8, including the C0 control + // characters. Only ESC (0x1B) is excluded here so that ISO-2022-JP data is + // not reported as UTF-8 by `isUTF8()`. + // In `detect()` this does not widen the result: isBINARY (0x00-0x07, 0xFF) + // and isJIS are both tried before UTF8. + if (b <= 0x7F && b !== 0x1B) { continue; } diff --git a/tests/test.js b/tests/test.js index 84a9897..89aa5ed 100644 --- a/tests/test.js +++ b/tests/test.js @@ -74,6 +74,27 @@ describe('encoding', function() { }); }); + it('UTF-8 with C0 control characters', function() { + // Every byte in the ASCII range is valid UTF-8, so a UTF-8 string that + // contains a control character must still be detected as UTF-8. + var utf8 = encoding.stringToCode('UTF8\u306E\u6587\u5B57\u5217\u3067\u3059'); + utf8 = encoding.convert(utf8, { to: 'UTF8', from: 'UNICODE' }); + + // 0x08 BS, 0x0B VT, 0x0C FF, 0x7F DEL + [0x08, 0x0B, 0x0C, 0x7F].forEach(function(control) { + assert.equal(encoding.detect(utf8.concat([control])), 'UTF8'); + assert.equal(encoding.detect([control].concat(utf8)), 'UTF8'); + }); + + // 0x00-0x07 are still reported as BINARY, which is checked before UTF8 + assert.equal(encoding.detect(utf8.concat([0x00])), 'BINARY'); + + // 0x1B is reserved for ISO-2022-JP detection, as in isASCII() + var jis = encoding.convert(utf8, { to: 'JIS', from: 'UTF8' }); + assert.equal(encoding.detect(jis), 'JIS'); + assert.equal(encoding.detect(jis, 'UTF8'), false); + }); + it('UTF-16, UTF-16BE', function() { var utf16 = [ 0xFE,0xFF,0x30,0x53,0x30,0x6E,0x30,0xC6,0x30,0xAD,0x30,0xB9,0x30,