diff --git a/src/encoding-convert.js b/src/encoding-convert.js index 63e9ad8..a7722d0 100644 --- a/src/encoding-convert.js +++ b/src/encoding-convert.js @@ -2,6 +2,7 @@ var config = require('./config'); var util = require('./util'); var EncodingDetect = require('./encoding-detect'); var EncodingTable = require('./encoding-table'); +var sjisExt = require('./sjis-ext'); /** * JIS to SJIS @@ -134,7 +135,7 @@ function SJISToJIS(data) { var index = 0; var len = data && data.length; var i = 0; - var b1, b2; + var b1, b2, remapped; var esc = [ 0x1B, 0x28, 0x42, @@ -160,8 +161,15 @@ function SJISToJIS(data) { results[results.length] = esc[5]; } - b1 <<= 1; b2 = data[++i]; + if (sjisExt.hasCP932DuplicateCode(b1)) { + // Remap CP932 duplicate codes and IBM extended characters + remapped = sjisExt.remapCP932DuplicateCode(b1, b2); + b1 = remapped >> 8; + b2 = remapped & 0xFF; + } + + b1 <<= 1; if (b2 < 0x9F) { if (b1 < 0x13F) { b1 -= 0xE1; @@ -211,7 +219,7 @@ function SJISToEUCJP(data) { var results = []; var len = data && data.length; var i = 0; - var b1, b2; + var b1, b2, remapped; for (; i < len; i++) { b1 = data[i]; @@ -220,6 +228,13 @@ function SJISToEUCJP(data) { results[results.length] = b1; } else if (b1 >= 0x81) { b2 = data[++i]; + if (sjisExt.hasCP932DuplicateCode(b1)) { + // Remap CP932 duplicate codes and IBM extended characters + remapped = sjisExt.remapCP932DuplicateCode(b1, b2); + b1 = remapped >> 8; + b2 = remapped & 0xFF; + } + b1 <<= 1; if (b2 < 0x9F) { if (b1 < 0x13F) { @@ -380,7 +395,7 @@ function SJISToUTF8(data) { var results = []; var i = 0; var len = data && data.length; - var b, b1, b2, u2, u3, jis, utf8; + var b, b1, b2, u2, u3, jis, utf8, remapped; for (; i < len; i++) { b = data[i]; @@ -393,9 +408,15 @@ function SJISToUTF8(data) { results[results.length] = u2 & 0xFF; results[results.length] = u3 & 0xFF; } else if (b >= 0x80) { - b1 = b << 1; b2 = data[++i]; + if (sjisExt.hasCP932DuplicateCode(b)) { + // Remap CP932 duplicate codes and IBM extended characters + remapped = sjisExt.remapCP932DuplicateCode(b, b2); + b = remapped >> 8; + b2 = remapped & 0xFF; + } + b1 = b << 1; if (b2 < 0x9F) { if (b1 < 0x13F) { b1 -= 0xE1; diff --git a/src/encoding-detect.js b/src/encoding-detect.js index 292aff0..eb0b766 100644 --- a/src/encoding-detect.js +++ b/src/encoding-detect.js @@ -153,32 +153,39 @@ function isEUCJP(data) { exports.isEUCJP = isEUCJP; /** - * Shift-JIS (SJIS) + * SJIS (Shift_JIS) + * + * Includes CP932 (Windows-31J) extensions: + * - NEC special character area (0x8740 - 0x879C) + * - NEC-selected IBM extended character area (0xED40 - 0xEEFC) + * - User-defined character area (0xF040 - 0xF9FC) + * - IBM extended character area (0xFA40 - 0xFC4B) */ function isSJIS(data) { var i = 0; var len = data && data.length; - var b; + var b, lead; - while (i < len && data[i] > 0x80) { - if (data[i++] > 0xFF) { + for (; i < len; i++) { + b = data[i]; + if (b > 0xFF) { return false; } - } - for (; i < len; i++) { - b = data[i]; if (b <= 0x80 || (0xA1 <= b && b <= 0xDF)) { continue; } - if (b === 0xA0 || b > 0xEF || i + 1 >= len) { + if (b === 0xA0 || b > 0xFC || i + 1 >= len) { return false; } + lead = b; b = data[++i]; - if (b < 0x40 || b === 0x7F || b > 0xFC) { + if (b < 0x40 || b > 0xFC || b === 0x7F || + // IBM extended character area ends at 0xFC4B + (lead === 0xFC && b > 0x4B)) { return false; } } diff --git a/src/sjis-ext.js b/src/sjis-ext.js new file mode 100644 index 0000000..aa9a092 --- /dev/null +++ b/src/sjis-ext.js @@ -0,0 +1,94 @@ +/** + * Remapping CP932 (Windows-31J) IBM extended characters (0xFA40 - 0xFC4B) + * + * Remaps IBM extended characters to their corresponding duplicate codes + * before conversion. These characters are also assigned in the NEC special + * characters (0x8740 - 0x879C), the NEC-selected IBM extended characters + * (0xED40 - 0xEEFC), and JIS X 0208. + * + * The first 28 characters (0xFA40 - 0xFA5B) have individual mappings defined + * in CP932_IBM_EXT_SYMBOL_MAP, and the rest are linearly mapped to 0xED40 - 0xEEEC. + */ +var CP932_IBM_EXT_SYMBOL_MAP = [ + // 0xFA40 - 0xFA49 [ⅰ-ⅹ] + 0xEEEF, 0xEEF0, 0xEEF1, 0xEEF2, 0xEEF3, 0xEEF4, 0xEEF5, 0xEEF6, 0xEEF7, 0xEEF8, + // 0xFA4A - 0xFA53 [Ⅰ-Ⅹ] + 0x8754, 0x8755, 0x8756, 0x8757, 0x8758, 0x8759, 0x875A, 0x875B, 0x875C, 0x875D, + // 0xFA54 - 0xFA57 [¬¦'"] + 0x81CA, 0xEEFA, 0xEEFB, 0xEEFC, + // 0xFA58 - 0xFA5B [㈱№℡∵] + 0x878A, 0x8782, 0x8784, 0x81E6 +]; + +// The number of IBM extended characters (0xFA40 - 0xFC4B) +var CP932_IBM_EXT_LEN = 388; +// The number of valid trail bytes per lead byte (0x40 - 0xFC, except 0x7F) +var CP932_IBM_EXT_TRAIL_BYTES_LEN = 188; + +function remapCP932_IBMExt(b1, b2) { + var leadOffset = (b1 - 0xFA) * CP932_IBM_EXT_TRAIL_BYTES_LEN; + var trailIndex = b2 - (b2 < 0x7F ? 0x40 : 0x41); + var ibmExtIndex = leadOffset + trailIndex; + + if (ibmExtIndex < 0 || ibmExtIndex >= CP932_IBM_EXT_LEN) { + return (b1 << 8) | b2; + } + + if (ibmExtIndex < CP932_IBM_EXT_SYMBOL_MAP.length) { + return CP932_IBM_EXT_SYMBOL_MAP[ibmExtIndex]; + } + + var necSelectedIbmOffset = ibmExtIndex - CP932_IBM_EXT_SYMBOL_MAP.length; + + // Remap to NEC-selected IBM extended characters (0xED40 - 0xEEEC) + b1 = 0xED; + if (necSelectedIbmOffset >= CP932_IBM_EXT_TRAIL_BYTES_LEN) { + necSelectedIbmOffset -= CP932_IBM_EXT_TRAIL_BYTES_LEN; + b1++; + } + + b2 = necSelectedIbmOffset + 0x40; + if (b2 >= 0x7F) { + b2++; // Skip invalid trail byte 0x7F + } + return (b1 << 8) | b2; +} + +/** + * Remapping duplicate codes for CP932 NEC special characters to their + * standard JIS X 0208 codes. + * + * This also includes 0xEEF9 in the NEC-selected IBM extended characters. + */ +var CP932_NEC_DUPLICATE_MAP = { + 0x8790: 0x81E0, // ≒ (NEC special characters) + 0x8791: 0x81DF, // ≡ + 0x8792: 0x81E7, // ∫ + 0x8795: 0x81E3, // √ + 0x8796: 0x81DB, // ⊥ + 0x8797: 0x81DA, // ∠ + 0x879A: 0x81E6, // ∵ + 0x879B: 0x81BF, // ∩ + 0x879C: 0x81BE, // ∪ + 0xEEF9: 0x81CA // ¬ (NEC-selected IBM extended characters) +}; + +function remapCP932DuplicateCode(b1, b2) { + if (b2 < 0x40 || b2 > 0xFC || b2 === 0x7F) { + return (b1 << 8) | (b2 & 0xFF); + } + + if (b1 >= 0xFA) { + return remapCP932_IBMExt(b1, b2); + } + + var code = (b1 << 8) | b2; + var remapped = CP932_NEC_DUPLICATE_MAP[code]; + return remapped == null ? code : remapped; +} +exports.remapCP932DuplicateCode = remapCP932DuplicateCode; + +function hasCP932DuplicateCode(b1) { + return b1 >= 0xFA || b1 === 0x87 || b1 === 0xEE; +} +exports.hasCP932DuplicateCode = hasCP932DuplicateCode; diff --git a/tests/cp932-ibm-ext-map.js b/tests/cp932-ibm-ext-map.js new file mode 100644 index 0000000..877476b --- /dev/null +++ b/tests/cp932-ibm-ext-map.js @@ -0,0 +1,401 @@ +/** + * SJIS (CP932) IBM extended characters (0xFA40 - 0xFC4B) + * + * Generated from CP932.TXT: + * https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT + * + * The keys are the SJIS code points and the values are the Unicode + * code points, in the same order as CP932.TXT. + * + * { SJIS : UNICODE } + */ +module.exports = { + 0xFA40: 0x2170, // ⅰ + 0xFA41: 0x2171, // ⅱ + 0xFA42: 0x2172, // ⅲ + 0xFA43: 0x2173, // ⅳ + 0xFA44: 0x2174, // ⅴ + 0xFA45: 0x2175, // ⅵ + 0xFA46: 0x2176, // ⅶ + 0xFA47: 0x2177, // ⅷ + 0xFA48: 0x2178, // ⅸ + 0xFA49: 0x2179, // ⅹ + 0xFA4A: 0x2160, // Ⅰ + 0xFA4B: 0x2161, // Ⅱ + 0xFA4C: 0x2162, // Ⅲ + 0xFA4D: 0x2163, // Ⅳ + 0xFA4E: 0x2164, // Ⅴ + 0xFA4F: 0x2165, // Ⅵ + 0xFA50: 0x2166, // Ⅶ + 0xFA51: 0x2167, // Ⅷ + 0xFA52: 0x2168, // Ⅸ + 0xFA53: 0x2169, // Ⅹ + 0xFA54: 0xFFE2, // ¬ + 0xFA55: 0xFFE4, // ¦ + 0xFA56: 0xFF07, // ' + 0xFA57: 0xFF02, // " + 0xFA58: 0x3231, // ㈱ + 0xFA59: 0x2116, // № + 0xFA5A: 0x2121, // ℡ + 0xFA5B: 0x2235, // ∵ + 0xFA5C: 0x7E8A, // 纊 + 0xFA5D: 0x891C, // 褜 + 0xFA5E: 0x9348, // 鍈 + 0xFA5F: 0x9288, // 銈 + 0xFA60: 0x84DC, // 蓜 + 0xFA61: 0x4FC9, // 俉 + 0xFA62: 0x70BB, // 炻 + 0xFA63: 0x6631, // 昱 + 0xFA64: 0x68C8, // 棈 + 0xFA65: 0x92F9, // 鋹 + 0xFA66: 0x66FB, // 曻 + 0xFA67: 0x5F45, // 彅 + 0xFA68: 0x4E28, // 丨 + 0xFA69: 0x4EE1, // 仡 + 0xFA6A: 0x4EFC, // 仼 + 0xFA6B: 0x4F00, // 伀 + 0xFA6C: 0x4F03, // 伃 + 0xFA6D: 0x4F39, // 伹 + 0xFA6E: 0x4F56, // 佖 + 0xFA6F: 0x4F92, // 侒 + 0xFA70: 0x4F8A, // 侊 + 0xFA71: 0x4F9A, // 侚 + 0xFA72: 0x4F94, // 侔 + 0xFA73: 0x4FCD, // 俍 + 0xFA74: 0x5040, // 偀 + 0xFA75: 0x5022, // 倢 + 0xFA76: 0x4FFF, // 俿 + 0xFA77: 0x501E, // 倞 + 0xFA78: 0x5046, // 偆 + 0xFA79: 0x5070, // 偰 + 0xFA7A: 0x5042, // 偂 + 0xFA7B: 0x5094, // 傔 + 0xFA7C: 0x50F4, // 僴 + 0xFA7D: 0x50D8, // 僘 + 0xFA7E: 0x514A, // 兊 + 0xFA80: 0x5164, // 兤 + 0xFA81: 0x519D, // 冝 + 0xFA82: 0x51BE, // 冾 + 0xFA83: 0x51EC, // 凬 + 0xFA84: 0x5215, // 刕 + 0xFA85: 0x529C, // 劜 + 0xFA86: 0x52A6, // 劦 + 0xFA87: 0x52C0, // 勀 + 0xFA88: 0x52DB, // 勛 + 0xFA89: 0x5300, // 匀 + 0xFA8A: 0x5307, // 匇 + 0xFA8B: 0x5324, // 匤 + 0xFA8C: 0x5372, // 卲 + 0xFA8D: 0x5393, // 厓 + 0xFA8E: 0x53B2, // 厲 + 0xFA8F: 0x53DD, // 叝 + 0xFA90: 0xFA0E, // 﨎 + 0xFA91: 0x549C, // 咜 + 0xFA92: 0x548A, // 咊 + 0xFA93: 0x54A9, // 咩 + 0xFA94: 0x54FF, // 哿 + 0xFA95: 0x5586, // 喆 + 0xFA96: 0x5759, // 坙 + 0xFA97: 0x5765, // 坥 + 0xFA98: 0x57AC, // 垬 + 0xFA99: 0x57C8, // 埈 + 0xFA9A: 0x57C7, // 埇 + 0xFA9B: 0xFA0F, // 﨏 + 0xFA9C: 0xFA10, // 塚 + 0xFA9D: 0x589E, // 增 + 0xFA9E: 0x58B2, // 墲 + 0xFA9F: 0x590B, // 夋 + 0xFAA0: 0x5953, // 奓 + 0xFAA1: 0x595B, // 奛 + 0xFAA2: 0x595D, // 奝 + 0xFAA3: 0x5963, // 奣 + 0xFAA4: 0x59A4, // 妤 + 0xFAA5: 0x59BA, // 妺 + 0xFAA6: 0x5B56, // 孖 + 0xFAA7: 0x5BC0, // 寀 + 0xFAA8: 0x752F, // 甯 + 0xFAA9: 0x5BD8, // 寘 + 0xFAAA: 0x5BEC, // 寬 + 0xFAAB: 0x5C1E, // 尞 + 0xFAAC: 0x5CA6, // 岦 + 0xFAAD: 0x5CBA, // 岺 + 0xFAAE: 0x5CF5, // 峵 + 0xFAAF: 0x5D27, // 崧 + 0xFAB0: 0x5D53, // 嵓 + 0xFAB1: 0xFA11, // 﨑 + 0xFAB2: 0x5D42, // 嵂 + 0xFAB3: 0x5D6D, // 嵭 + 0xFAB4: 0x5DB8, // 嶸 + 0xFAB5: 0x5DB9, // 嶹 + 0xFAB6: 0x5DD0, // 巐 + 0xFAB7: 0x5F21, // 弡 + 0xFAB8: 0x5F34, // 弴 + 0xFAB9: 0x5F67, // 彧 + 0xFABA: 0x5FB7, // 德 + 0xFABB: 0x5FDE, // 忞 + 0xFABC: 0x605D, // 恝 + 0xFABD: 0x6085, // 悅 + 0xFABE: 0x608A, // 悊 + 0xFABF: 0x60DE, // 惞 + 0xFAC0: 0x60D5, // 惕 + 0xFAC1: 0x6120, // 愠 + 0xFAC2: 0x60F2, // 惲 + 0xFAC3: 0x6111, // 愑 + 0xFAC4: 0x6137, // 愷 + 0xFAC5: 0x6130, // 愰 + 0xFAC6: 0x6198, // 憘 + 0xFAC7: 0x6213, // 戓 + 0xFAC8: 0x62A6, // 抦 + 0xFAC9: 0x63F5, // 揵 + 0xFACA: 0x6460, // 摠 + 0xFACB: 0x649D, // 撝 + 0xFACC: 0x64CE, // 擎 + 0xFACD: 0x654E, // 敎 + 0xFACE: 0x6600, // 昀 + 0xFACF: 0x6615, // 昕 + 0xFAD0: 0x663B, // 昻 + 0xFAD1: 0x6609, // 昉 + 0xFAD2: 0x662E, // 昮 + 0xFAD3: 0x661E, // 昞 + 0xFAD4: 0x6624, // 昤 + 0xFAD5: 0x6665, // 晥 + 0xFAD6: 0x6657, // 晗 + 0xFAD7: 0x6659, // 晙 + 0xFAD8: 0xFA12, // 晴 + 0xFAD9: 0x6673, // 晳 + 0xFADA: 0x6699, // 暙 + 0xFADB: 0x66A0, // 暠 + 0xFADC: 0x66B2, // 暲 + 0xFADD: 0x66BF, // 暿 + 0xFADE: 0x66FA, // 曺 + 0xFADF: 0x670E, // 朎 + 0xFAE0: 0xF929, // 朗 + 0xFAE1: 0x6766, // 杦 + 0xFAE2: 0x67BB, // 枻 + 0xFAE3: 0x6852, // 桒 + 0xFAE4: 0x67C0, // 柀 + 0xFAE5: 0x6801, // 栁 + 0xFAE6: 0x6844, // 桄 + 0xFAE7: 0x68CF, // 棏 + 0xFAE8: 0xFA13, // 﨓 + 0xFAE9: 0x6968, // 楨 + 0xFAEA: 0xFA14, // 﨔 + 0xFAEB: 0x6998, // 榘 + 0xFAEC: 0x69E2, // 槢 + 0xFAED: 0x6A30, // 樰 + 0xFAEE: 0x6A6B, // 橫 + 0xFAEF: 0x6A46, // 橆 + 0xFAF0: 0x6A73, // 橳 + 0xFAF1: 0x6A7E, // 橾 + 0xFAF2: 0x6AE2, // 櫢 + 0xFAF3: 0x6AE4, // 櫤 + 0xFAF4: 0x6BD6, // 毖 + 0xFAF5: 0x6C3F, // 氿 + 0xFAF6: 0x6C5C, // 汜 + 0xFAF7: 0x6C86, // 沆 + 0xFAF8: 0x6C6F, // 汯 + 0xFAF9: 0x6CDA, // 泚 + 0xFAFA: 0x6D04, // 洄 + 0xFAFB: 0x6D87, // 涇 + 0xFAFC: 0x6D6F, // 浯 + 0xFB40: 0x6D96, // 涖 + 0xFB41: 0x6DAC, // 涬 + 0xFB42: 0x6DCF, // 淏 + 0xFB43: 0x6DF8, // 淸 + 0xFB44: 0x6DF2, // 淲 + 0xFB45: 0x6DFC, // 淼 + 0xFB46: 0x6E39, // 渹 + 0xFB47: 0x6E5C, // 湜 + 0xFB48: 0x6E27, // 渧 + 0xFB49: 0x6E3C, // 渼 + 0xFB4A: 0x6EBF, // 溿 + 0xFB4B: 0x6F88, // 澈 + 0xFB4C: 0x6FB5, // 澵 + 0xFB4D: 0x6FF5, // 濵 + 0xFB4E: 0x7005, // 瀅 + 0xFB4F: 0x7007, // 瀇 + 0xFB50: 0x7028, // 瀨 + 0xFB51: 0x7085, // 炅 + 0xFB52: 0x70AB, // 炫 + 0xFB53: 0x710F, // 焏 + 0xFB54: 0x7104, // 焄 + 0xFB55: 0x715C, // 煜 + 0xFB56: 0x7146, // 煆 + 0xFB57: 0x7147, // 煇 + 0xFB58: 0xFA15, // 凞 + 0xFB59: 0x71C1, // 燁 + 0xFB5A: 0x71FE, // 燾 + 0xFB5B: 0x72B1, // 犱 + 0xFB5C: 0x72BE, // 犾 + 0xFB5D: 0x7324, // 猤 + 0xFB5E: 0xFA16, // 猪 + 0xFB5F: 0x7377, // 獷 + 0xFB60: 0x73BD, // 玽 + 0xFB61: 0x73C9, // 珉 + 0xFB62: 0x73D6, // 珖 + 0xFB63: 0x73E3, // 珣 + 0xFB64: 0x73D2, // 珒 + 0xFB65: 0x7407, // 琇 + 0xFB66: 0x73F5, // 珵 + 0xFB67: 0x7426, // 琦 + 0xFB68: 0x742A, // 琪 + 0xFB69: 0x7429, // 琩 + 0xFB6A: 0x742E, // 琮 + 0xFB6B: 0x7462, // 瑢 + 0xFB6C: 0x7489, // 璉 + 0xFB6D: 0x749F, // 璟 + 0xFB6E: 0x7501, // 甁 + 0xFB6F: 0x756F, // 畯 + 0xFB70: 0x7682, // 皂 + 0xFB71: 0x769C, // 皜 + 0xFB72: 0x769E, // 皞 + 0xFB73: 0x769B, // 皛 + 0xFB74: 0x76A6, // 皦 + 0xFB75: 0xFA17, // 益 + 0xFB76: 0x7746, // 睆 + 0xFB77: 0x52AF, // 劯 + 0xFB78: 0x7821, // 砡 + 0xFB79: 0x784E, // 硎 + 0xFB7A: 0x7864, // 硤 + 0xFB7B: 0x787A, // 硺 + 0xFB7C: 0x7930, // 礰 + 0xFB7D: 0xFA18, // 礼 + 0xFB7E: 0xFA19, // 神 + 0xFB80: 0xFA1A, // 祥 + 0xFB81: 0x7994, // 禔 + 0xFB82: 0xFA1B, // 福 + 0xFB83: 0x799B, // 禛 + 0xFB84: 0x7AD1, // 竑 + 0xFB85: 0x7AE7, // 竧 + 0xFB86: 0xFA1C, // 靖 + 0xFB87: 0x7AEB, // 竫 + 0xFB88: 0x7B9E, // 箞 + 0xFB89: 0xFA1D, // 精 + 0xFB8A: 0x7D48, // 絈 + 0xFB8B: 0x7D5C, // 絜 + 0xFB8C: 0x7DB7, // 綷 + 0xFB8D: 0x7DA0, // 綠 + 0xFB8E: 0x7DD6, // 緖 + 0xFB8F: 0x7E52, // 繒 + 0xFB90: 0x7F47, // 罇 + 0xFB91: 0x7FA1, // 羡 + 0xFB92: 0xFA1E, // 羽 + 0xFB93: 0x8301, // 茁 + 0xFB94: 0x8362, // 荢 + 0xFB95: 0x837F, // 荿 + 0xFB96: 0x83C7, // 菇 + 0xFB97: 0x83F6, // 菶 + 0xFB98: 0x8448, // 葈 + 0xFB99: 0x84B4, // 蒴 + 0xFB9A: 0x8553, // 蕓 + 0xFB9B: 0x8559, // 蕙 + 0xFB9C: 0x856B, // 蕫 + 0xFB9D: 0xFA1F, // 﨟 + 0xFB9E: 0x85B0, // 薰 + 0xFB9F: 0xFA20, // 蘒 + 0xFBA0: 0xFA21, // 﨡 + 0xFBA1: 0x8807, // 蠇 + 0xFBA2: 0x88F5, // 裵 + 0xFBA3: 0x8A12, // 訒 + 0xFBA4: 0x8A37, // 訷 + 0xFBA5: 0x8A79, // 詹 + 0xFBA6: 0x8AA7, // 誧 + 0xFBA7: 0x8ABE, // 誾 + 0xFBA8: 0x8ADF, // 諟 + 0xFBA9: 0xFA22, // 諸 + 0xFBAA: 0x8AF6, // 諶 + 0xFBAB: 0x8B53, // 譓 + 0xFBAC: 0x8B7F, // 譿 + 0xFBAD: 0x8CF0, // 賰 + 0xFBAE: 0x8CF4, // 賴 + 0xFBAF: 0x8D12, // 贒 + 0xFBB0: 0x8D76, // 赶 + 0xFBB1: 0xFA23, // 﨣 + 0xFBB2: 0x8ECF, // 軏 + 0xFBB3: 0xFA24, // 﨤 + 0xFBB4: 0xFA25, // 逸 + 0xFBB5: 0x9067, // 遧 + 0xFBB6: 0x90DE, // 郞 + 0xFBB7: 0xFA26, // 都 + 0xFBB8: 0x9115, // 鄕 + 0xFBB9: 0x9127, // 鄧 + 0xFBBA: 0x91DA, // 釚 + 0xFBBB: 0x91D7, // 釗 + 0xFBBC: 0x91DE, // 釞 + 0xFBBD: 0x91ED, // 釭 + 0xFBBE: 0x91EE, // 釮 + 0xFBBF: 0x91E4, // 釤 + 0xFBC0: 0x91E5, // 釥 + 0xFBC1: 0x9206, // 鈆 + 0xFBC2: 0x9210, // 鈐 + 0xFBC3: 0x920A, // 鈊 + 0xFBC4: 0x923A, // 鈺 + 0xFBC5: 0x9240, // 鉀 + 0xFBC6: 0x923C, // 鈼 + 0xFBC7: 0x924E, // 鉎 + 0xFBC8: 0x9259, // 鉙 + 0xFBC9: 0x9251, // 鉑 + 0xFBCA: 0x9239, // 鈹 + 0xFBCB: 0x9267, // 鉧 + 0xFBCC: 0x92A7, // 銧 + 0xFBCD: 0x9277, // 鉷 + 0xFBCE: 0x9278, // 鉸 + 0xFBCF: 0x92E7, // 鋧 + 0xFBD0: 0x92D7, // 鋗 + 0xFBD1: 0x92D9, // 鋙 + 0xFBD2: 0x92D0, // 鋐 + 0xFBD3: 0xFA27, // 﨧 + 0xFBD4: 0x92D5, // 鋕 + 0xFBD5: 0x92E0, // 鋠 + 0xFBD6: 0x92D3, // 鋓 + 0xFBD7: 0x9325, // 錥 + 0xFBD8: 0x9321, // 錡 + 0xFBD9: 0x92FB, // 鋻 + 0xFBDA: 0xFA28, // 﨨 + 0xFBDB: 0x931E, // 錞 + 0xFBDC: 0x92FF, // 鋿 + 0xFBDD: 0x931D, // 錝 + 0xFBDE: 0x9302, // 錂 + 0xFBDF: 0x9370, // 鍰 + 0xFBE0: 0x9357, // 鍗 + 0xFBE1: 0x93A4, // 鎤 + 0xFBE2: 0x93C6, // 鏆 + 0xFBE3: 0x93DE, // 鏞 + 0xFBE4: 0x93F8, // 鏸 + 0xFBE5: 0x9431, // 鐱 + 0xFBE6: 0x9445, // 鑅 + 0xFBE7: 0x9448, // 鑈 + 0xFBE8: 0x9592, // 閒 + 0xFBE9: 0xF9DC, // 隆 + 0xFBEA: 0xFA29, // 﨩 + 0xFBEB: 0x969D, // 隝 + 0xFBEC: 0x96AF, // 隯 + 0xFBED: 0x9733, // 霳 + 0xFBEE: 0x973B, // 霻 + 0xFBEF: 0x9743, // 靃 + 0xFBF0: 0x974D, // 靍 + 0xFBF1: 0x974F, // 靏 + 0xFBF2: 0x9751, // 靑 + 0xFBF3: 0x9755, // 靕 + 0xFBF4: 0x9857, // 顗 + 0xFBF5: 0x9865, // 顥 + 0xFBF6: 0xFA2A, // 飯 + 0xFBF7: 0xFA2B, // 飼 + 0xFBF8: 0x9927, // 餧 + 0xFBF9: 0xFA2C, // 館 + 0xFBFA: 0x999E, // 馞 + 0xFBFB: 0x9A4E, // 驎 + 0xFBFC: 0x9AD9, // 髙 + 0xFC40: 0x9ADC, // 髜 + 0xFC41: 0x9B75, // 魵 + 0xFC42: 0x9B72, // 魲 + 0xFC43: 0x9B8F, // 鮏 + 0xFC44: 0x9BB1, // 鮱 + 0xFC45: 0x9BBB, // 鮻 + 0xFC46: 0x9C00, // 鰀 + 0xFC47: 0x9D70, // 鵰 + 0xFC48: 0x9D6B, // 鵫 + 0xFC49: 0xFA2D, // 鶴 + 0xFC4A: 0x9E19, // 鸙 + 0xFC4B: 0x9ED1 // 黑 +}; diff --git a/tests/cp932-nec-duplicate-map.js b/tests/cp932-nec-duplicate-map.js new file mode 100644 index 0000000..2b0697c --- /dev/null +++ b/tests/cp932-nec-duplicate-map.js @@ -0,0 +1,26 @@ +/** + * Unicode mappings for SJIS (CP932) duplicate codes + * + * Includes: + * - NEC special characters (9 codes in 0x8740 - 0x879C) + * - NEC-selected IBM extended characters (0xEEF9) + * + * Reference: + * https://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT + * + * The keys are SJIS codes and the values are Unicode code points. + * + * { SJIS : UNICODE } + */ +module.exports = { + 0x8790: 0x2252, // ≒ (NEC special characters) + 0x8791: 0x2261, // ≡ + 0x8792: 0x222B, // ∫ + 0x8795: 0x221A, // √ + 0x8796: 0x22A5, // ⊥ + 0x8797: 0x2220, // ∠ + 0x879A: 0x2235, // ∵ + 0x879B: 0x2229, // ∩ + 0x879C: 0x222A, // ∪ + 0xEEF9: 0xFFE2 // ¬ (NEC-selected IBM extended characters) +}; diff --git a/tests/test.js b/tests/test.js index 89aa5ed..34813c8 100644 --- a/tests/test.js +++ b/tests/test.js @@ -155,6 +155,79 @@ describe('encoding', function() { assert(encoding.detect(utf32le) === 'UTF32'); }); + it('SJIS detection does not skip leading bytes above 0x80', function() { + // "あいう" in UTF-8 + // All bytes are above 0x80 and should not be skipped before SJIS validation + // In encoding.js 2.2.0 and earlier, leading bytes above 0x80 were skipped. + var utf8 = [0xE3, 0x81, 0x82, 0xE3, 0x81, 0x84, 0xE3, 0x81, 0x86]; + + assert.equal(encoding.detect(utf8, 'SJIS'), false); + assert.equal(encoding.detect([0x81], 'SJIS'), false); + assert.equal(encoding.detect([0x65E5], 'SJIS'), false); + }); + + it('SJIS (CP932) extended character areas', function() { + var isSJIS = function(data) { + return encoding.detect(data, 'SJIS'); + }; + + // NEC special character area (0x8740 - 0x879C) + assert.equal(isSJIS([0x87, 0x40]), 'SJIS'); + assert.equal(isSJIS([0x87, 0x9C]), 'SJIS'); + // NEC-selected IBM extended character area (0xED40 - 0xEEFC) + assert.equal(isSJIS([0xED, 0x40]), 'SJIS'); + assert.equal(isSJIS([0xEE, 0xFC]), 'SJIS'); + // User-defined character area (0xF040 - 0xF9FC) + assert.equal(isSJIS([0xF0, 0x40]), 'SJIS'); + assert.equal(isSJIS([0xF9, 0xFC]), 'SJIS'); + // IBM extended character area (0xFA40 - 0xFC4B) + assert.equal(isSJIS([0xFA, 0x40]), 'SJIS'); + assert.equal(isSJIS([0xFC, 0x4B]), 'SJIS'); + + // Check "テスト" followed by the first and last character of each extended area + var sjisPrefix = [0x83, 0x65, 0x83, 0x58, 0x83, 0x67]; // "テスト" in SJIS + + // "①" (0x8740) : the first NEC special character + assert.equal(encoding.detect(sjisPrefix.concat([0x87, 0x40])), 'SJIS'); + // "∪" (0x879C) : the last NEC special character + assert.equal(encoding.detect(sjisPrefix.concat([0x87, 0x9C])), 'SJIS'); + + // the first NEC-selected IBM extended character + assert.equal(encoding.detect(sjisPrefix.concat([0xED, 0x40])), 'SJIS'); + // the last NEC-selected IBM extended character + assert.equal(encoding.detect(sjisPrefix.concat([0xEE, 0xFC])), 'SJIS'); + + // the first User-defined character + assert.equal(encoding.detect(sjisPrefix.concat([0xF0, 0x40])), 'SJIS'); + // the last User-defined character + assert.equal(encoding.detect(sjisPrefix.concat([0xF9, 0xFC])), 'SJIS'); + + // the first IBM extended character + assert.equal(encoding.detect(sjisPrefix.concat([0xFA, 0x40])), 'SJIS'); + // the last IBM extended character + assert.equal(encoding.detect(sjisPrefix.concat([0xFC, 0x4B])), 'SJIS'); + }); + + it('Invalid SJIS byte sequences', function() { + // IBM extended character area ends at 0xFC4B + assert.equal(encoding.detect([0xFC, 0x4C], 'SJIS'), false); + assert.equal(encoding.detect([0x83, 0x65, 0xFC, 0x4C], 'SJIS'), false); + assert.equal(encoding.detect([0x83, 0x65, 0xFC, 0xFC], 'SJIS'), false); + + // Invalid lead bytes + assert.equal(encoding.detect([0xA0, 0x40], 'SJIS'), false); + assert.equal(encoding.detect([0xFD, 0x40], 'SJIS'), false); + + // Trail bytes outside 0x40 - 0xFC, and 0x7F is not used in SJIS + assert.equal(encoding.detect([0x81, 0x3F], 'SJIS'), false); + assert.equal(encoding.detect([0x81, 0x7F], 'SJIS'), false); + assert.equal(encoding.detect([0x81, 0xFD], 'SJIS'), false); + + // A lead byte that is not followed by a trail byte + assert.equal(encoding.detect([0xFA], 'SJIS'), false); + assert.equal(encoding.detect([0x83, 0x65, 0x83], 'SJIS'), false); + }); + it('Specifying multiple encodings', function() { var unicode = 'ユニコード'; @@ -1053,6 +1126,82 @@ describe('encoding', function() { }); }); + describe('convert CP932 extended characters', function() { + var ibmExtTable = require('./cp932-ibm-ext-map'); + var ibmExtSJIS = []; + var ibmExtUnicode = []; + + Object.keys(ibmExtTable).forEach(function(sjis) { + sjis = sjis | 0; + var unicode = ibmExtTable[sjis] | 0; + ibmExtSJIS.push(sjis >> 8, sjis & 0xFF); + ibmExtUnicode.push(unicode); + }); + + it('SJIS to UNICODE', function() { + var sjis = encoding.convert(ibmExtSJIS, { + to: 'unicode', + from: 'sjis' + }); + assert.deepEqual(sjis, ibmExtUnicode); + }); + + it('SJIS to UTF-8', function() { + var utf8 = encoding.convert(ibmExtSJIS, { + to: 'utf-8', + from: 'sjis' + }); + var res = encoding.convert(utf8, { + to: 'unicode', + from: 'utf-8' + }); + assert.deepEqual(res, ibmExtUnicode); + }); + + it('SJIS to EUC-JP', function() { + var eucjp = encoding.convert(ibmExtSJIS, { + to: 'euc-jp', + from: 'sjis' + }); + var res = encoding.convert(eucjp, { + to: 'unicode', + from: 'euc-jp' + }); + assert.deepEqual(res, ibmExtUnicode); + }); + + it('SJIS to JIS', function() { + var jis = encoding.convert(ibmExtSJIS, { + to: 'jis', + from: 'sjis' + }); + var res = encoding.convert(jis, { + to: 'unicode', + from: 'jis' + }); + assert.deepEqual(res, ibmExtUnicode); + }); + + describe('convert CP932 NEC duplicate codes', function() { + var cp932NECDuplicateMap = require('./cp932-nec-duplicate-map'); + + it('SJIS to UNICODE', function() { + Object.keys(cp932NECDuplicateMap).forEach(function(sjis) { + sjis = sjis | 0; + var unicode = cp932NECDuplicateMap[sjis]; + var bytes = [sjis >> 8, sjis & 0xFF]; + + var res = encoding.convert(bytes, { to: 'unicode', from: 'sjis' }); + assert.deepEqual( + res, + [unicode], + '0x' + sjis.toString(16).toUpperCase() + ); + }); + }); + }); + }); + describe('urlEncode/urlDecode', function() { encodings.forEach(function(encodingName) { it(encodingName, function () {