feat: add CP932 extensions in SJIS detection and conversion

src/encoding-convert.js

@@ -2,6 +2,7 @@ var config = require('./config');
 var util = require('./util');
 var EncodingDetect = require('./encoding-detect');
 var EncodingTable = require('./encoding-table');
+var sjisExt = require('./sjis-ext');
 
 /**
  * JIS to SJIS
@@ -134,7 +135,7 @@ function SJISToJIS(data) {
   var index = 0;
   var len = data && data.length;
   var i = 0;
-  var b1, b2;
+  var b1, b2, remapped;
 
   var esc = [
     0x1B, 0x28, 0x42,
@@ -160,8 +161,15 @@ function SJISToJIS(data) {
         results[results.length] = esc[5];
       }
 
-      b1 <<= 1;
       b2 = data[++i];
+      if (sjisExt.hasCP932DuplicateCode(b1)) {
+        // Remap CP932 duplicate codes and IBM extended characters
+        remapped = sjisExt.remapCP932DuplicateCode(b1, b2);
+        b1 = remapped >> 8;
+        b2 = remapped & 0xFF;
+      }
+
+      b1 <<= 1;
       if (b2 < 0x9F) {
         if (b1 < 0x13F) {
           b1 -= 0xE1;
@@ -211,7 +219,7 @@ function SJISToEUCJP(data) {
   var results = [];
   var len = data && data.length;
   var i = 0;
-  var b1, b2;
+  var b1, b2, remapped;
 
   for (; i < len; i++) {
     b1 = data[i];
@@ -220,6 +228,13 @@ function SJISToEUCJP(data) {
       results[results.length] = b1;
     } else if (b1 >= 0x81) {
       b2 = data[++i];
+      if (sjisExt.hasCP932DuplicateCode(b1)) {
+        // Remap CP932 duplicate codes and IBM extended characters
+        remapped = sjisExt.remapCP932DuplicateCode(b1, b2);
+        b1 = remapped >> 8;
+        b2 = remapped & 0xFF;
+      }
+
       b1 <<= 1;
       if (b2 < 0x9F) {
         if (b1 < 0x13F) {
@@ -380,7 +395,7 @@ function SJISToUTF8(data) {
   var results = [];
   var i = 0;
   var len = data && data.length;
-  var b, b1, b2, u2, u3, jis, utf8;
+  var b, b1, b2, u2, u3, jis, utf8, remapped;
 
   for (; i < len; i++) {
     b = data[i];
@@ -393,9 +408,15 @@ function SJISToUTF8(data) {
       results[results.length] = u2 & 0xFF;
       results[results.length] = u3 & 0xFF;
     } else if (b >= 0x80) {
-      b1 = b << 1;
       b2 = data[++i];
+      if (sjisExt.hasCP932DuplicateCode(b)) {
+        // Remap CP932 duplicate codes and IBM extended characters
+        remapped = sjisExt.remapCP932DuplicateCode(b, b2);
+        b = remapped >> 8;
+        b2 = remapped & 0xFF;
+      }
 
+      b1 = b << 1;
       if (b2 < 0x9F) {
         if (b1 < 0x13F) {
           b1 -= 0xE1;

src/encoding-detect.js

@@ -153,32 +153,39 @@ function isEUCJP(data) {
 exports.isEUCJP = isEUCJP;
 
 /**
- * Shift-JIS (SJIS)
+ * SJIS (Shift_JIS)
+ *
+ * Includes CP932 (Windows-31J) extensions:
+ * - NEC special character area (0x8740 - 0x879C)
+ * - NEC-selected IBM extended character area (0xED40 - 0xEEFC)
+ * - User-defined character area (0xF040 - 0xF9FC)
+ * - IBM extended character area (0xFA40 - 0xFC4B)
  */
 function isSJIS(data) {
   var i = 0;
   var len = data && data.length;
-  var b;
+  var b, lead;
 
-  while (i < len && data[i] > 0x80) {
-    if (data[i++] > 0xFF) {
+  for (; i < len; i++) {
+    b = data[i];
+    if (b > 0xFF) {
       return false;
     }
-  }
 
-  for (; i < len; i++) {
-    b = data[i];
     if (b <= 0x80 ||
         (0xA1 <= b && b <= 0xDF)) {
       continue;
     }
 
-    if (b === 0xA0 || b > 0xEF || i + 1 >= len) {
+    if (b === 0xA0 || b > 0xFC || i + 1 >= len) {
       return false;
     }
 
+    lead = b;
     b = data[++i];
-    if (b < 0x40 || b === 0x7F || b > 0xFC) {
+    if (b < 0x40 || b > 0xFC || b === 0x7F ||
+        // IBM extended character area ends at 0xFC4B
+        (lead === 0xFC && b > 0x4B)) {
       return false;
     }
   }

src/sjis-ext.js

@@ -0,0 +1,94 @@
+/**
+ * Remapping CP932 (Windows-31J) IBM extended characters (0xFA40 - 0xFC4B)
+ *
+ * Remaps IBM extended characters to their corresponding duplicate codes
+ * before conversion. These characters are also assigned in the NEC special
+ * characters (0x8740 - 0x879C), the NEC-selected IBM extended characters
+ * (0xED40 - 0xEEFC), and JIS X 0208.
+ *
+ * The first 28 characters (0xFA40 - 0xFA5B) have individual mappings defined
+ * in CP932_IBM_EXT_SYMBOL_MAP, and the rest are linearly mapped to 0xED40 - 0xEEEC.
+ */
+var CP932_IBM_EXT_SYMBOL_MAP = [
+  // 0xFA40 - 0xFA49 [ⅰ-ⅹ]
+  0xEEEF, 0xEEF0, 0xEEF1, 0xEEF2, 0xEEF3, 0xEEF4, 0xEEF5, 0xEEF6, 0xEEF7, 0xEEF8,
+  // 0xFA4A - 0xFA53 [Ⅰ-Ⅹ]
+  0x8754, 0x8755, 0x8756, 0x8757, 0x8758, 0x8759, 0x875A, 0x875B, 0x875C, 0x875D,
+  // 0xFA54 - 0xFA57 [￢￤＇＂]
+  0x81CA, 0xEEFA, 0xEEFB, 0xEEFC,
+  // 0xFA58 - 0xFA5B [㈱№℡∵]
+  0x878A, 0x8782, 0x8784, 0x81E6
+];
+
+// The number of IBM extended characters (0xFA40 - 0xFC4B)
+var CP932_IBM_EXT_LEN = 388;
+// The number of valid trail bytes per lead byte (0x40 - 0xFC, except 0x7F)
+var CP932_IBM_EXT_TRAIL_BYTES_LEN = 188;
+
+function remapCP932_IBMExt(b1, b2) {
+  var leadOffset = (b1 - 0xFA) * CP932_IBM_EXT_TRAIL_BYTES_LEN;
+  var trailIndex = b2 - (b2 < 0x7F ? 0x40 : 0x41);
+  var ibmExtIndex = leadOffset + trailIndex;
+
+  if (ibmExtIndex < 0 || ibmExtIndex >= CP932_IBM_EXT_LEN) {
+    return (b1 << 8) | b2;
+  }
+
+  if (ibmExtIndex < CP932_IBM_EXT_SYMBOL_MAP.length) {
+    return CP932_IBM_EXT_SYMBOL_MAP[ibmExtIndex];
+  }
+
+  var necSelectedIbmOffset = ibmExtIndex - CP932_IBM_EXT_SYMBOL_MAP.length;
+
+  // Remap to NEC-selected IBM extended characters (0xED40 - 0xEEEC)
+  b1 = 0xED;
+  if (necSelectedIbmOffset >= CP932_IBM_EXT_TRAIL_BYTES_LEN) {
+    necSelectedIbmOffset -= CP932_IBM_EXT_TRAIL_BYTES_LEN;
+    b1++;
+  }
+
+  b2 = necSelectedIbmOffset + 0x40;
+  if (b2 >= 0x7F) {
+    b2++; // Skip invalid trail byte 0x7F
+  }
+  return (b1 << 8) | b2;
+}
+
+/**
+ * Remapping duplicate codes for CP932 NEC special characters to their
+ * standard JIS X 0208 codes.
+ *
+ * This also includes 0xEEF9 in the NEC-selected IBM extended characters.
+ */
+var CP932_NEC_DUPLICATE_MAP = {
+  0x8790: 0x81E0, // ≒ (NEC special characters)
+  0x8791: 0x81DF, // ≡
+  0x8792: 0x81E7, // ∫
+  0x8795: 0x81E3, // √
+  0x8796: 0x81DB, // ⊥
+  0x8797: 0x81DA, // ∠
+  0x879A: 0x81E6, // ∵
+  0x879B: 0x81BF, // ∩
+  0x879C: 0x81BE, // ∪
+  0xEEF9: 0x81CA  // ￢ (NEC-selected IBM extended characters)
+};
+
+function remapCP932DuplicateCode(b1, b2) {
+  if (b2 < 0x40 || b2 > 0xFC || b2 === 0x7F) {
+    return (b1 << 8) | (b2 & 0xFF);
+  }
+
+  if (b1 >= 0xFA) {
+    return remapCP932_IBMExt(b1, b2);
+  }
+
+  var code = (b1 << 8) | b2;
+  var remapped = CP932_NEC_DUPLICATE_MAP[code];
+  return remapped == null ? code : remapped;
+}
+exports.remapCP932DuplicateCode = remapCP932DuplicateCode;
+
+function hasCP932DuplicateCode(b1) {
+  return b1 >= 0xFA || b1 === 0x87 || b1 === 0xEE;
+}
+exports.hasCP932DuplicateCode = hasCP932DuplicateCode;