diff options
Diffstat (limited to 'ext')
-rw-r--r-- | ext/nkf/nkf-utf8/nkf.c | 90 | ||||
-rw-r--r-- | ext/nkf/nkf-utf8/utf8tbl.c | 60 |
2 files changed, 131 insertions, 19 deletions
diff --git a/ext/nkf/nkf-utf8/nkf.c b/ext/nkf/nkf-utf8/nkf.c index aa7c459b83..458fbbc0e8 100644 --- a/ext/nkf/nkf-utf8/nkf.c +++ b/ext/nkf/nkf-utf8/nkf.c @@ -46,7 +46,7 @@ static char *CopyRight = static char *Version = "2.0"; static char *Patchlevel = - "4/0401/Shinji Kono"; + "4/0410/Shinji Kono"; /* ** @@ -198,7 +198,7 @@ static char *Patchlevel = #define UTF8 12 #define UTF8_INPUT 13 -#define UTF16_INPUT 14 +#define UTF16LE_INPUT 14 #define UTF16BE_INPUT 15 #define WISH_TRUE 15 @@ -376,8 +376,9 @@ static int x0201_f = NO_X0201; /* Assume NO JISX0201 */ #endif static int iso2022jp_f = FALSE; /* convert ISO-2022-JP */ #ifdef UTF8_OUTPUT_ENABLE -static int w_oconv16_begin_f= 0; /* utf-16 header */ +static int unicode_bom_f= 0; /* Output Unicode BOM */ static int w_oconv16_LE = 0; /* utf-16 little endian */ +static int ms_ucs_map_f = FALSE; /* Microsoft UCS Mapping Compatible */ #endif @@ -443,7 +444,7 @@ STATIC void s_status PROTO((struct input_code *, int)); #ifdef UTF8_INPUT_ENABLE STATIC void w_status PROTO((struct input_code *, int)); STATIC void w16_status PROTO((struct input_code *, int)); -static int utf16_mode = UTF16_INPUT; +static int utf16_mode = UTF16LE_INPUT; #endif struct input_code input_code_list[] = { @@ -892,6 +893,7 @@ struct { #ifdef UTF8_OUTPUT_ENABLE {"utf8", "w"}, {"utf16", "w16"}, + {"ms-ucs-map", ""}, #endif #ifdef UTF8_INPUT_ENABLE {"utf8-input", "W"}, @@ -1008,6 +1010,12 @@ options(cp) return; } #endif +#ifdef UTF8_OUTPUT_ENABLE + if (strcmp(long_option[i].name, "ms-ucs-map") == 0){ + ms_ucs_map_f = TRUE; + continue; + } +#endif if (strcmp(long_option[i].name, "prefix=") == 0){ if (*p == '=' && ' ' < p[1] && p[1] < 128){ for (i = 2; ' ' < p[i] && p[i] < 128; i++){ @@ -1082,17 +1090,23 @@ options(cp) if ('1'== cp[0] && '6'==cp[1]) { output_conv = w_oconv16; cp+=2; if (cp[0]=='L') { - w_oconv16_begin_f=2; cp++; + unicode_bom_f=2; cp++; w_oconv16_LE = 1; if (cp[0] == '0'){ - w_oconv16_begin_f=1; cp++; + unicode_bom_f=1; cp++; } } else if (cp[0] == 'B') { - w_oconv16_begin_f=2; cp++; + unicode_bom_f=2; cp++; if (cp[0] == '0'){ - w_oconv16_begin_f=1; cp++; + unicode_bom_f=1; cp++; } - } + } + } else if (cp[0] == '8') { + output_conv = w_oconv; cp++; + unicode_bom_f=2; + if (cp[0] == '0'){ + unicode_bom_f=1; cp++; + } } else output_conv = w_oconv; continue; @@ -1100,7 +1114,16 @@ options(cp) #ifdef UTF8_INPUT_ENABLE case 'W': /* UTF-8 input */ if ('1'== cp[0] && '6'==cp[1]) { - input_f = UTF16_INPUT; + input_f = UTF16LE_INPUT; + if (cp[0]=='L') { + cp++; + } else if (cp[0] == 'B') { + cp++; + input_f = UTF16BE_INPUT; + } + } else if (cp[0] == '8') { + cp++; + input_f = UTF8_INPUT; } else input_f = UTF8_INPUT; continue; @@ -1760,7 +1783,7 @@ module_connection() #ifdef UTF8_INPUT_ENABLE } else if (input_f == UTF8_INPUT) { set_iconv(-TRUE, w_iconv); - } else if (input_f == UTF16_INPUT) { + } else if (input_f == UTF16LE_INPUT) { set_iconv(-TRUE, w_iconv16); #endif } else { @@ -2364,7 +2387,7 @@ w_iconv16(c2, c1, c0) int ret; if (c2==0376 && c1==0377){ - utf16_mode = UTF16_INPUT; + utf16_mode = UTF16LE_INPUT; return 0; } else if (c2==0377 && c1==0376){ utf16_mode = UTF16BE_INPUT; @@ -2424,6 +2447,7 @@ e2w_conv(c2, c1) { extern unsigned short euc_to_utf8_1byte[]; extern unsigned short * euc_to_utf8_2bytes[]; + extern unsigned short * euc_to_utf8_2bytes_ms[]; unsigned short *p; if (c2 == X0201) { @@ -2432,7 +2456,7 @@ e2w_conv(c2, c1) c2 &= 0x7f; c2 = (c2&0x7f) - 0x21; if (0<=c2 && c2<sizeof_euc_to_utf8_2bytes) - p = euc_to_utf8_2bytes[c2]; + p = ms_ucs_map_f ? euc_to_utf8_2bytes_ms[c2] : euc_to_utf8_2bytes[c2]; else return 0; } @@ -2462,7 +2486,16 @@ w_oconv(c2, c1) if (c2 == EOF) { (*o_putc)(EOF); return; - } else if (c2 == 0) { + } + + if (unicode_bom_f==2) { + (*o_putc)('\357'); + (*o_putc)('\273'); + (*o_putc)('\277'); + unicode_bom_f=1; + } + + if (c2 == 0) { output_mode = ASCII; (*o_putc)(c1); } else if (c2 == ISO8859_1) { @@ -2489,7 +2522,7 @@ w_oconv16(c2, c1) return; } - if (w_oconv16_begin_f==2) { + if (unicode_bom_f==2) { if (w_oconv16_LE){ (*o_putc)((unsigned char)'\377'); (*o_putc)('\376'); @@ -2497,7 +2530,7 @@ w_oconv16(c2, c1) (*o_putc)('\376'); (*o_putc)((unsigned char)'\377'); } - w_oconv16_begin_f=1; + unicode_bom_f=1; } if (c2 == ISO8859_1) { @@ -3930,8 +3963,8 @@ reinit() } } #ifdef UTF8_OUTPUT_ENABLE - if (w_oconv16_begin_f) { - w_oconv16_begin_f = 2; + if (unicode_bom_f) { + unicode_bom_f = 2; } #endif f_line = 0; @@ -3990,7 +4023,13 @@ usage() #ifdef DEFAULT_CODE_UTF8 fprintf(stderr,"j,s,e,w Outout code is JIS 7 bit, Shift JIS, AT&T JIS (EUC), UTF-8 (DEFAULT)\n"); #endif +#ifdef UTF8_OUTPUT_ENABLE + fprintf(stderr," After 'w' you can add more options. (80?|16((B|L)0?)?) \n"); +#endif fprintf(stderr,"J,S,E,W Input assumption is JIS 7 bit , Shift JIS, AT&T JIS (EUC), UTF-8\n"); +#ifdef UTF8_INPUT_ENABLE + fprintf(stderr," After 'W' you can add more options. (8|16(B|L)?) \n"); +#endif fprintf(stderr,"t no conversion\n"); fprintf(stderr,"i_/o_ Output sequence to designate JIS-kanji/ASCII (DEFAULT B)\n"); fprintf(stderr,"r {de/en}crypt ROT13/47\n"); @@ -4012,8 +4051,21 @@ usage() fprintf(stderr,"I Convert non ISO-2022-JP charactor to GETA\n"); fprintf(stderr,"-L[uwm] line mode u:LF w:CRLF m:CR (DEFAULT noconversion)\n"); fprintf(stderr,"long name options\n"); - fprintf(stderr," --fj,--unix,--mac,--windows convert for the system\n"); + fprintf(stderr," --fj,--unix,--mac,--windows convert for the system\n"); fprintf(stderr," --jis,--euc,--sjis,--utf8,--utf16,--mime,--base64 convert for the code\n"); + fprintf(stderr," --hiragana, --katakana Hiragana/Katakana Conversion\n"); +#ifdef INPUT_OPTION + fprintf(stderr," --cap-input, --url-input Convert hex after ':' or '%'\n"); +#endif +#ifdef NUMCHAR_OPTION + fprintf(stderr," --numchar-input Convert Unicode Character Reference\n"); +#endif +#ifdef SHIFTJIS_CP932 + fprintf(stderr," --no-cp932 Don't convert Shift_JIS FAxx-FCxx to equivalnet CP932\n"); +#endif +#ifdef UTF8_OUTPUT_ENABLE + fprintf(stderr," --ms-ucs-map Microsoft UCS Mapping Compatible\n"); +#endif #ifdef OVERWRITE fprintf(stderr," --overwrite Overwrite original listed files by filtered result\n"); #endif diff --git a/ext/nkf/nkf-utf8/utf8tbl.c b/ext/nkf/nkf-utf8/utf8tbl.c index b2e49c49d2..7969081b4f 100644 --- a/ext/nkf/nkf-utf8/utf8tbl.c +++ b/ext/nkf/nkf-utf8/utf8tbl.c @@ -15,6 +15,23 @@ unsigned short euc_to_utf8_A1[] = { 0xFF04, 0x00A2, 0x00A3, 0xFF05, 0xFF03, 0xFF06, 0xFF0A, 0xFF20, 0x00A7, 0x2606, 0x2605, 0x25CB, 0x25CF, 0x25CE, 0x25C7, }; + +/* Microsoft UCS Mapping Compatible */ +unsigned short euc_to_utf8_A1_ms[] = { + 0x3000, 0x3001, 0x3002, 0xFF0C, 0xFF0E, 0x30FB, 0xFF1A, + 0xFF1B, 0xFF1F, 0xFF01, 0x309B, 0x309C, 0x00B4, 0xFF40, 0x00A8, + 0xFF3E, 0xFFE3, 0xFF3F, 0x30FD, 0x30FE, 0x309D, 0x309E, 0x3003, + 0x4EDD, 0x3005, 0x3006, 0x3007, 0x30FC, 0x2015, 0x2010, 0xFF0F, + 0xFF3C, 0xFF5E, 0x2225, 0xFF5C, 0x2026, 0x2025, 0x2018, 0x2019, + 0x201C, 0x201D, 0xFF08, 0xFF09, 0x3014, 0x3015, 0xFF3B, 0xFF3D, + 0xFF5B, 0xFF5D, 0x3008, 0x3009, 0x300A, 0x300B, 0x300C, 0x300D, + 0x300E, 0x300F, 0x3010, 0x3011, 0xFF0B, 0xFF0D, 0x00B1, 0x00D7, + 0x00F7, 0xFF1D, 0x2260, 0xFF1C, 0xFF1E, 0x2266, 0x2267, 0x221E, + 0x2234, 0x2642, 0x2640, 0x00B0, 0x2032, 0x2033, 0x2103, 0xFFE5, + 0xFF04, 0xFFE0, 0xFFE1, 0xFF05, 0xFF03, 0xFF06, 0xFF0A, 0xFF20, + 0x00A7, 0x2606, 0x2605, 0x25CB, 0x25CF, 0x25CE, 0x25C7, +}; + unsigned short euc_to_utf8_A2[] = { 0x25C6, 0x25A1, 0x25A0, 0x25B3, 0x25B2, 0x25BD, 0x25BC, 0x203B, 0x3012, 0x2192, 0x2190, 0x2191, 0x2193, 0x3013, 0, @@ -29,6 +46,22 @@ unsigned short euc_to_utf8_A2[] = { 0, 0, 0x212B, 0x2030, 0x266F, 0x266D, 0x266A, 0x2020, 0x2021, 0x00B6, 0, 0, 0, 0, 0x25EF, }; + +/* Microsoft UCS Mapping Compatible */ +unsigned short euc_to_utf8_A2_ms[] = { + 0x25C6, 0x25A1, 0x25A0, 0x25B3, 0x25B2, 0x25BD, 0x25BC, + 0x203B, 0x3012, 0x2192, 0x2190, 0x2191, 0x2193, 0x3013, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x2208, 0x220B, 0x2286, 0x2287, 0x2282, 0x2283, + 0x222A, 0x2229, 0, 0, 0, 0, 0, 0, + 0, 0, 0x2227, 0x2228, 0xFFE2, 0x21D2, 0x21D4, 0x2200, + 0x2203, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0x2220, 0x22A5, 0x2312, 0x2202, + 0x2207, 0x2261, 0x2252, 0x226A, 0x226B, 0x221A, 0x223D, 0x221D, + 0x2235, 0x222B, 0x222C, 0, 0, 0, 0, 0, + 0, 0, 0x212B, 0x2030, 0x266F, 0x266D, 0x266A, 0x2020, + 0x2021, 0x00B6, 0, 0, 0, 0, 0x25EF, +}; unsigned short euc_to_utf8_A3[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1287,6 +1320,33 @@ unsigned short * euc_to_utf8_2bytes[] = { 0, euc_to_utf8_F9, euc_to_utf8_FA, euc_to_utf8_FB, euc_to_utf8_FC, 0, 0, }; +/* Microsoft UCS Mapping Compatible */ +unsigned short * euc_to_utf8_2bytes_ms[] = { + euc_to_utf8_A1_ms,euc_to_utf8_A2_ms, euc_to_utf8_A3, + euc_to_utf8_A4, euc_to_utf8_A5, euc_to_utf8_A6, euc_to_utf8_A7, + euc_to_utf8_A8, euc_to_utf8_A9, euc_to_utf8_AA, euc_to_utf8_AB, + euc_to_utf8_AC, euc_to_utf8_AD, euc_to_utf8_AE, euc_to_utf8_AF, + euc_to_utf8_B0, euc_to_utf8_B1, euc_to_utf8_B2, euc_to_utf8_B3, + euc_to_utf8_B4, euc_to_utf8_B5, euc_to_utf8_B6, euc_to_utf8_B7, + euc_to_utf8_B8, euc_to_utf8_B9, euc_to_utf8_BA, euc_to_utf8_BB, + euc_to_utf8_BC, euc_to_utf8_BD, euc_to_utf8_BE, euc_to_utf8_BF, + euc_to_utf8_C0, euc_to_utf8_C1, euc_to_utf8_C2, euc_to_utf8_C3, + euc_to_utf8_C4, euc_to_utf8_C5, euc_to_utf8_C6, euc_to_utf8_C7, + euc_to_utf8_C8, euc_to_utf8_C9, euc_to_utf8_CA, euc_to_utf8_CB, + euc_to_utf8_CC, euc_to_utf8_CD, euc_to_utf8_CE, euc_to_utf8_CF, + euc_to_utf8_D0, euc_to_utf8_D1, euc_to_utf8_D2, euc_to_utf8_D3, + euc_to_utf8_D4, euc_to_utf8_D5, euc_to_utf8_D6, euc_to_utf8_D7, + euc_to_utf8_D8, euc_to_utf8_D9, euc_to_utf8_DA, euc_to_utf8_DB, + euc_to_utf8_DC, euc_to_utf8_DD, euc_to_utf8_DE, euc_to_utf8_DF, + euc_to_utf8_E0, euc_to_utf8_E1, euc_to_utf8_E2, euc_to_utf8_E3, + euc_to_utf8_E4, euc_to_utf8_E5, euc_to_utf8_E6, euc_to_utf8_E7, + euc_to_utf8_E8, euc_to_utf8_E9, euc_to_utf8_EA, euc_to_utf8_EB, + euc_to_utf8_EC, euc_to_utf8_ED, euc_to_utf8_EE, euc_to_utf8_EF, + euc_to_utf8_F0, euc_to_utf8_F1, euc_to_utf8_F2, euc_to_utf8_F3, + euc_to_utf8_F4, euc_to_utf8_F5, 0, 0, + 0, euc_to_utf8_F9, euc_to_utf8_FA, euc_to_utf8_FB, + euc_to_utf8_FC, 0, 0, +}; #endif /* UTF8_OUTPUT_ENABLE */ #ifdef UTF8_INPUT_ENABLE |