Dogcows Code - chaz/tar/blob - src/utf8.c

   1 /* Charset handling for GNU tar.
   2
   3    Copyright (C) 2004 Free Software Foundation, Inc.
   4
   5    This program is free software; you can redistribute it and/or modify it
   6    under the terms of the GNU General Public License as published by the
   7    Free Software Foundation; either version 2, or (at your option) any later
   8    version.
   9
  10    This program is distributed in the hope that it will be useful, but
  11    WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
  13    Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License along
  16    with this program; if not, write to the Free Software Foundation, Inc.,
  17    59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
  18
  19 #include "system.h"
  20 #include <quotearg.h>
  21 #include "common.h"
  22 #ifdef HAVE_ICONV_H
  23 # include <iconv.h>
  24 #endif
  25
  26 struct langtab
  27 {
  28   char *lang;        /* Language code */
  29   char *terr;        /* Territory code */
  30   char *charset;     /* Corresponding charset */
  31 };
  32
  33 /* The list of language codes defined in ISO 639 with the corresponding
  34    default character sets.
  35
  36    NOTES:
  37
  38    1) The list must be ordered by:
  39         a) lang field in ascending order
  40         b) terr field in descending order.
  41       NULL fields are considered less than non-null ones.
  42    2) Many entries have NULL charset fields. Please help fill them!
  43    3) The "default" character set for a given language is a matter
  44       of preference. Possibly the table should contain a *list* of
  45       possible character sets.
  46    4) LC_ALL "modifier" field is not taken into account */
  47
  48 static struct langtab langtab[] = {
  49   { "C",     NULL, "ASCII"},
  50   { "POSIX", NULL, "ASCII" },
  51   { "aa",    NULL, NULL},              /* Afar */
  52   { "ab",    NULL, NULL},              /* Abkhazian */
  53   { "ae",    NULL, NULL},              /* Avestan */
  54   { "af",    NULL, "iso-8859-1"},      /* Afrikaans */
  55   { "am",    NULL, "UTF-8"},           /* Amharic */
  56   { "ar",    NULL, "iso-8859-6"},      /* Arabic */
  57   { "as",    NULL, NULL},              /* Assamese */
  58   { "ay",    NULL, "iso-8859-1"},      /* Aymara */
  59   { "az",    NULL, NULL},              /* Azerbaijani */
  60   { "ba",    NULL, NULL},              /* Bashkir */
  61   { "be",    NULL, "UTF-8"},           /* Byelorussian; Belarusian */
  62   { "bg",    NULL, "iso-8859-5"},      /* Bulgarian */
  63   { "bh",    NULL, NULL},              /* Bihari */
  64   { "bi",    NULL, NULL},              /* Bislama */
  65   { "bn",    NULL, NULL},              /* Bengali; Bangla */
  66   { "bo",    NULL, NULL},              /* Tibetan */
  67   { "br",    NULL, "iso-8859-1"},      /* Breton: 1,5,8,9 */
  68   { "bs",    NULL, NULL},              /* Bosnian */
  69   { "ca",    NULL, "iso-8859-1"},      /* Catalan: 1,5,8,9 */
  70   { "ce",    NULL, NULL},              /* Chechen */
  71   { "ch",    NULL, NULL},              /* Chamorro */
  72   { "co",    NULL, "iso-8859-1"},      /* Corsican */
  73   { "cs",    NULL, "iso-8859-2"},      /* Czech */
  74   { "cu",    NULL, NULL },             /* Church Slavic */
  75   { "cv",    NULL, NULL},              /* Chuvash */
  76   { "cy",    NULL, "iso-8859-1"},      /* Welsh */
  77   { "da",    NULL, "iso-8859-1"},      /* Danish: 4-9 */
  78   { "de",    NULL, "iso-8859-1"},      /* German */
  79   { "dz",    NULL, NULL },             /* Dzongkha; Bhutani */
  80   { "el",    NULL, "iso-8859-7"},      /* Greek */
  81   { "en",    NULL, "iso-8859-1"},      /* English */
  82   { "eo",    NULL, "iso-8859-3"},      /* Esperanto */
  83   { "es",    NULL, "iso-8859-1"},      /* Spanish */
  84   { "et",    NULL, "iso-8859-15"},     /* Estonian: 6,7,9 */
  85   { "eu",    NULL, "iso-8859-1"},      /* Basque: 5,8,9 */
  86   { "fa",    NULL, "UTF-8"},           /* Persian */
  87   { "fi",    NULL, "iso-8859-15"},     /* Finnish */
  88   { "fj",    NULL, NULL },             /* Fijian; Fiji */
  89   { "fo",    NULL, "iso-8859-1"},      /* Faroese: 6,9 */
  90   { "fr",    NULL, "iso-8859-1"},      /* French */
  91   { "fy",    NULL, "iso-8859-1"},      /* Frisian */
  92   { "ga",    NULL, "iso-8859-14"},     /* Irish */
  93   { "gd",    NULL, "iso-8859-14" },    /* Scots; Gaelic */
  94   { "gl",    NULL, NULL },             /* Gallegan; Galician */
  95   { "gn",    NULL, NULL},              /* Guarani */
  96   { "gu",    NULL, NULL},              /* Gujarati */
  97   { "gv",    NULL, "iso-8859-14"},     /* Manx */
  98   { "ha",    NULL, NULL },             /* Hausa (?) */
  99   { "he",    NULL, "iso-8859-8" },     /* Hebrew */
 100   { "hi",    NULL, NULL},              /* Hindi */
 101   { "ho",    NULL, NULL},              /* Hiri Motu */
 102   { "hr",    NULL, "iso-8859-2"},      /* Croatian: 10 */
 103   { "hu",    NULL, "iso-8859-2"},      /* Hungarian */
 104   { "hy",    NULL, NULL},              /* Armenian */
 105   { "hz",    NULL, NULL},              /* Herero */
 106   { "id",    NULL, "iso-8859-1"},      /* Indonesian (formerly in) */
 107   { "ia",    NULL, NULL},              /* Interlingua */
 108   { "ie",    NULL, NULL},              /* Interlingue */
 109   { "ik",    NULL, NULL},              /* Inupiak */
 110   { "io",    NULL, NULL},              /* Ido */
 111   { "is",    NULL, "iso-8859-1"},      /* Icelandic */
 112   { "it",    NULL, "iso-8859-1"},      /* Italian */
 113   { "iu",    NULL, NULL},              /* Inuktitut */
 114   { "ja",    NULL, "EUC-JP"},          /* Japanese */
 115   { "jv",    NULL, NULL},              /* Javanese */
 116   { "ka",    NULL, NULL},              /* Georgian */
 117   { "ki",    NULL, NULL},              /* Kikuyu */
 118   { "kj",    NULL, NULL},              /* Kuanyama */
 119   { "kk",    NULL, NULL},              /* Kazakh */
 120   { "kl",    NULL, "iso-8859-1"},      /* Kalaallisut; Greenlandic */
 121   { "km",    NULL, NULL},              /* Khmer; Cambodian */
 122   { "kn",    NULL, NULL},              /* Kannada */
 123   { "ko",    NULL, "EUC-KR"},          /* Korean */
 124   { "ks",    NULL, NULL},              /* Kashmiri */
 125   { "ku",    NULL, NULL},              /* Kurdish */
 126   { "kv",    NULL, NULL},              /* Komi */
 127   { "kw",    NULL, "iso-8859-14"},     /* Cornish: 1,5,8 */
 128   { "ky",    NULL, NULL},              /* Kirghiz */
 129   { "la",    NULL, "iso-8859-1"},      /* Latin */
 130   { "lb",    NULL, "iso-8859-1"},      /* Letzeburgesch */
 131   { "ln",    NULL, NULL},              /* Lingala */
 132   { "lo",    NULL, NULL},              /* Lao; Laotian */
 133   { "lt",    NULL, "iso-8859-4"},      /* Lithuanian */
 134   { "lv",    NULL, "iso-8859-4"},      /* Latvian; Lettish */
 135   { "mg",    NULL, NULL},              /* Malagasy */
 136   { "mh",    NULL, NULL},              /* Marshall */
 137   { "mi",    NULL, NULL},              /* Maori */
 138   { "mk",    NULL, NULL},              /* Macedonian */
 139   { "ml",    NULL, NULL},              /* Malayalam */
 140   { "mn",    NULL, NULL},              /* Mongolian */
 141   { "mo",    NULL, "iso-8859-2"},      /* Moldavian */
 142   { "mr",    NULL, NULL},              /* Marathi */
 143   { "ms",    NULL, NULL},              /* Malay */
 144   { "mt",    NULL, "iso-8859-3"},      /* Maltese */
 145   { "my",    NULL, NULL},              /* Burmese */
 146   { "na",    NULL, NULL},              /* Nauru */
 147   { "nb",    NULL, "iso-8859-1"},      /* Norwegian Bokmål; Bokm@aa{}l  */
 148   { "nd",    NULL, NULL},              /* Ndebele, North */
 149   { "ne",    NULL, NULL},              /* Nepali */
 150   { "ng",    NULL, NULL},              /* Ndonga */
 151   { "nl",    NULL, "iso-8859-1"},      /* Dutch: 5,9 */
 152   { "nn",    NULL, "iso-8859-1"},      /* Norwegian Nynorsk */
 153   { "no",    NULL, "iso-8859-1"},      /* Norwegian */
 154   { "nr",    NULL, NULL},              /* Ndebele, South */
 155   { "nv",    NULL, NULL},              /* Navajo */
 156   { "ny",    NULL, NULL},              /* Chichewa; Nyanja */
 157   { "oc",    NULL, NULL},              /* Occitan; Provençal; Proven@,{c}al */
 158   { "om",    NULL, NULL},              /* (Afan) Oromo */
 159   { "or",    NULL, NULL},              /* Oriya */
 160   { "os",    NULL, NULL},              /* Ossetian; Ossetic */
 161   { "pa",    NULL, NULL},              /* Panjabi; Punjabi */
 162   { "pi",    NULL, NULL},              /* Pali */
 163   { "pl",    NULL, "iso-8859-2"},      /* Polish */
 164   { "ps",    NULL, NULL},              /* Pashto, Pushto */
 165   { "pt",    NULL, "iso-8859-1"},      /* Portuguese */
 166   { "qu",    NULL, "iso-8859-1"},      /* Quechua */
 167   { "rm",    NULL, "iso-8859-1"},      /* Rhaeto-Romance */
 168   { "rn",    NULL, NULL },             /* Rundi; Kirundi */
 169   { "ro",    NULL, "iso-8859-2"},      /* Romanian */
 170   { "ru",    NULL, "koi8-r"},          /* Russian */
 171   { "rw",    NULL, NULL},              /* Kinyarwanda */
 172   { "sa",    NULL, NULL},              /* Sanskrit */
 173   { "sc",    NULL, "iso-8859-1"},      /* Sardinian */
 174   { "sd",    NULL, NULL},              /* Sindhi */
 175   { "se",    NULL, "iso-8859-10"},     /* Northern Sami */
 176   { "sg",    NULL, NULL},              /* Sango; Sangro */
 177   { "si",    NULL, NULL},              /* Sinhalese */
 178   { "sk",    NULL, "iso-8859-2"},      /* Slovak */
 179   { "sl",    NULL, "iso-8859-1"},      /* Slovenian */
 180   { "sm",    NULL, NULL},              /* Samoan */
 181   { "sn",    NULL, NULL},              /* Shona */
 182   { "so",    NULL, NULL},              /* Somali */
 183   { "sq",    NULL, "iso-8859-1"},      /* Albanian: 2,5,8,9,10 */
 184   { "sr",    NULL, "iso-8859-2"},      /* Serbian */
 185   { "ss",    NULL, NULL},              /* Swati; Siswati */
 186   { "st",    NULL, NULL},              /* Sesotho; Sotho, Southern */
 187   { "su",    NULL, NULL},              /* Sundanese */
 188   { "sv",    NULL, "iso-8859-1"},      /* Swedish */
 189   { "sw",    NULL, NULL},              /* Swahili */
 190   { "ta",    NULL, NULL},              /* Tamil */
 191   { "te",    NULL, NULL},              /* Telugu */
 192   { "tg",    NULL, NULL},              /* Tajik */
 193   { "th",    NULL, "iso-8859-11"},     /* Thai */
 194   { "ti",    NULL, NULL},              /* Tigrinya */
 195   { "tk",    NULL, NULL},              /* Turkmen */
 196   { "tl",    NULL, "iso-8859-1"},      /* Tagalog */
 197   { "tn",    NULL, NULL},              /* Tswana; Setswana */
 198   { "to",    NULL, NULL},              /* Tonga (?) */
 199   { "tr",    NULL, "iso-8859-9"},      /* Turkish */
 200   { "ts",    NULL, NULL},              /* Tsonga */
 201   { "tt",    NULL, NULL},              /* Tatar */
 202   { "tw",    NULL, NULL},              /* Twi */
 203   { "ty",    NULL, NULL},              /* Tahitian */
 204   { "ug",    NULL, NULL},              /* Uighur */
 205   { "uk",    NULL, "koi8-u"},          /* Ukrainian */
 206   { "ur",    NULL, NULL},              /* Urdu */
 207   { "uz",    NULL, NULL},              /* Uzbek */
 208   { "vi",    NULL, NULL},              /* Vietnamese */
 209   { "vo",    NULL, NULL},              /* Volapük; Volap@"{u}k; Volapuk */
 210   { "wa",    NULL, "iso-8859-1"},      /* Walloon */
 211   { "wo",    NULL, NULL},              /* Wolof */
 212   { "xh",    NULL, NULL},              /* Xhosa */
 213   { "yi",    NULL, "iso-8859-8"},      /* Yiddish (formerly ji) */
 214   { "yo",    NULL, NULL},              /* Yoruba */
 215   { "za",    NULL, NULL},              /* Zhuang */
 216   { "zh",    "TW", "big5"},            /* Chinese */
 217   { "zh",    NULL, "gb2312"},          /* Chinese */
 218   { "zu",    NULL, NULL},              /* Zulu */
 219   { NULL }
 220 };
 221
 222 /* Given the language and (optionally) territory code, return the
 223    default character set for that language. See notes above. */
 224
 225 const char *
 226 charset_lookup (char *lang, char *terr)
 227 {
 228   static struct langtab *p;
 229
 230   if (!lang)
 231     return NULL;
 232   for (p = langtab; p->lang; p++)
 233     if (strcasecmp (p->lang, lang) == 0
 234         && (terr == NULL
 235             || p->terr == NULL
 236             || !strcasecmp (p->terr, terr) == 0))
 237       return p->charset;
 238   return NULL;
 239 }
 240
 241 static const char *
 242 get_input_charset ()
 243 {
 244   const char *charset = NULL;
 245   char *tmp;
 246
 247   /* Try to deduce the charset from LC_ALL or LANG variables */
 248
 249   tmp = getenv ("LC_ALL");
 250   if (!tmp)
 251     tmp = getenv ("LANG");
 252
 253   if (tmp)
 254     {
 255       char *lang;
 256       char *terr;
 257
 258       lang = strtok (tmp, "_");
 259       terr = strtok (NULL, ".");
 260       charset = strtok (NULL, "@");
 261
 262       if (!charset)
 263         charset = charset_lookup (lang, terr);
 264     }
 265
 266   if (!charset)
 267     charset = "iso-8859-1";
 268   return charset;
 269 }
 270
 271
 272 \f
 273 #ifndef HAVE_LIBICONV
 274
 275 iconv_t
 276 iconv_open (const char *tocode, const char *fromcode)
 277 {
 278   return (iconv_t)(-1);
 279 }
 280
 281 size_t
 282 iconv (iconv_t cd, ICONV_CONST char **inbuf, size_t *inbytesleft,
 283        char **outbuf, size_t *outbytesleft)
 284 {
 285   return 0;
 286 }
 287
 288 int
 289 iconv_close (iconv_t cd)
 290 {
 291   return 0;
 292 }
 293
 294 #endif /* !HAVE_LIBICONV */
 295
 296
 297 \f
 298
 299 static iconv_t conv_desc[2] = { (iconv_t) -1, (iconv_t) -1 };
 300
 301 static iconv_t
 302 utf8_init (bool to_utf)
 303 {
 304   if (conv_desc[(int) to_utf] == (iconv_t) -1)
 305     {
 306       if (to_utf)
 307         conv_desc[(int) to_utf] = iconv_open ("UTF-8", get_input_charset ());
 308       else
 309         conv_desc[(int) to_utf] = iconv_open (get_input_charset (), "UTF-8");
 310     }
 311   return conv_desc[(int) to_utf];
 312 }
 313
 314 bool
 315 utf8_convert(bool to_utf, const char *input, char **output)
 316 {
 317   const char *ib;
 318   char *ob;
 319   size_t inlen;
 320   size_t outlen;
 321   size_t rc;
 322   iconv_t cd = utf8_init (to_utf);
 323
 324   if (cd == 0)
 325     {
 326       *output = xstrdup (input);
 327       return true;
 328     }
 329   else if (cd == (iconv_t)-1)
 330     return false;
 331
 332   inlen = strlen (input) + 1;
 333   outlen = inlen * MB_LEN_MAX + 1;
 334   ob = *output = xmalloc (outlen);
 335   ib = input;
 336   rc = iconv (cd, &ib, &inlen, &ob, &outlen);
 337   *ob = 0;
 338   return rc != -1;
 339 }
 340 \f
 341
 342 bool
 343 string_ascii_p (const char *str)
 344 {
 345   const unsigned char *p = (const unsigned char *)str;
 346   for (; *p; p++)
 347     if (*p > 127)
 348       return false;
 349   return true;
 350 }