ruby/utf8.c

/**********************************************************************
  utf8.c -  Oniguruma (regular expression library)
**********************************************************************/
/*-
 * Copyright (c) 2002-2004  K.Kosako  <kosako AT sofnec DOT co DOT jp>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "regenc.h"

#define USE_INVALID_CODE_SCHEME

#ifdef USE_INVALID_CODE_SCHEME
/* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
#define INVALID_CODE_FE   0xfffffffe
#define INVALID_CODE_FF   0xffffffff
#define VALID_CODE_LIMIT  0x7fffffff
#endif

#define utf8_islead(c)     ((UChar )((c) & 0xc0) != 0x80)

static int EncLen_UTF8[] = {
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1
};

static int
utf8_mbc_enc_len(UChar* p)
{
  return EncLen_UTF8[*p];
}

static OnigCodePoint
utf8_mbc_to_code(UChar* p, UChar* end)
{
  int c, len;
  OnigCodePoint n;

  len = enc_len(ONIG_ENCODING_UTF8, p);
  c = *p++;
  if (len > 1) {
    len--;
    n = c & ((1 << (6 - len)) - 1);
    while (len--) {
      c = *p++;
      n = (n << 6) | (c & ((1 << 6) - 1));
    }
    return n;
  }
  else {
#ifdef USE_INVALID_CODE_SCHEME
    if (c > 0xfd) {
      return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
    }
#endif
    return (OnigCodePoint )c;
  }
}

static int
utf8_code_to_mbclen(OnigCodePoint code)
{
  if      ((code & 0xffffff80) == 0) return 1;
  else if ((code & 0xfffff800) == 0) {
    if (code <= 0xff && code >= 0xfe)
      return 1;
    return 2;
  }
  else if ((code & 0xffff0000) == 0) return 3;
  else if ((code & 0xffe00000) == 0) return 4;
  else if ((code & 0xfc000000) == 0) return 5;
  else if ((code & 0x80000000) == 0) return 6;
#ifdef USE_INVALID_CODE_SCHEME
  else if (code == INVALID_CODE_FE) return 1;
  else if (code == INVALID_CODE_FF) return 1;
#endif
  else
    return ONIGENCERR_TOO_BIG_WIDE_CHAR_VALUE;
}

#if 0
static int
utf8_code_to_mbc_first(OnigCodePoint code)
{
  if ((code & 0xffffff80) == 0)
    return code;
  else {
    if ((code & 0xfffff800) == 0)
      return ((code>>6)& 0x1f) | 0xc0;
    else if ((code & 0xffff0000) == 0)
      return ((code>>12) & 0x0f) | 0xe0;
    else if ((code & 0xffe00000) == 0)
      return ((code>>18) & 0x07) | 0xf0;
    else if ((code & 0xfc000000) == 0)
      return ((code>>24) & 0x03) | 0xf8;
    else if ((code & 0x80000000) == 0)
      return ((code>>30) & 0x01) | 0xfc;
    else {
      return ONIGENCERR_TOO_BIG_WIDE_CHAR_VALUE;
    }
  }
}
#endif

static int
utf8_code_to_mbc(OnigCodePoint code, UChar *buf)
{
#define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
#define UTF8_TRAIL0(code)        (UChar )(((code) & 0x3f) | 0x80)

  if ((code & 0xffffff80) == 0) {
    *buf = (UChar )code;
    return 1;
  }
  else {
    UChar *p = buf;

    if ((code & 0xfffff800) == 0) {
      *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
    }
    else if ((code & 0xffff0000) == 0) {
      *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
      *p++ = UTF8_TRAILS(code, 6);
    }
    else if ((code & 0xffe00000) == 0) {
      *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
      *p++ = UTF8_TRAILS(code, 12);
      *p++ = UTF8_TRAILS(code,  6);
    }
    else if ((code & 0xfc000000) == 0) {
      *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
      *p++ = UTF8_TRAILS(code, 18);
      *p++ = UTF8_TRAILS(code, 12);
      *p++ = UTF8_TRAILS(code,  6);
    }
    else if ((code & 0x80000000) == 0) {
      *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
      *p++ = UTF8_TRAILS(code, 24);
      *p++ = UTF8_TRAILS(code, 18);
      *p++ = UTF8_TRAILS(code, 12);
      *p++ = UTF8_TRAILS(code,  6);
    }
#ifdef USE_INVALID_CODE_SCHEME
    else if (code == INVALID_CODE_FE) {
      *p = 0xfe;
      return 1;
    }
    else if (code == INVALID_CODE_FF) {
      *p = 0xff;
      return 1;
    }
#endif
    else {
      return ONIGENCERR_TOO_BIG_WIDE_CHAR_VALUE;
    }

    *p++ = UTF8_TRAIL0(code);
    return p - buf;
  }
}

static int
utf8_mbc_to_normalize(OnigAmbigType flag, UChar** pp, UChar* end, UChar* lower)
{
  UChar* p = *pp;

  if (ONIGENC_IS_MBC_ASCII(p)) {
    if (end > p + 1 &&
        (flag & ONIGENC_AMBIGUOUS_MATCH_COMPOUND) != 0 &&
	((*p == 's' && *(p+1) == 's') ||
	 ((flag & ONIGENC_AMBIGUOUS_MATCH_ASCII_CASE) != 0 &&
	  (*p == 'S' && *(p+1) == 'S')))) {
      *lower++ = '\303';
      *lower   = '\237';
      (*pp) += 2;
      return 2;
    }

    if ((flag & ONIGENC_AMBIGUOUS_MATCH_ASCII_CASE) != 0) {
      *lower = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
    }
    else {
      *lower = *p;
    }
    (*pp)++;
    return 1; /* return byte length of converted char to lower */
  }
  else {
    int len;

    if (*p == 195) { /* 195 == '\303' */
      int c = *(p + 1);
      if (c >= 128) {
        if (c <= '\236' &&  /* upper */
            (flag & ONIGENC_AMBIGUOUS_MATCH_NONASCII_CASE) != 0) {
          if (c != '\227') {
            *lower++ = *p;
            *lower   = (UChar )(c + 32);
            (*pp) += 2;
            return 2;
          }
        }
#if 0
        else if (c == '\237' &&
                 (flag & ONIGENC_AMBIGUOUS_MATCH_COMPOUND) != 0) {
          *lower++ = '\303';
          *lower   = '\237';
          (*pp) += 2;
          return 2;
        }
#endif
      }
    }

    len = enc_len(ONIG_ENCODING_UTF8, p);
    if (lower != p) {
      int i;
      for (i = 0; i < len; i++) {
	*lower++ = *p++;
      }
    }
    (*pp) += len;
    return len; /* return byte length of converted char to lower */
  }
}

static int
utf8_is_mbc_ambiguous(OnigAmbigType flag, UChar** pp, UChar* end)
{
  UChar* p = *pp;

  if (ONIGENC_IS_MBC_ASCII(p)) {
    if (end > p + 1 &&
        (flag & ONIGENC_AMBIGUOUS_MATCH_COMPOUND) != 0 &&
	((*p == 's' && *(p+1) == 's') ||
	 ((flag & ONIGENC_AMBIGUOUS_MATCH_ASCII_CASE) != 0 &&
	  (*p == 'S' && *(p+1) == 'S')))) {
      (*pp) += 2;
      return TRUE;
    }

    (*pp)++;
    if ((flag & ONIGENC_AMBIGUOUS_MATCH_ASCII_CASE) != 0) {
      return ONIGENC_IS_ASCII_CODE_CASE_AMBIG(*p);
    }
  }
  else {
    (*pp) += enc_len(ONIG_ENCODING_UTF8, p);

    if (*p == 195) { /* 195 == '\303' */
      int c = *(p + 1);
      if (c >= 128) {
        if ((flag & ONIGENC_AMBIGUOUS_MATCH_NONASCII_CASE) != 0) {
          if (c <= '\236') { /* upper */
            if (c == '\227') return FALSE;
            return TRUE;
          }
          else if (c >= '\240' && c <= '\276') { /* lower */
            if (c == '\267') return FALSE;
            return TRUE;
          }
        }
        else if (c == '\237' &&
                 (flag & ONIGENC_AMBIGUOUS_MATCH_COMPOUND) != 0) {
	  return TRUE;
        }
      }
    }
  }

  return FALSE;
}

static int
utf8_is_code_ctype(OnigCodePoint code, unsigned int ctype)
{
  if (code < 256) {
    return ONIGENC_IS_UNICODE_ISO_8859_1_CTYPE(code, ctype);
  }

  if ((ctype & ONIGENC_CTYPE_WORD) != 0) {
#ifdef USE_INVALID_CODE_SCHEME
    if (code <= VALID_CODE_LIMIT)
#endif
      return TRUE;
  }

  return FALSE;
}

static int
utf8_get_ctype_code_range(int ctype, int* nsb, int* nmb,
			  OnigCodePointRange* sbr[], OnigCodePointRange* mbr[])
{
#define CR_SET(sbl,mbl) do { \
  *nsb = sizeof(sbl) / sizeof(OnigCodePointRange); \
  *nmb = sizeof(mbl) / sizeof(OnigCodePointRange); \
  *sbr = sbl; \
  *mbr = mbl; \
} while (0)

#define CR_SB_SET(sbl) do { \
  *nsb = sizeof(sbl) / sizeof(OnigCodePointRange); \
  *nmb = 0; \
  *sbr = sbl; \
} while (0)

  static OnigCodePointRange SBAlpha[] = {
    { 0x41, 0x5a }, { 0x61, 0x7a }
  };

  static OnigCodePointRange MBAlpha[] = {
    { 0xaa, 0xaa }, { 0xb5, 0xb5 },
    { 0xba, 0xba }, { 0xc0, 0xd6 },
    { 0xd8, 0xf6 }, { 0xf8, 0x220 }
  };

  static OnigCodePointRange SBBlank[] = {
    { 0x09, 0x09 }, { 0x20, 0x20 }
  };

  static OnigCodePointRange MBBlank[] = {
    { 0xa0, 0xa0 }
  };

  static OnigCodePointRange SBCntrl[] = {
    { 0x00, 0x1f }, { 0x7f, 0x7f }
  };

  static OnigCodePointRange MBCntrl[] = {
    { 0x80, 0x9f }
  };

  static OnigCodePointRange SBDigit[] = {
    { 0x30, 0x39 }
  };

  static OnigCodePointRange SBGraph[] = {
    { 0x21, 0x7e }
  };

  static OnigCodePointRange MBGraph[] = {
    { 0xa1, 0x220 }
  };

  static OnigCodePointRange SBLower[] = {
    { 0x61, 0x7a }
  };

  static OnigCodePointRange MBLower[] = {
    { 0xaa, 0xaa }, { 0xb5, 0xb5 },
    { 0xba, 0xba }, { 0xdf, 0xf6 },
    { 0xf8, 0xff }
  };

  static OnigCodePointRange SBPrint[] = {
    { 0x20, 0x7e }
  };

  static OnigCodePointRange MBPrint[] = {
    { 0xa0, 0x220 }
  };

  static OnigCodePointRange SBPunct[] = {
    { 0x21, 0x23 }, { 0x25, 0x2a },
    { 0x2c, 0x2f }, { 0x3a, 0x3b },
    { 0x3f, 0x40 }, { 0x5b, 0x5d },
    { 0x5f, 0x5f }, { 0x7b, 0x7b },
    { 0x7d, 0x7d }
  };

  static OnigCodePointRange MBPunct[] = {
    { 0xa1, 0xa1 }, { 0xab, 0xab },
    { 0xad, 0xad }, { 0xb7, 0xb7 },
    { 0xbb, 0xbb }, { 0xbf, 0xbf }
  };

  static OnigCodePointRange SBSpace[] = {
    { 0x09, 0x0d }, { 0x20, 0x20 }
  };

  static OnigCodePointRange MBSpace[] = {
    { 0xa0, 0xa0 }
  };

  static OnigCodePointRange SBUpper[] = {
    { 0x41, 0x5a }
  };

  static OnigCodePointRange MBUpper[] = {
    { 0xc0, 0xd6 }, { 0xd8, 0xde }
  };

  static OnigCodePointRange SBXDigit[] = {
    { 0x30, 0x39 }, { 0x41, 0x46 },
    { 0x61, 0x66 }
  };

  static OnigCodePointRange SBWord[] = {
    { 0x30, 0x39 }, { 0x41, 0x5a },
    { 0x5f, 0x5f }, { 0x61, 0x7a }
  };

  static OnigCodePointRange MBWord[] = {
    { 0xaa, 0xaa }, { 0xb2, 0xb3 },
    { 0xb5, 0xb5 }, { 0xb9, 0xba },
    { 0xbc, 0xbe }, { 0xc0, 0xd6 },
    { 0xd8, 0xf6 },
#if 0
    { 0xf8, 0x220 }
#else
    { 0xf8, 0x7fffffff } /* all multibyte code as word */
#endif
  };

  static OnigCodePointRange SBAscii[] = {
    { 0x00, 0x7f }
  };

  static OnigCodePointRange SBAlnum[] = {
    { 0x30, 0x39 }, { 0x41, 0x5a },
    { 0x61, 0x7a }
  };

  static OnigCodePointRange MBAlnum[] = {
    { 0xaa, 0xaa }, { 0xb5, 0xb5 },
    { 0xba, 0xba }, { 0xc0, 0xd6 },
    { 0xd8, 0xf6 }, { 0xf8, 0x220 }
  };

  switch (ctype) {
  case ONIGENC_CTYPE_ALPHA:
    CR_SET(SBAlpha, MBAlpha);
    break;
  case ONIGENC_CTYPE_BLANK:
    CR_SET(SBBlank, MBBlank);
    break;
  case ONIGENC_CTYPE_CNTRL:
    CR_SET(SBCntrl, MBCntrl);
    break;
  case ONIGENC_CTYPE_DIGIT:
    CR_SB_SET(SBDigit);
    break;
  case ONIGENC_CTYPE_GRAPH:
    CR_SET(SBGraph, MBGraph);
    break;
  case ONIGENC_CTYPE_LOWER:
    CR_SET(SBLower, MBLower);
    break;
  case ONIGENC_CTYPE_PRINT:
    CR_SET(SBPrint, MBPrint);
    break;
  case ONIGENC_CTYPE_PUNCT:
    CR_SET(SBPunct, MBPunct);
    break;
  case ONIGENC_CTYPE_SPACE:
    CR_SET(SBSpace, MBSpace);
    break;
  case ONIGENC_CTYPE_UPPER:
    CR_SET(SBUpper, MBUpper);
    break;
  case ONIGENC_CTYPE_XDIGIT:
    CR_SB_SET(SBXDigit);
    break;
  case ONIGENC_CTYPE_WORD:
    CR_SET(SBWord, MBWord);
    break;
  case ONIGENC_CTYPE_ASCII:
    CR_SB_SET(SBAscii);
    break;
  case ONIGENC_CTYPE_ALNUM:
    CR_SET(SBAlnum, MBAlnum);
    break;

  default:
    return ONIGENCERR_TYPE_BUG;
    break;
  }

  return 0;
}

static UChar*
utf8_left_adjust_char_head(UChar* start, UChar* s)
{
  UChar *p;

  if (s <= start) return s;
  p = s;

  while (!utf8_islead(*p) && p > start) p--;
  return p;
}

OnigEncodingType OnigEncodingUTF8 = {
  utf8_mbc_enc_len,
  "UTF-8",     /* name */
  6,           /* max byte length */
  1,           /* min byte length */
  (ONIGENC_AMBIGUOUS_MATCH_ASCII_CASE | 
   ONIGENC_AMBIGUOUS_MATCH_NONASCII_CASE | 
   ONIGENC_AMBIGUOUS_MATCH_COMPOUND),
  {
      (OnigCodePoint )'\\'                       /* esc */
    , (OnigCodePoint )ONIG_INEFFECTIVE_META_CHAR /* anychar '.'  */
    , (OnigCodePoint )ONIG_INEFFECTIVE_META_CHAR /* anytime '*'  */
    , (OnigCodePoint )ONIG_INEFFECTIVE_META_CHAR /* zero or one time '?' */
    , (OnigCodePoint )ONIG_INEFFECTIVE_META_CHAR /* one or more time '+' */
    , (OnigCodePoint )ONIG_INEFFECTIVE_META_CHAR /* anychar anytime */
  },
  onigenc_is_mbc_newline_0x0a,
  utf8_mbc_to_code,
  utf8_code_to_mbclen,
  utf8_code_to_mbc,
  utf8_mbc_to_normalize,
  utf8_is_mbc_ambiguous,
  onigenc_iso_8859_1_get_all_pair_ambig_codes,
  onigenc_ess_tsett_get_all_comp_ambig_codes,
  utf8_is_code_ctype,
  utf8_get_ctype_code_range,
  utf8_left_adjust_char_head,
  onigenc_always_true_is_allowed_reverse_match
};