Message/MIME/Charset.pm


=head1 NAME

Message::MIME::Charset Perl module

=head1 DESCRIPTION

Perl module for MIME charset.

=cut

## NOTE: You should not require/use other module (even it
##       is part of Message::* Perl Modules) as far as possible,
##       to be able to use this module (M::M::Charset) from
##       other (non-Message::*) modules.

package Message::MIME::Charset;
use strict;
use vars qw(%CHARSET %REG $VERSION);
$VERSION=do{my @r=(q$Revision: 1.9 $=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};

&_builtin_charset;
sub _builtin_charset () {

$CHARSET{'*DEFAULT'} = {
        preferred_name  => '',
        
        encoder => sub { $_[1] },
        decoder => sub { $_[1] },
        
        mime_text       => 1,   ## Suitability in use as MIME text/* charset
        #accept_cte     => [qw/7bit .../],
        cte_7bit_preferred      => 'quoted-printable',
};
$CHARSET{'*default'} = $CHARSET{'*DEFAULT'};

$CHARSET{'us-ascii'} = {
        preferred_name  => 'us-ascii',
        
        encoder => sub { $_[1] },
        decoder => sub { $_[1] },
        
        mime_text       => 1,
};

$CHARSET{'iso-2022-int-1'} = {
        preferred_name  => 'iso-2022-int-1',
        
        encoder => sub { $_[1] },
        decoder => sub { $_[1] },
        
        mime_text       => 1,
};

$CHARSET{'unknown-8bit'} = {
        preferred_name  => 'unknown-8bit',
        
        encoder => sub { $_[1] },
        decoder => sub { $_[1] },
        
        mime_text       => 0,
        cte_7bit_preferred      => 'base64',
};
$CHARSET{'x-unknown'} = $CHARSET{'unknown-8bit'};

}       # /builtin_charset

my %_MINIMUMIZER = (
        'euc-jp'        => \&_name_euc_japan,
        'euc-jisx0213'  => \&_name_euc_japan,
        'euc-jisx0213-plane1'   => \&_name_euc_japan,
        'x-euc-jisx0213-packed' => \&_name_euc_japan,
        'x-iso-2022'    => \&_name_8bit_iso_2022,
        'iso-2022-cn'   => \&_name_8bit_iso_2022,
        'iso-2022-cn-ext'       => \&_name_8bit_iso_2022,
        'iso-2022-int-1'        => \&_name_net_ascii_8bit,
        'iso-2022-jp'   => \&_name_8bit_iso_2022,
        'iso-2022-jp-1' => \&_name_8bit_iso_2022,
        'iso-2022-jp-2' => \&_name_8bit_iso_2022,
        'iso-2022-jp-3' => \&_name_8bit_iso_2022,
        'iso-2022-jp-3-plane1'  => \&_name_8bit_iso_2022,
        'iso-2022-kr'   => \&_name_8bit_iso_2022,
        'iso-8859-1'    => \&_name_8bit_iso_2022,
        'iso-10646-j-1' => \&_name_utf16be,
        'iso-10646-ucs-2'       => \&_name_utf16be,
        'iso-10646-ucs-4'       => \&_name_utf32be,
        'iso-10646-ucs-basic'   => \&_name_utf16be,
        'iso-10646-unicode-latin1'      => \&_name_utf16be,
        jis_x0201       => \&_name_shift_jis,
        junet   => \&_name_8bit_iso_2022,
        'x-junet8'      => \&_name_net_ascii_8bit,
        shift_jis       => \&_name_shift_jis,
        shift_jisx0213  => \&_name_shift_jis,
        'shift_jisx0213-plane1' => \&_name_shift_jis,
        'x-sjis'        => \&_name_shift_jis,
        'us-ascii'      => \&_name_net_ascii_8bit,
        'utf-8' => \&_name_net_ascii_8bit,
        'utf-16be'      => \&_name_utf16be,
        'utf-32be'      => \&_name_utf32be,
);

sub make_charset ($%) {
  my $name = shift;
  return unless $name;  ## Note: charset "0" is not supported.
  my %definition = @_;
  
  $definition{preferred_name} ||= $name;
  if ($definition{preferred_name} ne $name
      && ref $CHARSET{$definition{preferred_name}}) {
  ## New charset is an alias of defined charset,
    $CHARSET{$name} = $CHARSET{$definition{preferred_name}};
    return;
  } elsif ($definition{alias_of} && ref $CHARSET{$definition{alias_of}}) {
  ## New charset is an alias of defined charset,
    $CHARSET{$name} = $CHARSET{$definition{alias_of}};
    return;
  }
  $CHARSET{$name} = \%definition;
  
  ## Set default values
  #$definition{encoder} ||= sub { $_[1] };
  #$definition{decoder} ||= sub { $_[1] };

  $definition{mime_text} = 0 unless defined $definition{mime_text};
  $definition{cte_7bit_preferred} = 'base64'
    unless defined $definition{cte_7bit_preferred};
}

sub encode ($$) {
  my ($charset, $s) = (lc shift, shift);
  if (ref $CHARSET{$charset}->{encoder}) {
    return (&{$CHARSET{$charset}->{encoder}} ($charset, $s), success => 1);
  }
  ($s, success => 0);
}

sub decode ($$) {
  my ($charset, $s) = (lc shift, shift);
  if (ref $CHARSET{$charset}->{decoder}) {
    return (&{$CHARSET{$charset}->{decoder}} ($charset, $s), 1);
  }
  ($s, 0);
}

sub name_normalize ($) {
  my $name = lc shift;
  $CHARSET{$name}->{preferred_name} || $name;
}

sub name_minimumize ($$) {
  my ($charset, $s) = (lc shift, shift);
  if (ref $CHARSET{$charset}->{name_minimumizer} eq 'CODE') {
    return &{$CHARSET{$charset}->{name_minimumizer}} ($charset, $s);
  } elsif (ref $_MINIMUMIZER{$charset}) {
    return &{$_MINIMUMIZER{$charset}} ($charset, $s);
  }
  (charset => $charset);
}

sub _name_7bit_iso2022 ($$) {shift;
  my $s = shift;
  if ($s =~ /[\x0E\x0F\x1B]/) {
    return (charset => 'iso-2022-jp')
      unless $s =~ /\x1B[^\x24\x28]
                   |\x1B\x24[^\x40B]
                   |\x1B\x28[^BJ]
                   |\x0E|\x0F/x;
    return (charset => 'iso-2022-jp-1')
      unless $s =~ /\x1B[^\x24\x28]
                   |\x1B\x24[^\x40B\x28]
                   |\x1B\x24\x28[^D]
                   |\x1B\x28[^BJ]
                   |\x0E|\x0F/x;
    return (charset => 'iso-2022-jp-3-plane1')
      unless $s =~ /\x1B[^\x24\x28]
                   |\x1B\x24[^\x28]     #[^B\x28]
                   |\x1B\x24\x28[^O]
                   |\x1B\x28[^B]
                   |\x0E|\x0F/x;
    return (charset => 'iso-2022-jp-3')
      unless $s =~ /\x1B[^\x24\x28]
                   |\x1B\x24[^\x28]     #[^B\x28]
                   |\x1B\x24\x28[^OP]
                   |\x1B\x28[^B]
                   |\x0E|\x0F/x;
    return (charset => 'iso-2022-kr')
      unless $s =~ /\x1B[^\x24]
                   |\x1B\x24[^\x29]
                   |\x1B\x24\x29[^C]/x;
    return (charset => 'iso-2022-jp-2')
      unless $s =~ /\x1B[^\x24\x28\x2E\x4E]
                   |\x1B\x24[^\x40AB\x28]
                   |\x1B\x24\x28[^CD]
                   |\x1B\x28[^BJ]
                   |\x1B\x2E[^AF]
                   |\x0E|\x0F/x;
    return (charset => 'iso-2022-cn')
      unless $s =~ /\x1B[^\x4E\x24]
                   |\x1B\x24[^\x29\x2A]
                   |\x1B\x24\x29[^AG]
                   |\x1B\x24\x2A[^H]/x;
    return (charset => 'iso-2022-cn-ext')
      unless $s =~ /\x1B[^\x4E\x4F\x24]
                   |\x1B\x24[^\x29\x2A]
                   |\x1B\x24\x29[^AEG]
                   |\x1B\x24\x2A[^HIJKLM]/x;
    return (charset => 'iso-2022-int-1')
      unless $s =~ /\x1B[^\x24\x28\x2D]
                   |\x1B\x24[^\x40AB\x28\x29]
                   |\x1B\x24\x28[^DGH]
                   |\x1B\x24\x29[^C]
                   |\x1B\x28[^BJ]
                   |\x1B\x2D[^AF]/x;
    return (charset => 'junet')
      unless $s =~ /\x1B[^\x24\x28\x2C]
                   |\x1B\x24[^\x28\x2C\x40-\x42]
                   |\x1B\x24[\x28\x2C][^\x20-\x7E]
                   |\x1B\x24[\x28\x2C][\x20-\x2F]+[^\x30-\x7E]
                   |\x1B[\x28\x2C][^\x20-\x7E]
                   |\x1B[\x28\x2C][\x20-\x2F]+[^\x30-\x7E]
                   |\x0E|\x0F/x;
    return (charset => 'x-iso-2022');
  } else {
    return (charset => 'us-ascii');
  }
}

sub _name_net_ascii_8bit ($) {
  my $name = shift; my $s = shift;
  return (charset => 'us-ascii') unless $s =~ /[\x1B\x0E\x0F\x80-\xFF]/;
  if ($s =~ /[\x80-\xFF]/) {
    if ($s =~ /[\xC0-\xFD][\x80-\xBF]*[\x80-\x8F]/) {
      if ($s =~ /\x1B/) {
        return (charset => 'x-junet8'); ## junet + UTF-8
      } else {
        return (charset => 'utf-8');
      }
    } elsif ($s =~ /\x1B/) {
      return (charset => 'x-iso-2022'); ## 8bit ISO 2022
    } else {
      return (charset => 'iso-8859-1');
    }
  } else {      ## 7bit ISO 2022
    return _name_7bit_iso2022 ($name, $s);
  }
}

sub _name_8bit_iso_2022 ($$) {
  my $name = shift; my $s = shift;
  return (charset => 'us-ascii') unless $s =~ /[\x1B\x0E\x0F\x80-\xFF]/;
  if ($s =~ /[\x80-\xFF]/) {
    if ($s =~ /\x1B/) {
      return (charset => 'x-iso-2022'); ## 8bit ISO 2022
    } else {
      return (charset => 'iso-8859-1');
    }
  } else {      ## 7bit ISO 2022
    return _name_7bit_iso2022 ($name, $s);
  }
}

## Not completed.
## TODO: gb18030, cn-gb-12345
## TODO: _name_euc_gbf (cn-gb-12345, gb2312)
sub _name_euc_gb ($$) {
  my $name = shift; my $s = shift;
  if ($s =~ /[\x80-\xFF]/) {
    if ($s =~ /
                  (?:\G|[\x00-\x3F\x7F\x80\xFF])
                  (?:[\xA1-\xA9\xB0-\xFE][\xA1-\xFE]
                    |[\x40-\x7E])*
        (?:
          [\x81-\xA0\xAA-\xAF][\x40-\xFE]
         |[\xA1-\xFE][\x40-\xA0]
        )
      /x) {
      (charset => 'gbk');
    } elsif ($s =~ /
        (?:\xA2[\xA1-\xAA]
          |\xA6[\xE0-\xF5]
          |\xA8[\xBB-\xC0]
        )
          (?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]|\z))
      /x) {
      (charset => 'gbk');
    } elsif ($s =~ /
        (?:\xA3\xE7|\xA7[\xDD-\xF2]
          |\xA8[\xBB-\xC0]
          |[\xAA-\xAF\xF8-\xFE][\xA1-\xFE]
        )
          (?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]|\z))
      /x) {
      (charset => 'cn-gb-isoir165', 'charset-edition' => 1992);
    } elsif ($s =~ /\xEF\xF1    ## Typo bug of GB 2312
          (?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]|\z))
      /x) {
      (charset => 'gb2312');
    } else {
      (charset => 'gb2312', 'charset-edition' => 1980);
    }
  } elsif ($s =~ /[\x0E\x0F]/) {
    (charset => 'gb2312');      ## Actually, this is not "gb2312"
  } else {
    _name_7bit_iso_2022 ($name, $s);
  }
}

sub _name_euc_japan ($$) {
  my $name = shift; my $s = shift;
  if ($s =~ /[\x80-\xFF]/) {
    if ($s =~ /\x8F[\xA1\xA3-\xA5\xA8\xAC-\xAF\xEE-\xFE][\xA1-\xFE]/) {
      if ($s =~ /\x8F[\xA2\xA6\xA7\xA9-\xAB\xB0-\xED][\xA1-\xFE]/) {
      ## JIS X 0213 plane 2 + JIS X 0212
        (charset => 'x-euc-jisx0213-packed');
      } else {
        (charset => 'euc-jisx0213');
      }
    } elsif ($s =~ m{(?<![\x8E\x8F])    ## Not G2/G3 character
                    (?: ## JIS X 0213:2000
                       [\xA9-\xAF\xF5-\xFE][\xA1-\xFE]
                      |\xA2[\xAF-\xB9\xC2-\xC9\xD1-\xDB\xE9-\xF1\xFA-\xFD]
                      |\xA3[\xA1-\xAF\xBA-\xC0\xDB-\xE0\xFB-\xFE]
                      |\xA4[\xF4-\xFE]|\xA5[\xF7-\xFE]
                      |\xA6[\xB9-\xC0\xD9-\xFE]|\xA7[\xC2-\xD0\xF2-\xFE]
                      |\xA8[\xC1-\xFE]|\xCF[\xD4-\xFE]|\xF4[\xA7-\xFE]
                    )
                    (?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]|\z))}x) {
      if ($s =~ /\x8F/) {       ## JIS X 0213 plane 1 + JIS X 0212
        (charset => 'x-euc-jisx0213-packed');
      } else {
        (charset => 'euc-jisx0213-plane1');
      }
    } else {
      (charset => 'euc-jp');
    }
  } elsif ($s =~ /\x0E|\x0F|\x1B[\x4E\x4F]/) {
    (charset => 'euc-jisx0213');        ## Actually, this is not euc-japan
  } else {
    _name_7bit_iso_2022 ($name, $s);
  }
}

sub _name_shift_jis ($$) {
  my $name = shift; my $s = shift;
  if ($s =~ /[\x80-\xFF]/) {
    if ($s =~ /[\x0E\x0F\x1B]/) {
      (charset => 'x-sjis');
    } elsif ($s =~ /
                  (?:\G|[\x00-\x3F\x7F])
                  (?:[\x81-\x9F\xE0-\xFC][\x40-\x7E\x80-\xFC]
                    |[\x40-\x7E\xA1-\xDF])*
               [\xF0-\xFC][\x40-\x7E\x80-\xFC]
      /x) {
      (charset => 'shift_jisx0213');
    } elsif ($s =~ /
                  (?:\G|[\x00-\x3F\x7F])
                  (?:[\x81-\x9F\xE0-\xFC][\x40-\x7E\x80-\xFC]
                    |[\x40-\x7E\xA1-\xDF])*
              (?:
               [\x85-\x87\xEB-\xEF][\x40-\x7E\x80-\xFC]
              |\x81[\xAD-\xB7\xC0-\xC7\xCF-\xD9\xE9-\xEF\xF8-\xFB]
              |\x82[\x40-\x4E\x59-\x5F\x7A-\x80\x9B-\x9E\xF2-\xFC]
              |\x83[\x97-\x9E\xB7-\xBE\xD7-\xFC]
              |\x84[\x61-\x6F\x72-\x9E\xBF-\xFC]
              |\x88[\x40-\x9E]|\x98[\x73-\x9E]|\xEA[\xA5-\xFC]
              )
    /x) {
      (charset => 'shift_jisx0213-plane1');
    } else {
      (charset => 'shift_jis');
    }
  } elsif ($s =~ /[\x5C\x7E]/) {
    if ($s =~ /\x1B\x0E\x0F/) {
      (charset => 'x-sjis');    ## ISO 2022 with implied "ESC ( J"
        ## BUG: "ESC ( B foobar\aaa ESC ( J aiueo" also matchs this
    } else {
      (charset => 'jis_x0201');
    }
  } else {
    _name_7bit_iso_2022 ($name, $s);
  }
}

sub _name_utf16be ($$) {
  shift; my $s = shift;
  if ($s =~ /[\xD8-\xDB][\x00-\xFF][\xDC-\xDF][\x00-\xFF]
             (?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
    (charset => 'utf-16be');
  } elsif ($s =~ /[\x01-\xFF][\x00-\xFF]
             (?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
    if ($s =~ /([^\x00\x03\x04\x23\x25\x30\xFE\xFF]
                     [\x00-\xFF]        # ^\x20\x22\x4E-\x9F\xF9\xFA
                  |\x03[^\x00-\x6F\xD0-\xFF]
                  #|\x20[^\x00-\x6F]
                  |\x25[^\x00-\x7F]
                  |\xFE[^\x30-\x4F]
                  |\xFF[^\x00-\xEF]
                  ## note 1 of RFC 1816 is ambitious, so block entire
                  ## is excepted
                    |\x30[\x00-\x3F]
                  )
             (?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
      (charset => 'iso-10646-ucs-2');
    } else {
      (charset => 'iso-10646-j-1');
    }
  } elsif ($s =~ /\x00[\x80-\xFF]
             (?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
    (charset => 'iso-10646-unicode-latin1');
  } else {
    (charset => 'iso-10646-ucs-basic');
  }
}

sub _name_utf32be ($$) {
  shift; my $s = shift;
  if ($s =~ /
    ([\x01-\x7F][\x00-\xFF]{3}
    |\x00[\x11-\xFF][\x00-\xFF][\x00-\xFF])
             (?=(?:[\x00-\xFF]{4})*\z)/sx) {
    (charset => 'iso-10646-ucs-4');
  } else {
    (charset => 'utf-32be');
  }
}

=head1 LICENSE

Copyright 2002 wakaba E<lt>w@suika.fam.cxE<gt>.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; see the file COPYING.  If not, write to
the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.

=head1 CHANGE

See F<ChangeLog>.
$Date: 2002/06/23 12:16:10 $

=cut

1;
1	wakaba	1.1
2			=head1 NAME
3
4			Message::MIME::Charset Perl module
5
6			=head1 DESCRIPTION
7
8			Perl module for MIME charset.
9
10			=cut
11
12	wakaba	1.10	## NOTE: You should not require/use other module (even it
13			## is part of Message::* Perl Modules) as far as possible,
14			## to be able to use this module (M::M::Charset) from
15			## other (non-Message::*) modules.
16
17	wakaba	1.1	package Message::MIME::Charset;
18			use strict;
19	wakaba	1.9	use vars qw(%CHARSET %REG $VERSION);
20	wakaba	1.10	$VERSION=do{my @r=(q$Revision: 1.9 $=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
21	wakaba	1.1
22	wakaba	1.9	&_builtin_charset;
23			sub _builtin_charset () {
24	wakaba	1.4
25			$CHARSET{'*DEFAULT'} = {
26			preferred_name => '',
27
28			encoder => sub { $_[1] },
29			decoder => sub { $_[1] },
30
31			mime_text => 1, ## Suitability in use as MIME text/* charset
32			#accept_cte => [qw/7bit .../],
33			cte_7bit_preferred => 'quoted-printable',
34			};
35			$CHARSET{'default'} = $CHARSET{'DEFAULT'};
36
37			$CHARSET{'us-ascii'} = {
38			preferred_name => 'us-ascii',
39
40			encoder => sub { $_[1] },
41			decoder => sub { $_[1] },
42	wakaba	1.5
43			mime_text => 1,
44	wakaba	1.4	};
45
46			$CHARSET{'iso-2022-int-1'} = {
47			preferred_name => 'iso-2022-int-1',
48
49			encoder => sub { $_[1] },
50			decoder => sub { $_[1] },
51	wakaba	1.5
52			mime_text => 1,
53	wakaba	1.4	};
54
55			$CHARSET{'unknown-8bit'} = {
56			preferred_name => 'unknown-8bit',
57
58			encoder => sub { $_[1] },
59			decoder => sub { $_[1] },
60
61			mime_text => 0,
62	wakaba	1.6	cte_7bit_preferred => 'base64',
63	wakaba	1.4	};
64			$CHARSET{'x-unknown'} = $CHARSET{'unknown-8bit'};
65
66	wakaba	1.9	} # /builtin_charset
67
68			my %_MINIMUMIZER = (
69			'euc-jp' => \&_name_euc_japan,
70			'euc-jisx0213' => \&_name_euc_japan,
71			'euc-jisx0213-plane1' => \&_name_euc_japan,
72			'x-euc-jisx0213-packed' => \&_name_euc_japan,
73			'x-iso-2022' => \&_name_8bit_iso_2022,
74			'iso-2022-cn' => \&_name_8bit_iso_2022,
75			'iso-2022-cn-ext' => \&_name_8bit_iso_2022,
76			'iso-2022-int-1' => \&_name_net_ascii_8bit,
77			'iso-2022-jp' => \&_name_8bit_iso_2022,
78			'iso-2022-jp-1' => \&_name_8bit_iso_2022,
79			'iso-2022-jp-2' => \&_name_8bit_iso_2022,
80			'iso-2022-jp-3' => \&_name_8bit_iso_2022,
81			'iso-2022-jp-3-plane1' => \&_name_8bit_iso_2022,
82			'iso-2022-kr' => \&_name_8bit_iso_2022,
83			'iso-8859-1' => \&_name_8bit_iso_2022,
84			'iso-10646-j-1' => \&_name_utf16be,
85			'iso-10646-ucs-2' => \&_name_utf16be,
86			'iso-10646-ucs-4' => \&_name_utf32be,
87			'iso-10646-ucs-basic' => \&_name_utf16be,
88			'iso-10646-unicode-latin1' => \&_name_utf16be,
89			jis_x0201 => \&_name_shift_jis,
90			junet => \&_name_8bit_iso_2022,
91			'x-junet8' => \&_name_net_ascii_8bit,
92			shift_jis => \&_name_shift_jis,
93			shift_jisx0213 => \&_name_shift_jis,
94			'shift_jisx0213-plane1' => \&_name_shift_jis,
95			'x-sjis' => \&_name_shift_jis,
96			'us-ascii' => \&_name_net_ascii_8bit,
97			'utf-8' => \&_name_net_ascii_8bit,
98			'utf-16be' => \&_name_utf16be,
99			'utf-32be' => \&_name_utf32be,
100			);
101
102	wakaba	1.4	sub make_charset ($%) {
103			my $name = shift;
104			return unless $name; ## Note: charset "0" is not supported.
105			my %definition = @_;
106	wakaba	1.6
107			$definition{preferred_name} \|\|= $name;
108	wakaba	1.4	if ($definition{preferred_name} ne $name
109			&& ref $CHARSET{$definition{preferred_name}}) {
110			## New charset is an alias of defined charset,
111			$CHARSET{$name} = $CHARSET{$definition{preferred_name}};
112			return;
113			} elsif ($definition{alias_of} && ref $CHARSET{$definition{alias_of}}) {
114			## New charset is an alias of defined charset,
115			$CHARSET{$name} = $CHARSET{$definition{alias_of}};
116			return;
117			}
118			$CHARSET{$name} = \%definition;
119
120			## Set default values
121	wakaba	1.6	#$definition{encoder} \|\|= sub { $_[1] };
122			#$definition{decoder} \|\|= sub { $_[1] };
123	wakaba	1.4
124			$definition{mime_text} = 0 unless defined $definition{mime_text};
125			$definition{cte_7bit_preferred} = 'base64'
126			unless defined $definition{cte_7bit_preferred};
127			}
128	wakaba	1.1
129			sub encode ($$) {
130			my ($charset, $s) = (lc shift, shift);
131	wakaba	1.4	if (ref $CHARSET{$charset}->{encoder}) {
132	wakaba	1.5	return (&{$CHARSET{$charset}->{encoder}} ($charset, $s), success => 1);
133	wakaba	1.1	}
134	wakaba	1.5	($s, success => 0);
135	wakaba	1.1	}
136
137			sub decode ($$) {
138			my ($charset, $s) = (lc shift, shift);
139	wakaba	1.4	if (ref $CHARSET{$charset}->{decoder}) {
140			return (&{$CHARSET{$charset}->{decoder}} ($charset, $s), 1);
141	wakaba	1.1	}
142	wakaba	1.2	($s, 0);
143	wakaba	1.1	}
144
145			sub name_normalize ($) {
146			my $name = lc shift;
147	wakaba	1.4	$CHARSET{$name}->{preferred_name} \|\| $name;
148	wakaba	1.1	}
149
150	wakaba	1.6	sub name_minimumize ($$) {
151			my ($charset, $s) = (lc shift, shift);
152	wakaba	1.9	if (ref $CHARSET{$charset}->{name_minimumizer} eq 'CODE') {
153	wakaba	1.6	return &{$CHARSET{$charset}->{name_minimumizer}} ($charset, $s);
154	wakaba	1.9	} elsif (ref $_MINIMUMIZER{$charset}) {
155			return &{$_MINIMUMIZER{$charset}} ($charset, $s);
156			}
157			(charset => $charset);
158			}
159
160			sub _name_7bit_iso2022 ($$) {shift;
161			my $s = shift;
162			if ($s =~ /[\x0E\x0F\x1B]/) {
163			return (charset => 'iso-2022-jp')
164			unless $s =~ /\x1B[^\x24\x28]
165			\|\x1B\x24[^\x40B]
166			\|\x1B\x28[^BJ]
167			\|\x0E\|\x0F/x;
168			return (charset => 'iso-2022-jp-1')
169			unless $s =~ /\x1B[^\x24\x28]
170			\|\x1B\x24[^\x40B\x28]
171			\|\x1B\x24\x28[^D]
172			\|\x1B\x28[^BJ]
173			\|\x0E\|\x0F/x;
174			return (charset => 'iso-2022-jp-3-plane1')
175			unless $s =~ /\x1B[^\x24\x28]
176			\|\x1B\x24[^\x28] #[^B\x28]
177			\|\x1B\x24\x28[^O]
178			\|\x1B\x28[^B]
179			\|\x0E\|\x0F/x;
180			return (charset => 'iso-2022-jp-3')
181			unless $s =~ /\x1B[^\x24\x28]
182			\|\x1B\x24[^\x28] #[^B\x28]
183			\|\x1B\x24\x28[^OP]
184			\|\x1B\x28[^B]
185			\|\x0E\|\x0F/x;
186			return (charset => 'iso-2022-kr')
187			unless $s =~ /\x1B[^\x24]
188			\|\x1B\x24[^\x29]
189			\|\x1B\x24\x29[^C]/x;
190			return (charset => 'iso-2022-jp-2')
191			unless $s =~ /\x1B[^\x24\x28\x2E\x4E]
192			\|\x1B\x24[^\x40AB\x28]
193			\|\x1B\x24\x28[^CD]
194			\|\x1B\x28[^BJ]
195			\|\x1B\x2E[^AF]
196			\|\x0E\|\x0F/x;
197			return (charset => 'iso-2022-cn')
198			unless $s =~ /\x1B[^\x4E\x24]
199			\|\x1B\x24[^\x29\x2A]
200			\|\x1B\x24\x29[^AG]
201			\|\x1B\x24\x2A[^H]/x;
202			return (charset => 'iso-2022-cn-ext')
203			unless $s =~ /\x1B[^\x4E\x4F\x24]
204			\|\x1B\x24[^\x29\x2A]
205			\|\x1B\x24\x29[^AEG]
206			\|\x1B\x24\x2A[^HIJKLM]/x;
207			return (charset => 'iso-2022-int-1')
208			unless $s =~ /\x1B[^\x24\x28\x2D]
209			\|\x1B\x24[^\x40AB\x28\x29]
210			\|\x1B\x24\x28[^DGH]
211			\|\x1B\x24\x29[^C]
212			\|\x1B\x28[^BJ]
213			\|\x1B\x2D[^AF]/x;
214			return (charset => 'junet')
215			unless $s =~ /\x1B[^\x24\x28\x2C]
216			\|\x1B\x24[^\x28\x2C\x40-\x42]
217			\|\x1B\x24[\x28\x2C][^\x20-\x7E]
218			\|\x1B\x24[\x28\x2C][\x20-\x2F]+[^\x30-\x7E]
219			\|\x1B[\x28\x2C][^\x20-\x7E]
220			\|\x1B[\x28\x2C][\x20-\x2F]+[^\x30-\x7E]
221			\|\x0E\|\x0F/x;
222			return (charset => 'x-iso-2022');
223			} else {
224			return (charset => 'us-ascii');
225	wakaba	1.6	}
226			}
227
228	wakaba	1.9	sub _name_net_ascii_8bit ($) {
229			my $name = shift; my $s = shift;
230	wakaba	1.8	return (charset => 'us-ascii') unless $s =~ /[\x1B\x0E\x0F\x80-\xFF]/;
231			if ($s =~ /[\x80-\xFF]/) {
232			if ($s =~ /[\xC0-\xFD][\x80-\xBF]*[\x80-\x8F]/) {
233			if ($s =~ /\x1B/) {
234	wakaba	1.9	return (charset => 'x-junet8'); ## junet + UTF-8
235	wakaba	1.8	} else {
236			return (charset => 'utf-8');
237			}
238			} elsif ($s =~ /\x1B/) {
239	wakaba	1.9	return (charset => 'x-iso-2022'); ## 8bit ISO 2022
240			} else {
241			return (charset => 'iso-8859-1');
242			}
243			} else { ## 7bit ISO 2022
244			return _name_7bit_iso2022 ($name, $s);
245			}
246			}
247
248			sub _name_8bit_iso_2022 ($$) {
249			my $name = shift; my $s = shift;
250			return (charset => 'us-ascii') unless $s =~ /[\x1B\x0E\x0F\x80-\xFF]/;
251			if ($s =~ /[\x80-\xFF]/) {
252			if ($s =~ /\x1B/) {
253			return (charset => 'x-iso-2022'); ## 8bit ISO 2022
254	wakaba	1.8	} else {
255			return (charset => 'iso-8859-1');
256			}
257	wakaba	1.9	} else { ## 7bit ISO 2022
258			return _name_7bit_iso2022 ($name, $s);
259			}
260			}
261
262			## Not completed.
263			## TODO: gb18030, cn-gb-12345
264			## TODO: _name_euc_gbf (cn-gb-12345, gb2312)
265			sub _name_euc_gb ($$) {
266			my $name = shift; my $s = shift;
267			if ($s =~ /[\x80-\xFF]/) {
268			if ($s =~ /
269			(?:\G\|[\x00-\x3F\x7F\x80\xFF])
270			(?:[\xA1-\xA9\xB0-\xFE][\xA1-\xFE]
271			\|[\x40-\x7E])*
272			(?:
273			[\x81-\xA0\xAA-\xAF][\x40-\xFE]
274			\|[\xA1-\xFE][\x40-\xA0]
275			)
276			/x) {
277			(charset => 'gbk');
278			} elsif ($s =~ /
279			(?:\xA2[\xA1-\xAA]
280			\|\xA6[\xE0-\xF5]
281			\|\xA8[\xBB-\xC0]
282			)
283			(?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]\|\z))
284			/x) {
285			(charset => 'gbk');
286			} elsif ($s =~ /
287			(?:\xA3\xE7\|\xA7[\xDD-\xF2]
288			\|\xA8[\xBB-\xC0]
289			\|[\xAA-\xAF\xF8-\xFE][\xA1-\xFE]
290			)
291			(?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]\|\z))
292			/x) {
293			(charset => 'cn-gb-isoir165', 'charset-edition' => 1992);
294			} elsif ($s =~ /\xEF\xF1 ## Typo bug of GB 2312
295			(?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]\|\z))
296			/x) {
297			(charset => 'gb2312');
298			} else {
299			(charset => 'gb2312', 'charset-edition' => 1980);
300			}
301			} elsif ($s =~ /[\x0E\x0F]/) {
302			(charset => 'gb2312'); ## Actually, this is not "gb2312"
303			} else {
304			_name_7bit_iso_2022 ($name, $s);
305			}
306			}
307
308			sub _name_euc_japan ($$) {
309			my $name = shift; my $s = shift;
310			if ($s =~ /[\x80-\xFF]/) {
311			if ($s =~ /\x8F[\xA1\xA3-\xA5\xA8\xAC-\xAF\xEE-\xFE][\xA1-\xFE]/) {
312			if ($s =~ /\x8F[\xA2\xA6\xA7\xA9-\xAB\xB0-\xED][\xA1-\xFE]/) {
313			## JIS X 0213 plane 2 + JIS X 0212
314			(charset => 'x-euc-jisx0213-packed');
315			} else {
316			(charset => 'euc-jisx0213');
317			}
318			} elsif ($s =~ m{(?<![\x8E\x8F]) ## Not G2/G3 character
319			(?: ## JIS X 0213:2000
320			[\xA9-\xAF\xF5-\xFE][\xA1-\xFE]
321			\|\xA2[\xAF-\xB9\xC2-\xC9\xD1-\xDB\xE9-\xF1\xFA-\xFD]
322			\|\xA3[\xA1-\xAF\xBA-\xC0\xDB-\xE0\xFB-\xFE]
323			\|\xA4[\xF4-\xFE]\|\xA5[\xF7-\xFE]
324			\|\xA6[\xB9-\xC0\xD9-\xFE]\|\xA7[\xC2-\xD0\xF2-\xFE]
325			\|\xA8[\xC1-\xFE]\|\xCF[\xD4-\xFE]\|\xF4[\xA7-\xFE]
326			)
327			(?=(?:[\xA1-\xFE][\xA1-\xFE])*(?:[\x00-\xA0\xFF]\|\z))}x) {
328			if ($s =~ /\x8F/) { ## JIS X 0213 plane 1 + JIS X 0212
329			(charset => 'x-euc-jisx0213-packed');
330			} else {
331			(charset => 'euc-jisx0213-plane1');
332			}
333			} else {
334			(charset => 'euc-jp');
335			}
336			} elsif ($s =~ /\x0E\|\x0F\|\x1B[\x4E\x4F]/) {
337			(charset => 'euc-jisx0213'); ## Actually, this is not euc-japan
338			} else {
339			_name_7bit_iso_2022 ($name, $s);
340			}
341			}
342
343			sub _name_shift_jis ($$) {
344			my $name = shift; my $s = shift;
345			if ($s =~ /[\x80-\xFF]/) {
346			if ($s =~ /[\x0E\x0F\x1B]/) {
347			(charset => 'x-sjis');
348			} elsif ($s =~ /
349			(?:\G\|[\x00-\x3F\x7F])
350			(?:[\x81-\x9F\xE0-\xFC][\x40-\x7E\x80-\xFC]
351			\|[\x40-\x7E\xA1-\xDF])*
352			[\xF0-\xFC][\x40-\x7E\x80-\xFC]
353			/x) {
354			(charset => 'shift_jisx0213');
355			} elsif ($s =~ /
356			(?:\G\|[\x00-\x3F\x7F])
357			(?:[\x81-\x9F\xE0-\xFC][\x40-\x7E\x80-\xFC]
358			\|[\x40-\x7E\xA1-\xDF])*
359			(?:
360			[\x85-\x87\xEB-\xEF][\x40-\x7E\x80-\xFC]
361			\|\x81[\xAD-\xB7\xC0-\xC7\xCF-\xD9\xE9-\xEF\xF8-\xFB]
362			\|\x82[\x40-\x4E\x59-\x5F\x7A-\x80\x9B-\x9E\xF2-\xFC]
363			\|\x83[\x97-\x9E\xB7-\xBE\xD7-\xFC]
364			\|\x84[\x61-\x6F\x72-\x9E\xBF-\xFC]
365			\|\x88[\x40-\x9E]\|\x98[\x73-\x9E]\|\xEA[\xA5-\xFC]
366			)
367			/x) {
368			(charset => 'shift_jisx0213-plane1');
369			} else {
370			(charset => 'shift_jis');
371			}
372			} elsif ($s =~ /[\x5C\x7E]/) {
373			if ($s =~ /\x1B\x0E\x0F/) {
374			(charset => 'x-sjis'); ## ISO 2022 with implied "ESC ( J"
375			## BUG: "ESC ( B foobar\aaa ESC ( J aiueo" also matchs this
376			} else {
377			(charset => 'jis_x0201');
378			}
379			} else {
380			_name_7bit_iso_2022 ($name, $s);
381			}
382			}
383
384			sub _name_utf16be ($$) {
385			shift; my $s = shift;
386			if ($s =~ /[\xD8-\xDB][\x00-\xFF][\xDC-\xDF][\x00-\xFF]
387			(?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
388			(charset => 'utf-16be');
389			} elsif ($s =~ /[\x01-\xFF][\x00-\xFF]
390			(?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
391			if ($s =~ /([^\x00\x03\x04\x23\x25\x30\xFE\xFF]
392			[\x00-\xFF] # ^\x20\x22\x4E-\x9F\xF9\xFA
393			\|\x03[^\x00-\x6F\xD0-\xFF]
394			#\|\x20[^\x00-\x6F]
395			\|\x25[^\x00-\x7F]
396			\|\xFE[^\x30-\x4F]
397			\|\xFF[^\x00-\xEF]
398			## note 1 of RFC 1816 is ambitious, so block entire
399			## is excepted
400			\|\x30[\x00-\x3F]
401			)
402			(?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
403			(charset => 'iso-10646-ucs-2');
404			} else {
405			(charset => 'iso-10646-j-1');
406			}
407			} elsif ($s =~ /\x00[\x80-\xFF]
408			(?=(?:[\x00-\xFF][\x00-\xFF])*\z)/sx) {
409			(charset => 'iso-10646-unicode-latin1');
410			} else {
411			(charset => 'iso-10646-ucs-basic');
412			}
413			}
414
415			sub _name_utf32be ($$) {
416			shift; my $s = shift;
417			if ($s =~ /
418			([\x01-\x7F][\x00-\xFF]{3}
419			\|\x00[\x11-\xFF][\x00-\xFF][\x00-\xFF])
420			(?=(?:[\x00-\xFF]{4})*\z)/sx) {
421			(charset => 'iso-10646-ucs-4');
422			} else {
423			(charset => 'utf-32be');
424	wakaba	1.8	}
425			}
426
427	wakaba	1.1	=head1 LICENSE
428
429			Copyright 2002 wakaba E<lt>w@suika.fam.cxE<gt>.
430
431			This program is free software; you can redistribute it and/or modify
432			it under the terms of the GNU General Public License as published by
433			the Free Software Foundation; either version 2 of the License, or
434			(at your option) any later version.
435
436			This program is distributed in the hope that it will be useful,
437			but WITHOUT ANY WARRANTY; without even the implied warranty of
438			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
439			GNU General Public License for more details.
440
441			You should have received a copy of the GNU General Public License
442			along with this program; see the file COPYING. If not, write to
443			the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
444			Boston, MA 02111-1307, USA.
445
446			=head1 CHANGE
447
448			See F<ChangeLog>.
449	wakaba	1.10	$Date: 2002/06/23 12:16:10 $
450	wakaba	1.1
451			=cut
452
453			1;