Whatpm/CSS/Tokenizer.pm

package Whatpm::CSS::Tokenizer;
use strict;

sub new ($) {
  my $self = bless {token => []}, shift;
  return $self;
} # new

sub init ($) {
  my $self = shift;
  $self->{state} = BEFORE_TOKEN_STATE;
  $self->{c} = $self->{get_char}->();
} # init

sub get_next_token ($) {
  my $self = shift;
  if (@{$self->{token}}) {
    return shift @{$self->{token}};
  }

  my $current_token;
  my $char;
  my $num; # |{num}|, if any.
  my $i; # |$i + 1|th character in |unicode| in |escape|.
  my $q; # |$q == 0 ? "in |ident|" : "in |string$q| or in |invalid$q|"|

  A: {
    if ($self->{state} == BEFORE_TOKEN_STATE) {
      if ($self->{c} == 0x002D) { # -
        ## NOTE: |-| in |ident| in |IDENT|
        $current_token = {type => IDENT_TOKEN, value => '-'};
        $self->{state} = BEFORE_NMSTART_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
               (0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
               $self->{c} == 0x005F or # _
               $self->{c} > 0x007F) { # nonascii
        ## NOTE: |nmstart| in |ident| in |IDENT|
        $current_token = {type => IDENT_TOKEN, value => chr $self->{char}};
        $self->{state} = NAME_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x005C) { # \
        ## NOTE: |nmstart| in |ident| in |IDENT|
        $current_token = {type => IDENT_TOKEN, value => ''};
        $self->{state} = ESCAPE_OPEN_STATE; $q = 0;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x0040) { # @
        ## NOTE: |@| in |ATKEYWORD|
        $current_token = {type => ATKEYWORD_TOKEN, value => ''};
        $self->{state} = BEFORE_NMSTART_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x0022) { # "
        ## NOTE: |"| in |string1| in |string| in |STRING|, or
        ## |"| in |invalid1| in |invalid| in |INVALID|.
        $current_token = {type => STRING_TOKEN, value => ''};
        $self->{state} = STRING_STATE; $q = 1;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x0027) { # '
        ## NOTE: |'| in |string2| in |string| in |STRING|, or
        ## |'| in |invalid2| in |invalid| in |INVALID|.
        $current_token = {type => STRING_TOKEN, value => ''};
        $self->{state} = STRING_STATE; $q = 2;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x0023) { # #
        ## NOTE: |#| in |HASH|.
        $current_token = {type => HASH_TOKEN, value => ''};
        $self->{state} = HASH_OPEN_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
        ## NOTE: |num|.
        $current_token = {type => NUMBER_TOKEN, value => chr $self->{c}};
        $self->{state} = NUMBER_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x002E) { # .
        ## NOTE: |num|.
        $current_token = {type => NUMBER_TOKEN, value => '.'};
        $self->{state} = NUMBER_FRACTION_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x003C) { # <
        ## NOTE: |CDO|
        $self->{c} = $self->{get_char}->();
        if ($self->{c} == 0x0021) { # !
          $self->{c} = $self->{get_char}->();
          if ($self->{c} == 0x002C) { # -
            $self->{c} = $self->{get_char}->();
            if ($self->{c} == 0x002C) { # -
              $self->{state} = BEFORE_TOKEN_STATE;
              $self->{c} = $self->{get_char}->();
              return {type => CDO_TOKEN};
              #redo A;
            } else {
              unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'};
              ## NOTE: |-| in |ident| in |IDENT|
              $current_token = {type => IDENT_TOKEN, value => '-'};
              $self->{state} = BEFORE_NMSTART_STATE;
              #reprocess
              return {type => DELIM_TOKEN, value => '<'};
              #redo A;
            }
          } else {
            unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'};
            $self->{state} = BEFORE_TOKEN_STATE;
            #reprocess
            return {type => DELIM_TOKEN, value => '<'};
            #redo A;
          }
        } else {
          $self->{state} = BEFORE_TOKEN_STATE;
          #reprocess
          return {type => DELIM_TOKEN, value => '<'};
          #redo A;
        }
      } elsif ({
                0x003B => 1, # ;
                0x007B => 1, # {
                0x007D => 1, # }
                0x0028 => 1, # (
                0x0029 => 1, # )
                0x005B => 1, # [
                0x005D => 1, # ]
               }->{$self->{c}}) {
        # stay in the state
        $self->{c} = $self->{get_char}->();
        return {type => chr $self->{c}};
        # redo A;
      } elsif ({
                0x0020 => 1, # SP
                0x0009 => 1, # \t
                0x000D => 1, # \r
                0x000A => 1, # \n
                0x000C => 1, # \f
               }->{$self->{c}}) {
        W: {
          $self->{c} = $self->{get_char}->();
          if ({
                0x0020 => 1, # SP
                0x0009 => 1, # \t
                0x000D => 1, # \r
                0x000A => 1, # \n
                0x000C => 1, # \f
              }->{$self->{c}}) {
            redo W;
          } elsif (my $v = {
                            0x002B => PLUS_TOKEN, # +
                            0x003E => GREATER_TOKEN, # >
                            0x002C => COMMA_TOKEN, # ,
                            0x007E => TILDE_TOKEN, # ~
                           }->{$self->{c}}) {
            # stay in the state
            $self->{c} = $self->{get_char}->();
            return {type => $v};
            #redo A;
          } else {
            # stay in the state
            # reprocess
            return {type => S_TOKEN};
            #redo A;
          }
        } # W
      } elsif (my $v = {
                        0x007C => DASHMATCH_TOKEN, # |
                        0x005E => PREFIXMATCH_TOKEN, # ^
                        0x0024 => SUFFIXMATCH_TOKEN, # $
                        0x002A => SUBSTRINGMATCH_TOKEN, # *
                       }->{$self->{c}}) {
        $self->{c} = $self->{get_char}->();
        if ($self->{c} == 0x003D) { # =
          # stay in the state
          $self->{c} = $self->{get_char}->();
          return {type => $v};
          #redo A;
        } else {
          # stay in the state
          # reprocess
          return {type => DELIM_TOKEN, value => chr $self->{c}};
          #redo A;
        }
      } elsif ($self->{c} == 0x002B) { # +
        # stay in the state
        $self->{c} = $self->{get_char}->();
        return {type => PLUS_TOKEN};
        #redo A;
      } elsif ($self->{c} == 0x003E) { # >
        # stay in the state
        $self->{c} = $self->{get_char}->();
        return {type => GREATER_TOKEN};
        #redo A;
      } elsif ($self->{c} == 0x002C) { # ,
        # stay in the state
        $self->{c} = $self->{get_char}->();
        return {type => COMMA_TOKEN};
        #redo A;
      } elsif ($self->{c} == 0x007E) { # ~
        $self->{c} = $self->{get_char}->();
        if ($self->{c} == 0x003D) { # =
          # stay in the state
          $self->{c} = $self->{get_char}->();
          return {type => INCLUDES_TOKEN};
          #redo A;
        } else {
          # stay in the state
          # reprocess
          return {type => TILDE_TOKEN};
          #redo A;
        }
      } elsif ($self->{c} == -1) {
        # stay in the state
        $self->{c} = $self->{get_char}->();
        return {type => EOF_TOKEN};
        #redo A;
      } else {
        # stay in the state
        $current_token = {type => DELIM_TOKEN, value => chr $self->{c}};
        $self->{c} = $self->{get_char}->();
        return $current_token;
        #redo A;
      }
    } elsif ($self->{state} == BEFORE_NMSTART_STATE) {
      ## NOTE: |nmstart| in |ident| in (|IDENT| or |ATKEYWORD|)
      if ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
          (0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
          $self->{c} == 0x005F or # _
          $self->{c} > 0x007F) { # nonascii
        $current_token->{value} .= chr $self->{char};
        $self->{state} = NAME_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x005C) { # \
        $self->{state} = ESCAPE_OPEN_STATE; $q = 0;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x002D and # -
               $current_token->{type} == IDENT_TOKEN) {
        $self->{c} = $self->{get_char}->();
        if ($self->{c} == 0x003E) { # >
          $self->{state} = BEFORE_TOKEN_STATE;
          $self->{c} = $self->{get_char}->();
          return {type => CDC_TOKEN};
          #redo A;
        } else {
          ## NOTE: |-|, |-|, $self->{c}
          #$current_token = {type => IDENT_TOKEN, value => '-'};
          # stay in the state
          # reconsume
          return {type => DELIM_TOKEN, value => '-'};
          #redo A;
        }
      } else {
        if ($current_token->{type} == NUMBER_TOKEN) {
          ## NOTE: |-| after |num|.
          unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'};
          $self->{state} = BEFORE_TOKEN_STATE;
          $self->{c} = $self->{get_char}->();
          return $current_token;
        } elsif ($current_token->{type} == ATKEYWORD_TOKEN) {
          ## NOTE: |-| after |@|.
          unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '@'};
          $self->{state} = BEFORE_TOKEN_STATE;
          $self->{c} = $self->{get_char}->();
          return $current_token;
        } else {
          ## NOTE: |-| not followed by |nmstart|.
          $self->{state} = BEFORE_TOKEN_STATE;
          $self->{c} = $self->{get_char}->();
          return {type => DELIM_TOKEN, value => '-'};
        }
      }
    } elsif ($self->{state} == AFTER_NUMBER_STATE) {
      if ($self->{c} == 0x002D) { # -
        ## NOTE: |-| in |ident|.
        $current_token->{value} = '-';
        $self->{state} = BEFORE_NMSTART_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
               (0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
               $self->{c} == 0x005F or # _
               $self->{c} > 0x007F) { # nonascii
        ## NOTE: |nmstart| in |ident|.
        $current_token->{value} = chr $self->{char};
        $self->{state} = NAME_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x005C) { # \
        ## NOTE: |nmstart| in |ident| in |IDENT|
        $current_token->{value} = '';
        $self->{state} = ESCAPE_OPEN_STATE; $q = 0;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x0025) { # %
        $current_token->{type} = PERCENTAGE_TOKEN;
        $self->{state} = BEFORE_TOKEN_STATE;
        $self->{c} = $self->{get_char}->();
        return $current_token;
        #redo A;
      } else {
        $self->{state} = BEFORE_TOKEN_STATE;
        # reprocess
        return $current_token;
        #redo A;
      }
    } elsif ($self->{state} == HASH_OPEN_STATE) {
      ## NOTE: The first |nmchar| in |name| in |HASH|.
      if ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
          (0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
          (0x0030 <= $self->{c} or $self->{c} <= 0x0039) or # 0..9
          $self->{c} == 0x002D or # -
          $self->{c} == 0x005F or # _
          $self->{c} > 0x007F) { # nonascii
        $current_token->{value} .= chr $self->{char};
        $self->{state} = NAME_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x005C) { # \
        $self->{state} = ESCAPE_OPEN_STATE; $q = 0;
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        $self->{state} = BEFORE_TOKEN_STATE;
        $self->{c} = $self->{get_char}->();
        return {type => DELIM_TOKEN, value => '#'};
        #redo A;
      }
    } elsif ($self->{state} == NAME_STATE) {
      ## NOTE: |nmchar| in (|ident| or |name|).
      if ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
          (0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
          (0x0030 <= $self->{c} or $self->{c} <= 0x0039) or # 0..9
          $self->{c} == 0x005F or # _
          $self->{c} == 0x002D or # -
          $self->{c} > 0x007F) { # nonascii
        $current_token->{value} .= chr $self->{char};
        # stay in the state
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x005C) { # \
        $self->{state} = ESCAPE_OPEN_STATE; # $q = 0;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x0028 and # (
               $current_token->{type} == IDENT_TOKEN) { # (
        if (not $current_token->{has_escape} and
            {url => 1, Url => 1, uRl => 1, urL => 1,
             URl => 1, UrL => 1, uRL => 1, URL => 1}
            ->{$current_token->{value}}) {
          $current_token->{type} = URI_TOKEN;
          $self->{state} = URI_BEFORE_WSP_STATE;
          $self->{c} = $self->{get_char}->();

          ## NOTE: This version of the tokenizer does not support the |URI|
          ## token type.  Note that browsers disagree in how to tokenize
          ## |url| function.
          $current_token->{type} = FUNCTION_TOKEN;
          $self->{state} = BEFORE_TOKEN_STATE;
          $self->{c} = $self->{get_char}->();
          return $current_token;

          redo A;
        } else {
          $current_token->{type} = FUNCTION_TOKEN;
          $self->{state} = BEFORE_TOKEN_STATE;
          $self->{c} = $self->{get_char}->();
          return $current_token;
          #redo A;
        }
      } else {
        $self->{state} = BEFORE_TOKEN_STATE;
        # reconsume
        return $current_token;
        #redo A;
      }
    } elsif ($self->{state} == ESCAPE_OPEN_STATE) {
      $current_token->{has_escape} = 1;
      if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
        ## NOTE: second character of |unicode| in |escape|.
        $char = $self->{c} - 0x0030;
        $self->{state} = ESCAPE_STATE; $i = 2;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif (0x0041 <= $self->{c} and $self->{c} <= 0x0046) { # A..F
        ## NOTE: second character of |unicode| in |escape|.
        $char = $self->{c} - 0x0041 + 0xA;
        $self->{state} = ESCAPE_STATE; $i = 2;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif (0x0061 <= $self->{c} or $self->{c} <= 0x0066) { # a..f
        ## NOTE: second character of |unicode| in |escape|.
        $char = $self->{c} - 0x0061 - 0xA;
        $self->{state} = ESCAPE_STATE; $i = 2;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x000A or # \n
               $self->{c} == 0x000C) { # \f
        if ($q == 0) {
          ## NOTE: In |escape| in ... in |ident|.
          $self->{state} = BEFORE_TOKEN_STATE;
          unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
          return $current_token;
          # reconsume
          #redo A;
        } else {
          ## Note: In |nl| in ... in |string| or |ident|.
          $current_token->{value} .= chr $self->{c};
          $self->{state} = STRING_STATE;
          $self->{c} = $self->{get_char}->();
          redo A;
        }
      } elsif ($self->{c} == 0x000D) { # \r
        if ($q == 0) {
          ## NOTE: In |escape| in ... in |ident|.
          $self->{state} = BEFORE_TOKEN_STATE;
          unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
          return $current_token;
          # reconsume
          #redo A;
        } else {
          ## Note: In |nl| in ... in |string| or |ident|.
          $current_token->{value} .= "\x0D\x0A";
          $self->{state} = ESCAPE_BEFORE_LF_STATE;
          $self->{c} = $self->{get_char}->();
          redo A;
        }
      } else {
        ## NOTE: second character of |escape|.
        $current_token->{value} .= chr $self->{c};
        $self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      }
    } elsif ($self->{state} == ESCAPE_STATE) {
      ## NOTE: third..seventh character of |unicode| in |escape|.
      if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
        $char = $char * 0x10 + $self->{c} - 0x0030;
        $self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif (0x0041 <= $self->{c} and $self->{c} <= 0x0046) { # A..F
        $char = $char * 0x10 + $self->{c} - 0x0041 + 0xA;
        $self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif (0x0061 <= $self->{c} or $self->{c} <= 0x0066) { # a..f
        $char = $char * 0x10 + $self->{c} - 0x0061 - 0xA;
        $self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x0020 or # SP
               $self->{c} == 0x000A or # \n
               $self->{c} == 0x0009 or # \t
               $self->{c} == 0x000C) { # \f
        $current_token->{value} .= chr $char;
        $self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x000D) { # \r
        $self->{state} = ESCAPE_BEFORE_LF_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        $current_token->{value} .= chr $char;
        $self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
        # reconsume
        redo A;
      }
    } elsif ($self->{state} == ESCAPE_BEFORE_NL_STATE) {
      ## NOTE: eightth character of |unicode| in |escape|.
      if ($self->{c} == 0x0020 or # SP
          $self->{c} == 0x000A or # \n
          $self->{c} == 0x0009 or # \t
          $self->{c} == 0x000C) { # \f
        $current_token->{value} .= chr $char;
        $self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x000D) { # \r
        $self->{state} = ESCAPE_BEFORE_NL_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        $current_token->{value} .= chr $char;
        $self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
        # reconsume
        redo A;
      }
    } elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {
      ## NOTE: |\n| in |\r\n| in |unicode| in |escape|.
      if ($self->{c} == 0x000A) { # \n
        $current_token->{value} .= chr $char;
        $self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        $current_token->{value} .= chr $char;
        $self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
        # reconsume
        redo A;
      }
    } elsif ($self->{state} == STRING_STATE) {
      ## NOTE: A character in |string$Q| in |string| in |STRING|, or
      ## a character in |invalid$Q| in |invalid| in |INVALID|,
      ## where |$Q = $q == 0x0022 ? 1 : 2|.
      if ($self->{c} == 0x005C) { # \
        $self->{state} = ESCAPE_OPEN_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == $q) { # " | '
        $self->{state} = BEFORE_TOKEN_STATE;
        $self->{c} = $self->{get_char}->();
        return $current_token;
        #redo A;
      } elsif ($self->{c} == 0x000A or # \n
               $self->{c} == 0x000D or # \r
               $self->{c} == 0x000C or # \f
               $self->{c} == -1) {
        $current_token->{type} = INVALID_TOKEN;
        $self->{state} = BEFORE_TOKEN_STATE;
        # reconsume
        return $current_token;
        #redo A;
      } else {
        $current_token->{value} .= chr $self->{c};
        # stay in the state
        $self->{c} = $self->{get_char}->();
        redo A;
      }
    } elsif ($self->{state} == NUMBER_STATE) {
      ## NOTE: 2nd, 3rd, or ... character in |num| before |.|.
      if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
        $current_token->{value} .= chr $self->{c};
        # stay in the state
        $self->{c} = $self->{get_char}->();
        redo A;
      } elsif ($self->{c} == 0x002E) { # .
        $self->{state} = NUMBER_DOT_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        $self->{number} = $self->{value};
        $self->{value} = '';
        $self->{state} = AFTER_NUMBER_STATE;
        # reprocess
        return $current_token;
        #redo A;
      }
    } elsif ($self->{state} == NUMBER_DOT_STATE) {
      ## NOTE: The character immediately following |.| in |num|.
      if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
        $current_token->{value} .= chr $self->{c};
        $self->{state} = NUMBER_DOT_NUMBER_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        unshift @{$self->{token}}, {type => DELIM_STATE, value => '.'};
        $self->{number} = $self->{value};
        $self->{value} = '';
        $self->{state} = BEFORE_TOKEN_STATE;
        # reprocess
        return $current_token;
        #redo A;
      }
    } elsif ($self->{state} == NUMBER_FRACTION_STATE) {
      ## NOTE: The character immediately following |.| at the beginning of |num|.
      if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
        $current_token->{value} .= chr $self->{c};
        $self->{state} = NUMBER_DOT_NUMBER_STATE;
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        $self->{state} = BEFORE_TOKEN_STATE;
        $self->{c} = $self->{get_char}->();
        return {type => DELIM_TOKEN, value => '.'};
        #redo A;
      }
    } elsif ($self->{state} == NUMBER_DOT_NUMBER_STATE) {
      ## NOTE: |[0-9]| in |num| after |.|.
      if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
        $current_token->{value} .= chr $self->{c};
        # stay in the state
        $self->{c} = $self->{get_char}->();
        redo A;
      } else {
        $self->{number} = $self->{value};
        $self->{value} = '';
        $self->{state} = AFTER_NUMBER_STATE;
        # reprocess
        return $current_token;
        #redo A;
      }
    } else {
      die "$0: Unknown state |$self->{state}|";
    }
  } # A

  ## TODO: |URI|, |UNICODE-RANGE|, |COMMENT|

} # get_next_token

1;
# $Date:$
1	package Whatpm::CSS::Tokenizer;
2	use strict;
3
4	sub new ($) {
5	my $self = bless {token => []}, shift;
6	return $self;
7	} # new
8
9	sub init ($) {
10	my $self = shift;
11	$self->{state} = BEFORE_TOKEN_STATE;
12	$self->{c} = $self->{get_char}->();
13	} # init
14
15	sub get_next_token ($) {
16	my $self = shift;
17	if (@{$self->{token}}) {
18	return shift @{$self->{token}};
19	}
20
21	my $current_token;
22	my $char;
23	my $num; # \|{num}\|, if any.
24	my $i; # \|$i + 1\|th character in \|unicode\| in \|escape\|.
25	my $q; # \|$q == 0 ? "in \|ident\|" : "in \|string$q\| or in \|invalid$q\|"\|
26
27	A: {
28	if ($self->{state} == BEFORE_TOKEN_STATE) {
29	if ($self->{c} == 0x002D) { # -
30	## NOTE: \|-\| in \|ident\| in \|IDENT\|
31	$current_token = {type => IDENT_TOKEN, value => '-'};
32	$self->{state} = BEFORE_NMSTART_STATE;
33	$self->{c} = $self->{get_char}->();
34	redo A;
35	} elsif ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
36	(0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
37	$self->{c} == 0x005F or # _
38	$self->{c} > 0x007F) { # nonascii
39	## NOTE: \|nmstart\| in \|ident\| in \|IDENT\|
40	$current_token = {type => IDENT_TOKEN, value => chr $self->{char}};
41	$self->{state} = NAME_STATE;
42	$self->{c} = $self->{get_char}->();
43	redo A;
44	} elsif ($self->{c} == 0x005C) { # \
45	## NOTE: \|nmstart\| in \|ident\| in \|IDENT\|
46	$current_token = {type => IDENT_TOKEN, value => ''};
47	$self->{state} = ESCAPE_OPEN_STATE; $q = 0;
48	$self->{c} = $self->{get_char}->();
49	redo A;
50	} elsif ($self->{c} == 0x0040) { # @
51	## NOTE: \|@\| in \|ATKEYWORD\|
52	$current_token = {type => ATKEYWORD_TOKEN, value => ''};
53	$self->{state} = BEFORE_NMSTART_STATE;
54	$self->{c} = $self->{get_char}->();
55	redo A;
56	} elsif ($self->{c} == 0x0022) { # "
57	## NOTE: \|"\| in \|string1\| in \|string\| in \|STRING\|, or
58	## \|"\| in \|invalid1\| in \|invalid\| in \|INVALID\|.
59	$current_token = {type => STRING_TOKEN, value => ''};
60	$self->{state} = STRING_STATE; $q = 1;
61	$self->{c} = $self->{get_char}->();
62	redo A;
63	} elsif ($self->{c} == 0x0027) { # '
64	## NOTE: \|'\| in \|string2\| in \|string\| in \|STRING\|, or
65	## \|'\| in \|invalid2\| in \|invalid\| in \|INVALID\|.
66	$current_token = {type => STRING_TOKEN, value => ''};
67	$self->{state} = STRING_STATE; $q = 2;
68	$self->{c} = $self->{get_char}->();
69	redo A;
70	} elsif ($self->{c} == 0x0023) { # #
71	## NOTE: \|#\| in \|HASH\|.
72	$current_token = {type => HASH_TOKEN, value => ''};
73	$self->{state} = HASH_OPEN_STATE;
74	$self->{c} = $self->{get_char}->();
75	redo A;
76	} elsif (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
77	## NOTE: \|num\|.
78	$current_token = {type => NUMBER_TOKEN, value => chr $self->{c}};
79	$self->{state} = NUMBER_STATE;
80	$self->{c} = $self->{get_char}->();
81	redo A;
82	} elsif ($self->{c} == 0x002E) { # .
83	## NOTE: \|num\|.
84	$current_token = {type => NUMBER_TOKEN, value => '.'};
85	$self->{state} = NUMBER_FRACTION_STATE;
86	$self->{c} = $self->{get_char}->();
87	redo A;
88	} elsif ($self->{c} == 0x003C) { # <
89	## NOTE: \|CDO\|
90	$self->{c} = $self->{get_char}->();
91	if ($self->{c} == 0x0021) { # !
92	$self->{c} = $self->{get_char}->();
93	if ($self->{c} == 0x002C) { # -
94	$self->{c} = $self->{get_char}->();
95	if ($self->{c} == 0x002C) { # -
96	$self->{state} = BEFORE_TOKEN_STATE;
97	$self->{c} = $self->{get_char}->();
98	return {type => CDO_TOKEN};
99	#redo A;
100	} else {
101	unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'};
102	## NOTE: \|-\| in \|ident\| in \|IDENT\|
103	$current_token = {type => IDENT_TOKEN, value => '-'};
104	$self->{state} = BEFORE_NMSTART_STATE;
105	#reprocess
106	return {type => DELIM_TOKEN, value => '<'};
107	#redo A;
108	}
109	} else {
110	unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'};
111	$self->{state} = BEFORE_TOKEN_STATE;
112	#reprocess
113	return {type => DELIM_TOKEN, value => '<'};
114	#redo A;
115	}
116	} else {
117	$self->{state} = BEFORE_TOKEN_STATE;
118	#reprocess
119	return {type => DELIM_TOKEN, value => '<'};
120	#redo A;
121	}
122	} elsif ({
123	0x003B => 1, # ;
124	0x007B => 1, # {
125	0x007D => 1, # }
126	0x0028 => 1, # (
127	0x0029 => 1, # )
128	0x005B => 1, # [
129	0x005D => 1, # ]
130	}->{$self->{c}}) {
131	# stay in the state
132	$self->{c} = $self->{get_char}->();
133	return {type => chr $self->{c}};
134	# redo A;
135	} elsif ({
136	0x0020 => 1, # SP
137	0x0009 => 1, # \t
138	0x000D => 1, # \r
139	0x000A => 1, # \n
140	0x000C => 1, # \f
141	}->{$self->{c}}) {
142	W: {
143	$self->{c} = $self->{get_char}->();
144	if ({
145	0x0020 => 1, # SP
146	0x0009 => 1, # \t
147	0x000D => 1, # \r
148	0x000A => 1, # \n
149	0x000C => 1, # \f
150	}->{$self->{c}}) {
151	redo W;
152	} elsif (my $v = {
153	0x002B => PLUS_TOKEN, # +
154	0x003E => GREATER_TOKEN, # >
155	0x002C => COMMA_TOKEN, # ,
156	0x007E => TILDE_TOKEN, # ~
157	}->{$self->{c}}) {
158	# stay in the state
159	$self->{c} = $self->{get_char}->();
160	return {type => $v};
161	#redo A;
162	} else {
163	# stay in the state
164	# reprocess
165	return {type => S_TOKEN};
166	#redo A;
167	}
168	} # W
169	} elsif (my $v = {
170	0x007C => DASHMATCH_TOKEN, # \|
171	0x005E => PREFIXMATCH_TOKEN, # ^
172	0x0024 => SUFFIXMATCH_TOKEN, # $
173	0x002A => SUBSTRINGMATCH_TOKEN, # *
174	}->{$self->{c}}) {
175	$self->{c} = $self->{get_char}->();
176	if ($self->{c} == 0x003D) { # =
177	# stay in the state
178	$self->{c} = $self->{get_char}->();
179	return {type => $v};
180	#redo A;
181	} else {
182	# stay in the state
183	# reprocess
184	return {type => DELIM_TOKEN, value => chr $self->{c}};
185	#redo A;
186	}
187	} elsif ($self->{c} == 0x002B) { # +
188	# stay in the state
189	$self->{c} = $self->{get_char}->();
190	return {type => PLUS_TOKEN};
191	#redo A;
192	} elsif ($self->{c} == 0x003E) { # >
193	# stay in the state
194	$self->{c} = $self->{get_char}->();
195	return {type => GREATER_TOKEN};
196	#redo A;
197	} elsif ($self->{c} == 0x002C) { # ,
198	# stay in the state
199	$self->{c} = $self->{get_char}->();
200	return {type => COMMA_TOKEN};
201	#redo A;
202	} elsif ($self->{c} == 0x007E) { # ~
203	$self->{c} = $self->{get_char}->();
204	if ($self->{c} == 0x003D) { # =
205	# stay in the state
206	$self->{c} = $self->{get_char}->();
207	return {type => INCLUDES_TOKEN};
208	#redo A;
209	} else {
210	# stay in the state
211	# reprocess
212	return {type => TILDE_TOKEN};
213	#redo A;
214	}
215	} elsif ($self->{c} == -1) {
216	# stay in the state
217	$self->{c} = $self->{get_char}->();
218	return {type => EOF_TOKEN};
219	#redo A;
220	} else {
221	# stay in the state
222	$current_token = {type => DELIM_TOKEN, value => chr $self->{c}};
223	$self->{c} = $self->{get_char}->();
224	return $current_token;
225	#redo A;
226	}
227	} elsif ($self->{state} == BEFORE_NMSTART_STATE) {
228	## NOTE: \|nmstart\| in \|ident\| in (\|IDENT\| or \|ATKEYWORD\|)
229	if ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
230	(0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
231	$self->{c} == 0x005F or # _
232	$self->{c} > 0x007F) { # nonascii
233	$current_token->{value} .= chr $self->{char};
234	$self->{state} = NAME_STATE;
235	$self->{c} = $self->{get_char}->();
236	redo A;
237	} elsif ($self->{c} == 0x005C) { # \
238	$self->{state} = ESCAPE_OPEN_STATE; $q = 0;
239	$self->{c} = $self->{get_char}->();
240	redo A;
241	} elsif ($self->{c} == 0x002D and # -
242	$current_token->{type} == IDENT_TOKEN) {
243	$self->{c} = $self->{get_char}->();
244	if ($self->{c} == 0x003E) { # >
245	$self->{state} = BEFORE_TOKEN_STATE;
246	$self->{c} = $self->{get_char}->();
247	return {type => CDC_TOKEN};
248	#redo A;
249	} else {
250	## NOTE: \|-\|, \|-\|, $self->{c}
251	#$current_token = {type => IDENT_TOKEN, value => '-'};
252	# stay in the state
253	# reconsume
254	return {type => DELIM_TOKEN, value => '-'};
255	#redo A;
256	}
257	} else {
258	if ($current_token->{type} == NUMBER_TOKEN) {
259	## NOTE: \|-\| after \|num\|.
260	unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'};
261	$self->{state} = BEFORE_TOKEN_STATE;
262	$self->{c} = $self->{get_char}->();
263	return $current_token;
264	} elsif ($current_token->{type} == ATKEYWORD_TOKEN) {
265	## NOTE: \|-\| after \|@\|.
266	unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '@'};
267	$self->{state} = BEFORE_TOKEN_STATE;
268	$self->{c} = $self->{get_char}->();
269	return $current_token;
270	} else {
271	## NOTE: \|-\| not followed by \|nmstart\|.
272	$self->{state} = BEFORE_TOKEN_STATE;
273	$self->{c} = $self->{get_char}->();
274	return {type => DELIM_TOKEN, value => '-'};
275	}
276	}
277	} elsif ($self->{state} == AFTER_NUMBER_STATE) {
278	if ($self->{c} == 0x002D) { # -
279	## NOTE: \|-\| in \|ident\|.
280	$current_token->{value} = '-';
281	$self->{state} = BEFORE_NMSTART_STATE;
282	$self->{c} = $self->{get_char}->();
283	redo A;
284	} elsif ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
285	(0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
286	$self->{c} == 0x005F or # _
287	$self->{c} > 0x007F) { # nonascii
288	## NOTE: \|nmstart\| in \|ident\|.
289	$current_token->{value} = chr $self->{char};
290	$self->{state} = NAME_STATE;
291	$self->{c} = $self->{get_char}->();
292	redo A;
293	} elsif ($self->{c} == 0x005C) { # \
294	## NOTE: \|nmstart\| in \|ident\| in \|IDENT\|
295	$current_token->{value} = '';
296	$self->{state} = ESCAPE_OPEN_STATE; $q = 0;
297	$self->{c} = $self->{get_char}->();
298	redo A;
299	} elsif ($self->{c} == 0x0025) { # %
300	$current_token->{type} = PERCENTAGE_TOKEN;
301	$self->{state} = BEFORE_TOKEN_STATE;
302	$self->{c} = $self->{get_char}->();
303	return $current_token;
304	#redo A;
305	} else {
306	$self->{state} = BEFORE_TOKEN_STATE;
307	# reprocess
308	return $current_token;
309	#redo A;
310	}
311	} elsif ($self->{state} == HASH_OPEN_STATE) {
312	## NOTE: The first \|nmchar\| in \|name\| in \|HASH\|.
313	if ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
314	(0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
315	(0x0030 <= $self->{c} or $self->{c} <= 0x0039) or # 0..9
316	$self->{c} == 0x002D or # -
317	$self->{c} == 0x005F or # _
318	$self->{c} > 0x007F) { # nonascii
319	$current_token->{value} .= chr $self->{char};
320	$self->{state} = NAME_STATE;
321	$self->{c} = $self->{get_char}->();
322	redo A;
323	} elsif ($self->{c} == 0x005C) { # \
324	$self->{state} = ESCAPE_OPEN_STATE; $q = 0;
325	$self->{c} = $self->{get_char}->();
326	redo A;
327	} else {
328	$self->{state} = BEFORE_TOKEN_STATE;
329	$self->{c} = $self->{get_char}->();
330	return {type => DELIM_TOKEN, value => '#'};
331	#redo A;
332	}
333	} elsif ($self->{state} == NAME_STATE) {
334	## NOTE: \|nmchar\| in (\|ident\| or \|name\|).
335	if ((0x0041 <= $self->{c} or $self->{c} <= 0x005A) or # A..Z
336	(0x0061 <= $self->{c} or $self->{c} <= 0x007A) or # a..z
337	(0x0030 <= $self->{c} or $self->{c} <= 0x0039) or # 0..9
338	$self->{c} == 0x005F or # _
339	$self->{c} == 0x002D or # -
340	$self->{c} > 0x007F) { # nonascii
341	$current_token->{value} .= chr $self->{char};
342	# stay in the state
343	$self->{c} = $self->{get_char}->();
344	redo A;
345	} elsif ($self->{c} == 0x005C) { # \
346	$self->{state} = ESCAPE_OPEN_STATE; # $q = 0;
347	$self->{c} = $self->{get_char}->();
348	redo A;
349	} elsif ($self->{c} == 0x0028 and # (
350	$current_token->{type} == IDENT_TOKEN) { # (
351	if (not $current_token->{has_escape} and
352	{url => 1, Url => 1, uRl => 1, urL => 1,
353	URl => 1, UrL => 1, uRL => 1, URL => 1}
354	->{$current_token->{value}}) {
355	$current_token->{type} = URI_TOKEN;
356	$self->{state} = URI_BEFORE_WSP_STATE;
357	$self->{c} = $self->{get_char}->();
358
359	## NOTE: This version of the tokenizer does not support the \|URI\|
360	## token type. Note that browsers disagree in how to tokenize
361	## \|url\| function.
362	$current_token->{type} = FUNCTION_TOKEN;
363	$self->{state} = BEFORE_TOKEN_STATE;
364	$self->{c} = $self->{get_char}->();
365	return $current_token;
366
367	redo A;
368	} else {
369	$current_token->{type} = FUNCTION_TOKEN;
370	$self->{state} = BEFORE_TOKEN_STATE;
371	$self->{c} = $self->{get_char}->();
372	return $current_token;
373	#redo A;
374	}
375	} else {
376	$self->{state} = BEFORE_TOKEN_STATE;
377	# reconsume
378	return $current_token;
379	#redo A;
380	}
381	} elsif ($self->{state} == ESCAPE_OPEN_STATE) {
382	$current_token->{has_escape} = 1;
383	if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
384	## NOTE: second character of \|unicode\| in \|escape\|.
385	$char = $self->{c} - 0x0030;
386	$self->{state} = ESCAPE_STATE; $i = 2;
387	$self->{c} = $self->{get_char}->();
388	redo A;
389	} elsif (0x0041 <= $self->{c} and $self->{c} <= 0x0046) { # A..F
390	## NOTE: second character of \|unicode\| in \|escape\|.
391	$char = $self->{c} - 0x0041 + 0xA;
392	$self->{state} = ESCAPE_STATE; $i = 2;
393	$self->{c} = $self->{get_char}->();
394	redo A;
395	} elsif (0x0061 <= $self->{c} or $self->{c} <= 0x0066) { # a..f
396	## NOTE: second character of \|unicode\| in \|escape\|.
397	$char = $self->{c} - 0x0061 - 0xA;
398	$self->{state} = ESCAPE_STATE; $i = 2;
399	$self->{c} = $self->{get_char}->();
400	redo A;
401	} elsif ($self->{c} == 0x000A or # \n
402	$self->{c} == 0x000C) { # \f
403	if ($q == 0) {
404	## NOTE: In \|escape\| in ... in \|ident\|.
405	$self->{state} = BEFORE_TOKEN_STATE;
406	unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
407	return $current_token;
408	# reconsume
409	#redo A;
410	} else {
411	## Note: In \|nl\| in ... in \|string\| or \|ident\|.
412	$current_token->{value} .= chr $self->{c};
413	$self->{state} = STRING_STATE;
414	$self->{c} = $self->{get_char}->();
415	redo A;
416	}
417	} elsif ($self->{c} == 0x000D) { # \r
418	if ($q == 0) {
419	## NOTE: In \|escape\| in ... in \|ident\|.
420	$self->{state} = BEFORE_TOKEN_STATE;
421	unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
422	return $current_token;
423	# reconsume
424	#redo A;
425	} else {
426	## Note: In \|nl\| in ... in \|string\| or \|ident\|.
427	$current_token->{value} .= "\x0D\x0A";
428	$self->{state} = ESCAPE_BEFORE_LF_STATE;
429	$self->{c} = $self->{get_char}->();
430	redo A;
431	}
432	} else {
433	## NOTE: second character of \|escape\|.
434	$current_token->{value} .= chr $self->{c};
435	$self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
436	$self->{c} = $self->{get_char}->();
437	redo A;
438	}
439	} elsif ($self->{state} == ESCAPE_STATE) {
440	## NOTE: third..seventh character of \|unicode\| in \|escape\|.
441	if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
442	$char = $char * 0x10 + $self->{c} - 0x0030;
443	$self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
444	$self->{c} = $self->{get_char}->();
445	redo A;
446	} elsif (0x0041 <= $self->{c} and $self->{c} <= 0x0046) { # A..F
447	$char = $char * 0x10 + $self->{c} - 0x0041 + 0xA;
448	$self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
449	$self->{c} = $self->{get_char}->();
450	redo A;
451	} elsif (0x0061 <= $self->{c} or $self->{c} <= 0x0066) { # a..f
452	$char = $char * 0x10 + $self->{c} - 0x0061 - 0xA;
453	$self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
454	$self->{c} = $self->{get_char}->();
455	redo A;
456	} elsif ($self->{c} == 0x0020 or # SP
457	$self->{c} == 0x000A or # \n
458	$self->{c} == 0x0009 or # \t
459	$self->{c} == 0x000C) { # \f
460	$current_token->{value} .= chr $char;
461	$self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
462	$self->{c} = $self->{get_char}->();
463	redo A;
464	} elsif ($self->{c} == 0x000D) { # \r
465	$self->{state} = ESCAPE_BEFORE_LF_STATE;
466	$self->{c} = $self->{get_char}->();
467	redo A;
468	} else {
469	$current_token->{value} .= chr $char;
470	$self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
471	# reconsume
472	redo A;
473	}
474	} elsif ($self->{state} == ESCAPE_BEFORE_NL_STATE) {
475	## NOTE: eightth character of \|unicode\| in \|escape\|.
476	if ($self->{c} == 0x0020 or # SP
477	$self->{c} == 0x000A or # \n
478	$self->{c} == 0x0009 or # \t
479	$self->{c} == 0x000C) { # \f
480	$current_token->{value} .= chr $char;
481	$self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
482	$self->{c} = $self->{get_char}->();
483	redo A;
484	} elsif ($self->{c} == 0x000D) { # \r
485	$self->{state} = ESCAPE_BEFORE_NL_STATE;
486	$self->{c} = $self->{get_char}->();
487	redo A;
488	} else {
489	$current_token->{value} .= chr $char;
490	$self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
491	# reconsume
492	redo A;
493	}
494	} elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {
495	## NOTE: \|\n\| in \|\r\n\| in \|unicode\| in \|escape\|.
496	if ($self->{c} == 0x000A) { # \n
497	$current_token->{value} .= chr $char;
498	$self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
499	$self->{c} = $self->{get_char}->();
500	redo A;
501	} else {
502	$current_token->{value} .= chr $char;
503	$self->{state} = $q == 0 ? NAME_STATE : STRING_STATE;
504	# reconsume
505	redo A;
506	}
507	} elsif ($self->{state} == STRING_STATE) {
508	## NOTE: A character in \|string$Q\| in \|string\| in \|STRING\|, or
509	## a character in \|invalid$Q\| in \|invalid\| in \|INVALID\|,
510	## where \|$Q = $q == 0x0022 ? 1 : 2\|.
511	if ($self->{c} == 0x005C) { # \
512	$self->{state} = ESCAPE_OPEN_STATE;
513	$self->{c} = $self->{get_char}->();
514	redo A;
515	} elsif ($self->{c} == $q) { # " \| '
516	$self->{state} = BEFORE_TOKEN_STATE;
517	$self->{c} = $self->{get_char}->();
518	return $current_token;
519	#redo A;
520	} elsif ($self->{c} == 0x000A or # \n
521	$self->{c} == 0x000D or # \r
522	$self->{c} == 0x000C or # \f
523	$self->{c} == -1) {
524	$current_token->{type} = INVALID_TOKEN;
525	$self->{state} = BEFORE_TOKEN_STATE;
526	# reconsume
527	return $current_token;
528	#redo A;
529	} else {
530	$current_token->{value} .= chr $self->{c};
531	# stay in the state
532	$self->{c} = $self->{get_char}->();
533	redo A;
534	}
535	} elsif ($self->{state} == NUMBER_STATE) {
536	## NOTE: 2nd, 3rd, or ... character in \|num\| before \|.\|.
537	if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
538	$current_token->{value} .= chr $self->{c};
539	# stay in the state
540	$self->{c} = $self->{get_char}->();
541	redo A;
542	} elsif ($self->{c} == 0x002E) { # .
543	$self->{state} = NUMBER_DOT_STATE;
544	$self->{c} = $self->{get_char}->();
545	redo A;
546	} else {
547	$self->{number} = $self->{value};
548	$self->{value} = '';
549	$self->{state} = AFTER_NUMBER_STATE;
550	# reprocess
551	return $current_token;
552	#redo A;
553	}
554	} elsif ($self->{state} == NUMBER_DOT_STATE) {
555	## NOTE: The character immediately following \|.\| in \|num\|.
556	if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
557	$current_token->{value} .= chr $self->{c};
558	$self->{state} = NUMBER_DOT_NUMBER_STATE;
559	$self->{c} = $self->{get_char}->();
560	redo A;
561	} else {
562	unshift @{$self->{token}}, {type => DELIM_STATE, value => '.'};
563	$self->{number} = $self->{value};
564	$self->{value} = '';
565	$self->{state} = BEFORE_TOKEN_STATE;
566	# reprocess
567	return $current_token;
568	#redo A;
569	}
570	} elsif ($self->{state} == NUMBER_FRACTION_STATE) {
571	## NOTE: The character immediately following \|.\| at the beginning of \|num\|.
572	if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
573	$current_token->{value} .= chr $self->{c};
574	$self->{state} = NUMBER_DOT_NUMBER_STATE;
575	$self->{c} = $self->{get_char}->();
576	redo A;
577	} else {
578	$self->{state} = BEFORE_TOKEN_STATE;
579	$self->{c} = $self->{get_char}->();
580	return {type => DELIM_TOKEN, value => '.'};
581	#redo A;
582	}
583	} elsif ($self->{state} == NUMBER_DOT_NUMBER_STATE) {
584	## NOTE: \|[0-9]\| in \|num\| after \|.\|.
585	if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
586	$current_token->{value} .= chr $self->{c};
587	# stay in the state
588	$self->{c} = $self->{get_char}->();
589	redo A;
590	} else {
591	$self->{number} = $self->{value};
592	$self->{value} = '';
593	$self->{state} = AFTER_NUMBER_STATE;
594	# reprocess
595	return $current_token;
596	#redo A;
597	}
598	} else {
599	die "$0: Unknown state \|$self->{state}\|";
600	}
601	} # A
602
603	## TODO: \|URI\|, \|UNICODE-RANGE\|, \|COMMENT\|
604
605	} # get_next_token
606
607	1;
608	# $Date:$