/[suikacvs]/markup/html/whatpm/Whatpm/CSS/Tokenizer.pm
Suika

Diff of /markup/html/whatpm/Whatpm/CSS/Tokenizer.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.4 by wakaba, Sat Sep 8 02:58:24 2007 UTC revision 1.17 by wakaba, Sun Jan 20 04:02:25 2008 UTC
# Line 1  Line 1 
1  package Whatpm::CSS::Tokenizer;  package Whatpm::CSS::Tokenizer;
2  use strict;  use strict;
3    our $VERSION=do{my @r=(q$Revision$=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
4    
5    require Exporter;
6    push our @ISA, 'Exporter';
7    
8  sub BEFORE_TOKEN_STATE () { 0 }  sub BEFORE_TOKEN_STATE () { 0 }
9  sub BEFORE_NMSTART_STATE () { 1 }  sub BEFORE_NMSTART_STATE () { 1 }
# Line 36  sub NUMBER_TOKEN () { 11 } Line 40  sub NUMBER_TOKEN () { 11 }
40  sub DIMENSION_TOKEN () { 12 }  sub DIMENSION_TOKEN () { 12 }
41  sub PERCENTAGE_TOKEN () { 13 }  sub PERCENTAGE_TOKEN () { 13 }
42  sub UNICODE_RANGE_TOKEN () { 14 }  sub UNICODE_RANGE_TOKEN () { 14 }
 sub UNICODE_RANGE_INVALID_TOKEN () { 15 }  
43  sub DELIM_TOKEN () { 16 }  sub DELIM_TOKEN () { 16 }
44  sub PLUS_TOKEN () { 17 }  sub PLUS_TOKEN () { 17 }
45  sub GREATER_TOKEN () { 18 }  sub GREATER_TOKEN () { 18 }
# Line 60  sub CDC_TOKEN () { 35 } Line 63  sub CDC_TOKEN () { 35 }
63  sub COMMENT_TOKEN () { 36 }  sub COMMENT_TOKEN () { 36 }
64  sub COMMENT_INVALID_TOKEN () { 37 }  sub COMMENT_INVALID_TOKEN () { 37 }
65  sub EOF_TOKEN () { 38 }  sub EOF_TOKEN () { 38 }
66    sub MINUS_TOKEN () { 39 }
67    sub STAR_TOKEN () { 40 }
68    sub VBAR_TOKEN () { 41 }
69    sub DOT_TOKEN () { 42 }
70    sub COLON_TOKEN () { 43 }
71    sub MATCH_TOKEN () { 44 }
72    sub EXCLAMATION_TOKEN () { 45 }
73    
74  our @TokenName = qw(  our @TokenName = qw(
75    0 IDENT ATKEYWORD HASH FUNCTION URI URI_INVALID URI_PREFIX URI_PREFIX_INVALID    0 IDENT ATKEYWORD HASH FUNCTION URI URI_INVALID URI_PREFIX URI_PREFIX_INVALID
76    STRING INVALID NUMBER DIMENSION PERCENTAGE UNICODE_RANGE    STRING INVALID NUMBER DIMENSION PERCENTAGE UNICODE_RANGE
77    UNICODE_RANGE_INVALID DELIM PLUS GREATER COMMA TILDE DASHMATCH    0 DELIM PLUS GREATER COMMA TILDE DASHMATCH
78    PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH INCLUDES SEMICOLON    PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH INCLUDES SEMICOLON
79    LBRACE RBRACE LPAREN RPAREN LBRACKET RBRACKET S CDO CDC COMMENT    LBRACE RBRACE LPAREN RPAREN LBRACKET RBRACKET S CDO CDC COMMENT
80    COMMENT_INVALID EOF    COMMENT_INVALID EOF MINUS STAR VBAR DOT COLON MATCH EXCLAMATION
81    );
82    
83    our @EXPORT_OK = qw(
84      IDENT_TOKEN ATKEYWORD_TOKEN HASH_TOKEN FUNCTION_TOKEN URI_TOKEN
85      URI_INVALID_TOKEN URI_PREFIX_TOKEN URI_PREFIX_INVALID_TOKEN
86      STRING_TOKEN INVALID_TOKEN NUMBER_TOKEN DIMENSION_TOKEN PERCENTAGE_TOKEN
87      UNICODE_RANGE_TOKEN DELIM_TOKEN PLUS_TOKEN GREATER_TOKEN COMMA_TOKEN
88      TILDE_TOKEN DASHMATCH_TOKEN PREFIXMATCH_TOKEN SUFFIXMATCH_TOKEN
89      SUBSTRINGMATCH_TOKEN INCLUDES_TOKEN SEMICOLON_TOKEN LBRACE_TOKEN
90      RBRACE_TOKEN LPAREN_TOKEN RPAREN_TOKEN LBRACKET_TOKEN RBRACKET_TOKEN
91      S_TOKEN CDO_TOKEN CDC_TOKEN COMMENT_TOKEN COMMENT_INVALID_TOKEN EOF_TOKEN
92      MINUS_TOKEN STAR_TOKEN VBAR_TOKEN DOT_TOKEN COLON_TOKEN MATCH_TOKEN
93      EXCLAMATION_TOKEN
94  );  );
95    
96    our %EXPORT_TAGS = ('token' => [@EXPORT_OK]);
97    
98  sub new ($) {  sub new ($) {
99    my $self = bless {token => [], get_char => sub { -1 },    my $self = bless {token => [], get_char => sub { -1 }}, shift;
                     onerror => sub { }}, shift;  
100    return $self;    return $self;
101  } # new  } # new
102    
# Line 80  sub init ($) { Line 104  sub init ($) {
104    my $self = shift;    my $self = shift;
105    $self->{state} = BEFORE_TOKEN_STATE;    $self->{state} = BEFORE_TOKEN_STATE;
106    $self->{c} = $self->{get_char}->();    $self->{c} = $self->{get_char}->();
107      #$self->{t} = {type => token-type, value => value, number => number};
108  } # init  } # init
109    
110  sub get_next_token ($) {  sub get_next_token ($) {
# Line 88  sub get_next_token ($) { Line 113  sub get_next_token ($) {
113      return shift @{$self->{token}};      return shift @{$self->{token}};
114    }    }
115    
   my $current_token;  
116    my $char;    my $char;
117    my $num; # |{num}|, if any.    my $num; # |{num}|, if any.
118    my $i; # |$i + 1|th character in |unicode| in |escape|.    my $i; # |$i + 1|th character in |unicode| in |escape|.
# Line 103  sub get_next_token ($) { Line 127  sub get_next_token ($) {
127      if ($self->{state} == BEFORE_TOKEN_STATE) {      if ($self->{state} == BEFORE_TOKEN_STATE) {
128        if ($self->{c} == 0x002D) { # -        if ($self->{c} == 0x002D) { # -
129          ## NOTE: |-| in |ident| in |IDENT|          ## NOTE: |-| in |ident| in |IDENT|
130          $current_token = {type => IDENT_TOKEN, value => '-'};          $self->{t} = {type => IDENT_TOKEN, value => '-', hyphen => 1};
131          $self->{state} = BEFORE_NMSTART_STATE;          $self->{state} = BEFORE_NMSTART_STATE;
132          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
133          redo A;          redo A;
134          } elsif ($self->{c} == 0x0055 or $self->{c} == 0x0075) { # U or u
135            $self->{t} = {type => IDENT_TOKEN, value => chr $self->{c}};
136            $self->{c} = $self->{get_char}->();
137            if ($self->{c} == 0x002B) { # +
138              $self->{c} = $self->{get_char}->();
139              if ((0x0030 <= $self->{c} and $self->{c} <= 0x0039) or # 0..9
140                  (0x0041 <= $self->{c} and $self->{c} <= 0x0046) or # A..F
141                  (0x0061 <= $self->{c} and $self->{c} <= 0x0066) or # a..f
142                  $self->{c} == 0x003F) { # ?
143                $self->{t}->{value} = chr $self->{c};
144                $self->{t}->{type} = UNICODE_RANGE_TOKEN;
145                $self->{c} = $self->{get_char}->();
146                C: for (2..6) {
147                  if ((0x0030 <= $self->{c} and $self->{c} <= 0x0039) or # 0..9
148                      (0x0041 <= $self->{c} and $self->{c} <= 0x0046) or # A..F
149                      (0x0061 <= $self->{c} and $self->{c} <= 0x0066) or # a..f
150                      $self->{c} == 0x003F) { # ?
151                    $self->{t}->{value} .= chr $self->{c};
152                    $self->{c} = $self->{get_char}->();
153                  } else {
154                    last C;
155                  }
156                } # C
157    
158                if ($self->{c} == 0x002D) { # -
159                  $self->{c} = $self->{get_char}->();
160                  if ((0x0030 <= $self->{c} and $self->{c} <= 0x0039) or # 0..9
161                      (0x0041 <= $self->{c} and $self->{c} <= 0x0046) or # A..F
162                      (0x0061 <= $self->{c} and $self->{c} <= 0x0066)) { # a..f
163                    $self->{t}->{value} .= '-' . chr $self->{c};
164                    $self->{c} = $self->{get_char}->();
165                    C: for (2..6) {
166                      if ((0x0030 <= $self->{c} and $self->{c} <= 0x0039) or # 0..9
167                          (0x0041 <= $self->{c} and $self->{c} <= 0x0046) or # A..F
168                          (0x0061 <= $self->{c} and $self->{c} <= 0x0066)) { # a..f
169                        $self->{t}->{value} .= chr $self->{c};
170                        $self->{c} = $self->{get_char}->();
171                      } else {
172                        last C;
173                      }
174                    } # C
175                    
176                    #
177                  } else {
178                    my $token = $self->{t};
179                    $self->{t} = {type => IDENT_TOKEN, value => '-'};
180                    $self->{state} = BEFORE_NMSTART_STATE;
181                    # reprocess
182                    return $token;
183                    #redo A;
184                  }
185                }
186    
187                $self->{state} = BEFORE_TOKEN_STATE;
188                # reprocess
189                return $self->{t};
190                #redo A;
191              } else {
192                unshift @{$self->{token}}, {type => PLUS_TOKEN};
193                $self->{state} = BEFORE_TOKEN_STATE;
194                # reprocess
195                return $self->{t};
196                #redo A;
197              }
198            } else {
199              $self->{state} = NAME_STATE;
200              # reprocess
201              redo A;
202            }
203        } elsif ((0x0041 <= $self->{c} and $self->{c} <= 0x005A) or # A..Z        } elsif ((0x0041 <= $self->{c} and $self->{c} <= 0x005A) or # A..Z
204                 (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z                 (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z
205                 $self->{c} == 0x005F or # _                 $self->{c} == 0x005F or # _
206                 $self->{c} > 0x007F) { # nonascii                 $self->{c} > 0x007F) { # nonascii
207          ## NOTE: |nmstart| in |ident| in |IDENT|          ## NOTE: |nmstart| in |ident| in |IDENT|
208          $current_token = {type => IDENT_TOKEN, value => chr $self->{c}};          $self->{t} = {type => IDENT_TOKEN, value => chr $self->{c}};
209          $self->{state} = NAME_STATE;          $self->{state} = NAME_STATE;
210          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
211          redo A;          redo A;
212        } elsif ($self->{c} == 0x005C) { # \        } elsif ($self->{c} == 0x005C) { # \
213          ## NOTE: |nmstart| in |ident| in |IDENT|          ## NOTE: |nmstart| in |ident| in |IDENT|
214          $current_token = {type => IDENT_TOKEN, value => ''};          $self->{t} = {type => IDENT_TOKEN, value => ''};
215          $self->{state} = ESCAPE_OPEN_STATE; $q = 0;          $self->{state} = ESCAPE_OPEN_STATE; $q = 0;
216          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
217          redo A;          redo A;
218        } elsif ($self->{c} == 0x0040) { # @        } elsif ($self->{c} == 0x0040) { # @
219          ## NOTE: |@| in |ATKEYWORD|          ## NOTE: |@| in |ATKEYWORD|
220          $current_token = {type => ATKEYWORD_TOKEN, value => ''};          $self->{t} = {type => ATKEYWORD_TOKEN, value => ''};
221          $self->{state} = AFTER_AT_STATE;          $self->{state} = AFTER_AT_STATE;
222          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
223          redo A;          redo A;
224        } elsif ($self->{c} == 0x0022 or $self->{c} == 0x0027) { # " or '        } elsif ($self->{c} == 0x0022 or $self->{c} == 0x0027) { # " or '
225          $current_token = {type => STRING_TOKEN, value => ''};          $self->{t} = {type => STRING_TOKEN, value => ''};
226          $self->{state} = STRING_STATE; $q = $self->{c};          $self->{state} = STRING_STATE; $q = $self->{c};
227          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
228          redo A;          redo A;
229        } elsif ($self->{c} == 0x0023) { # #        } elsif ($self->{c} == 0x0023) { # #
230          ## NOTE: |#| in |HASH|.          ## NOTE: |#| in |HASH|.
231          $current_token = {type => HASH_TOKEN, value => ''};          $self->{t} = {type => HASH_TOKEN, value => ''};
232          $self->{state} = HASH_OPEN_STATE;          $self->{state} = HASH_OPEN_STATE;
233          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
234          redo A;          redo A;
235        } elsif (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9        } elsif (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
236          ## NOTE: |num|.          ## NOTE: |num|.
237          $current_token = {type => NUMBER_TOKEN, value => chr $self->{c}};          $self->{t} = {type => NUMBER_TOKEN, value => chr $self->{c}};
238          $self->{state} = NUMBER_STATE;          $self->{state} = NUMBER_STATE;
239          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
240          redo A;          redo A;
241        } elsif ($self->{c} == 0x002E) { # .        } elsif ($self->{c} == 0x002E) { # .
242          ## NOTE: |num|.          ## NOTE: |num|.
243          $current_token = {type => NUMBER_TOKEN, value => '0'};          $self->{t} = {type => NUMBER_TOKEN, value => '0'};
244          $self->{state} = NUMBER_FRACTION_STATE;          $self->{state} = NUMBER_FRACTION_STATE;
245          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
246          redo A;          redo A;
# Line 183  sub get_next_token ($) { Line 276  sub get_next_token ($) {
276          } else {          } else {
277            # stay in the state.            # stay in the state.
278            # reprocess            # reprocess
279            return {type => DELIM_STATE, value => '/'};            return {type => DELIM_TOKEN, value => '/'};
280            #redo A;            #redo A;
281          }                  }        
282        } elsif ($self->{c} == 0x003C) { # <        } elsif ($self->{c} == 0x003C) { # <
# Line 191  sub get_next_token ($) { Line 284  sub get_next_token ($) {
284          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
285          if ($self->{c} == 0x0021) { # !          if ($self->{c} == 0x0021) { # !
286            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
287            if ($self->{c} == 0x002C) { # -            if ($self->{c} == 0x002D) { # -
288              $self->{c} = $self->{get_char}->();              $self->{c} = $self->{get_char}->();
289              if ($self->{c} == 0x002C) { # -              if ($self->{c} == 0x002D) { # -
290                $self->{state} = BEFORE_TOKEN_STATE;                $self->{state} = BEFORE_TOKEN_STATE;
291                $self->{c} = $self->{get_char}->();                $self->{c} = $self->{get_char}->();
292                return {type => CDO_TOKEN};                return {type => CDO_TOKEN};
293                #redo A;                #redo A;
294              } else {              } else {
295                unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'};                unshift @{$self->{token}}, {type => EXCLAMATION_TOKEN};
296                ## NOTE: |-| in |ident| in |IDENT|                ## NOTE: |-| in |ident| in |IDENT|
297                $current_token = {type => IDENT_TOKEN, value => '-'};                $self->{t} = {type => IDENT_TOKEN, value => '-'};
298                $self->{state} = BEFORE_NMSTART_STATE;                $self->{state} = BEFORE_NMSTART_STATE;
299                #reprocess                #reprocess
300                return {type => DELIM_TOKEN, value => '<'};                return {type => DELIM_TOKEN, value => '<'};
301                #redo A;                #redo A;
302              }              }
303            } else {            } else {
304              unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'};              unshift @{$self->{token}}, {type => EXCLAMATION_TOKEN};
305              $self->{state} = BEFORE_TOKEN_STATE;              $self->{state} = BEFORE_TOKEN_STATE;
306              #reprocess              #reprocess
307              return {type => DELIM_TOKEN, value => '<'};              return {type => DELIM_TOKEN, value => '<'};
# Line 221  sub get_next_token ($) { Line 314  sub get_next_token ($) {
314            #redo A;            #redo A;
315          }          }
316        } elsif (my $t = {        } elsif (my $t = {
317                  0x003B => SEMICOLON_TOKEN, # ;                          0x0021 => EXCLAMATION_TOKEN, # !
318                  0x007B => LBRACE_TOKEN, # {                          0x002D => MINUS_TOKEN, # -
319                  0x007D => RBRACE_TOKEN, # }                          0x002E => DOT_TOKEN, # .
320                  0x0028 => LPAREN_TOKEN, # (                          0x003A => COLON_TOKEN, # :
321                  0x0029 => RPAREN_TOKEN, # )                          0x003B => SEMICOLON_TOKEN, # ;
322                  0x005B => LBRACKET_TOKEN, # [                          0x003D => MATCH_TOKEN, # =
323                  0x005D => RBRACKET_TOKEN, # ]                          0x007B => LBRACE_TOKEN, # {
324                            0x007D => RBRACE_TOKEN, # }
325                            0x0028 => LPAREN_TOKEN, # (
326                            0x0029 => RPAREN_TOKEN, # )
327                            0x005B => LBRACKET_TOKEN, # [
328                            0x005D => RBRACKET_TOKEN, # ]
329                 }->{$self->{c}}) {                 }->{$self->{c}}) {
330            my ($l, $c) = ($self->{line}, $self->{column});
331          # stay in the state          # stay in the state
332          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->($self);
333          return {type => $t};          return {type => $t, line => $l, column => $c};
334          # redo A;          # redo A;
335        } elsif ({        } elsif ({
336                  0x0020 => 1, # SP                  0x0020 => 1, # SP
# Line 280  sub get_next_token ($) { Line 379  sub get_next_token ($) {
379            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
380            return {type => $v};            return {type => $v};
381            #redo A;            #redo A;
382            } elsif ($v = {
383                           0x002A => STAR_TOKEN, # *
384                           0x007C => VBAR_TOKEN, # |
385                          }->{$c}) {
386              # stay in the state.
387              # reprocess
388              return {type => $v};
389              #redo A;
390          } else {          } else {
391            # stay in the state            # stay in the state
392            # reprocess            # reprocess
# Line 321  sub get_next_token ($) { Line 428  sub get_next_token ($) {
428          #redo A;          #redo A;
429        } else {        } else {
430          # stay in the state          # stay in the state
431          $current_token = {type => DELIM_TOKEN, value => chr $self->{c}};          $self->{t} = {type => DELIM_TOKEN, value => chr $self->{c}};
432          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
433          return $current_token;          return $self->{t};
434          #redo A;          #redo A;
435        }        }
436      } elsif ($self->{state} == BEFORE_NMSTART_STATE) {      } elsif ($self->{state} == BEFORE_NMSTART_STATE) {
# Line 333  sub get_next_token ($) { Line 440  sub get_next_token ($) {
440            (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z            (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z
441            $self->{c} == 0x005F or # _            $self->{c} == 0x005F or # _
442            $self->{c} > 0x007F) { # nonascii            $self->{c} > 0x007F) { # nonascii
443          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
444          $current_token->{type} = DIMENSION_TOKEN          $self->{t}->{type} = DIMENSION_TOKEN
445              if $current_token->{type} == NUMBER_TOKEN;              if $self->{t}->{type} == NUMBER_TOKEN;
446          $self->{state} = NAME_STATE;          $self->{state} = NAME_STATE;
447          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
448          redo A;          redo A;
449        } elsif ($self->{c} == 0x005C) { # \        } elsif ($self->{c} == 0x005C) { # \
 ## TODO: 12-\X, 12-\{nl}  
450          $self->{state} = ESCAPE_OPEN_STATE; $q = 0;          $self->{state} = ESCAPE_OPEN_STATE; $q = 0;
451          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
452          redo A;          redo A;
453        } elsif ($self->{c} == 0x002D and # -        } elsif ($self->{c} == 0x002D) { # -
454                 $current_token->{type} == IDENT_TOKEN) {          if ($self->{t}->{type} == IDENT_TOKEN) {
         $self->{c} = $self->{get_char}->();  
         if ($self->{c} == 0x003E) { # >  
           $self->{state} = BEFORE_TOKEN_STATE;  
455            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
456            return {type => CDC_TOKEN};            if ($self->{c} == 0x003E) { # >
457            #redo A;              $self->{state} = BEFORE_TOKEN_STATE;
458                $self->{c} = $self->{get_char}->();
459                return {type => CDC_TOKEN};
460                #redo A;
461              } else {
462                ## NOTE: |-|, |-|, $self->{c}
463                #$self->{t} = {type => IDENT_TOKEN, value => '-'};
464                # stay in the state
465                # reconsume
466                return {type => MINUS_TOKEN};
467                #redo A;
468              }
469            } elsif ($self->{t}->{type} == DIMENSION_TOKEN) {
470              $self->{c} = $self->{get_char}->();
471              if ($self->{c} == 0x003E) { # >
472                unshift @{$self->{token}}, {type => CDC_TOKEN};
473                $self->{t}->{type} = NUMBER_TOKEN;
474                $self->{t}->{value} = '';
475                $self->{state} = BEFORE_TOKEN_STATE;
476                $self->{c} = $self->{get_char}->();
477                return $self->{t};
478                #redo A;
479              } else {
480                ## NOTE: |-|, |-|, $self->{c}
481                my $t = $self->{t};
482                $t->{type} = NUMBER_TOKEN;
483                $t->{value} = '';
484                $self->{t} = {type => IDENT_TOKEN, value => '-', hyphen => 1};
485                unshift @{$self->{token}}, {type => MINUS_TOKEN};
486                # stay in the state
487                # reconsume
488                return $t;
489                #redo A;
490              }
491          } else {          } else {
492            ## NOTE: |-|, |-|, $self->{c}            #
           #$current_token = {type => IDENT_TOKEN, value => '-'};  
           # stay in the state  
           # reconsume  
           return {type => DELIM_TOKEN, value => '-'};  
           #redo A;  
493          }          }
494        } else {        } else {
495          if ($current_token->{type} == NUMBER_TOKEN) {          #
496            ## NOTE: |-| after |NUMBER|.        }
497            unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'};        
498            $self->{state} = BEFORE_TOKEN_STATE;        if ($self->{t}->{type} == DIMENSION_TOKEN) {
499            # reconsume          ## NOTE: |-| after |NUMBER|.
500            $current_token->{value} = $current_token->{number};          unshift @{$self->{token}}, {type => MINUS_TOKEN};
501            delete $current_token->{number};          $self->{state} = BEFORE_TOKEN_STATE;
502            return $current_token;          # reprocess
503          } else {          $self->{t}->{type} = NUMBER_TOKEN;
504            ## NOTE: |-| not followed by |nmstart|.          $self->{t}->{value} = '';
505            $self->{state} = BEFORE_TOKEN_STATE;          return $self->{t};
506            $self->{c} = $self->{get_char}->();        } else {
507            return {type => DELIM_TOKEN, value => '-'};          ## NOTE: |-| not followed by |nmstart|.
508          }          $self->{state} = BEFORE_TOKEN_STATE;
509            # reprocess
510            return {type => MINUS_TOKEN};
511        }        }
512      } elsif ($self->{state} == AFTER_AT_STATE) {      } elsif ($self->{state} == AFTER_AT_STATE) {
513        if ((0x0041 <= $self->{c} and $self->{c} <= 0x005A) or # A..Z        if ((0x0041 <= $self->{c} and $self->{c} <= 0x005A) or # A..Z
514            (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z            (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z
515            $self->{c} == 0x005F or # _            $self->{c} == 0x005F or # _
516            $self->{c} > 0x007F) { # nonascii            $self->{c} > 0x007F) { # nonascii
517          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
518          $self->{state} = NAME_STATE;          $self->{state} = NAME_STATE;
519          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
520          redo A;          redo A;
521        } elsif ($self->{c} == 0x002D) { # -        } elsif ($self->{c} == 0x002D) { # -
522          $current_token->{value} .= '-';          $self->{t}->{value} .= '-';
523          $self->{state} = AFTER_AT_HYPHEN_STATE;          $self->{state} = AFTER_AT_HYPHEN_STATE;
524          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
525          redo A;          redo A;
# Line 404  sub get_next_token ($) { Line 537  sub get_next_token ($) {
537            (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z            (0x0061 <= $self->{c} and $self->{c} <= 0x007A) or # a..z
538            $self->{c} == 0x005F or # _            $self->{c} == 0x005F or # _
539            $self->{c} > 0x007F) { # nonascii            $self->{c} > 0x007F) { # nonascii
540          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
541          $self->{state} = NAME_STATE;          $self->{state} = NAME_STATE;
542          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
543          redo A;          redo A;
# Line 417  sub get_next_token ($) { Line 550  sub get_next_token ($) {
550            return {type => DELIM_TOKEN, value => '@'};            return {type => DELIM_TOKEN, value => '@'};
551            #redo A;            #redo A;
552          } else {          } else {
553            unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'};            unshift @{$self->{token}}, {type => MINUS_TOKEN};
554            $current_token = {type => IDENT_TOKEN, value => '-'};            $self->{t} = {type => IDENT_TOKEN, value => '-'};
555            $self->{state} = BEFORE_NMSTART_STATE;            $self->{state} = BEFORE_NMSTART_STATE;
556            # reprocess            # reprocess
557            return {type => DELIM_TOKEN, value => '@'};            return {type => DELIM_TOKEN, value => '@'};
# Line 430  sub get_next_token ($) { Line 563  sub get_next_token ($) {
563          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
564          redo A;          redo A;
565        } else {        } else {
566          unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'};          unshift @{$self->{token}}, {type => MINUS_TOKEN};
567          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
568          # reprocess          # reprocess
569          return {type => DELIM_TOKEN, value => '@'};          return {type => DELIM_TOKEN, value => '@'};
# Line 438  sub get_next_token ($) { Line 571  sub get_next_token ($) {
571      } elsif ($self->{state} == AFTER_NUMBER_STATE) {      } elsif ($self->{state} == AFTER_NUMBER_STATE) {
572        if ($self->{c} == 0x002D) { # -        if ($self->{c} == 0x002D) { # -
573          ## NOTE: |-| in |ident|.          ## NOTE: |-| in |ident|.
574          $current_token->{value} = '-';          $self->{t}->{hyphen} = 1;
575            $self->{t}->{value} = '-';
576            $self->{t}->{type} = DIMENSION_TOKEN;
577          $self->{state} = BEFORE_NMSTART_STATE;          $self->{state} = BEFORE_NMSTART_STATE;
578          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
579          redo A;          redo A;
# Line 447  sub get_next_token ($) { Line 582  sub get_next_token ($) {
582                 $self->{c} == 0x005F or # _                 $self->{c} == 0x005F or # _
583                 $self->{c} > 0x007F) { # nonascii                 $self->{c} > 0x007F) { # nonascii
584          ## NOTE: |nmstart| in |ident|.          ## NOTE: |nmstart| in |ident|.
585          $current_token->{value} = chr $self->{c};          $self->{t}->{value} = chr $self->{c};
586          $current_token->{type} = DIMENSION_TOKEN;          $self->{t}->{type} = DIMENSION_TOKEN;
587          $self->{state} = NAME_STATE;          $self->{state} = NAME_STATE;
588          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
589          redo A;          redo A;
590        } elsif ($self->{c} == 0x005C) { # \        } elsif ($self->{c} == 0x005C) { # \
591          ## NOTE: |nmstart| in |ident| in |IDENT|          ## NOTE: |nmstart| in |ident| in |IDENT|
592          $current_token->{value} = '';          $self->{t}->{value} = '';
593            $self->{t}->{type} = DIMENSION_TOKEN;
594          $self->{state} = ESCAPE_OPEN_STATE; $q = 0;          $self->{state} = ESCAPE_OPEN_STATE; $q = 0;
595          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
596          redo A;          redo A;
597        } elsif ($self->{c} == 0x0025) { # %        } elsif ($self->{c} == 0x0025) { # %
598          $current_token->{type} = PERCENTAGE_TOKEN;          $self->{t}->{type} = PERCENTAGE_TOKEN;
599          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
600          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
601          return $current_token;          return $self->{t};
602          #redo A;          #redo A;
603        } else {        } else {
604          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
605          # reprocess          # reprocess
606          return $current_token;          return $self->{t};
607          #redo A;          #redo A;
608        }        }
609      } elsif ($self->{state} == HASH_OPEN_STATE) {      } elsif ($self->{state} == HASH_OPEN_STATE) {
# Line 478  sub get_next_token ($) { Line 614  sub get_next_token ($) {
614            $self->{c} == 0x002D or # -            $self->{c} == 0x002D or # -
615            $self->{c} == 0x005F or # _            $self->{c} == 0x005F or # _
616            $self->{c} > 0x007F) { # nonascii            $self->{c} > 0x007F) { # nonascii
617          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
618          $self->{state} = NAME_STATE;          $self->{state} = NAME_STATE;
619          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
620          redo A;          redo A;
# Line 488  sub get_next_token ($) { Line 624  sub get_next_token ($) {
624          redo A;          redo A;
625        } else {        } else {
626          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
627          $self->{c} = $self->{get_char}->();          # reprocess
628          return {type => DELIM_TOKEN, value => '#'};          return {type => DELIM_TOKEN, value => '#'};
629          #redo A;          #redo A;
630        }        }
# Line 500  sub get_next_token ($) { Line 636  sub get_next_token ($) {
636            $self->{c} == 0x005F or # _            $self->{c} == 0x005F or # _
637            $self->{c} == 0x002D or # -            $self->{c} == 0x002D or # -
638            $self->{c} > 0x007F) { # nonascii            $self->{c} > 0x007F) { # nonascii
639          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
640          # stay in the state          # stay in the state
641          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
642          redo A;          redo A;
# Line 509  sub get_next_token ($) { Line 645  sub get_next_token ($) {
645          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
646          redo A;          redo A;
647        } elsif ($self->{c} == 0x0028 and # (        } elsif ($self->{c} == 0x0028 and # (
648                 $current_token->{type} == IDENT_TOKEN) { # (                 $self->{t}->{type} == IDENT_TOKEN) { # (
649          my $func_name = $current_token->{value};          my $func_name = $self->{t}->{value};
650          $func_name =~ tr/A-Z/a-z/; ## TODO: Unicode or ASCII case-insensitive?          $func_name =~ tr/A-Z/a-z/; ## TODO: Unicode or ASCII case-insensitive?
651          if ($func_name eq 'url' or $func_name eq 'url-prefix') {          if ($func_name eq 'url' or $func_name eq 'url-prefix') {
652            if ($current_token->{has_escape}) {            if ($self->{t}->{has_escape}) {
653              ## TODO: warn              ## TODO: warn
654            }            }
655            $current_token->{type}            $self->{t}->{type}
656                = $func_name eq 'url' ? URI_TOKEN : URI_PREFIX_TOKEN;                = $func_name eq 'url' ? URI_TOKEN : URI_PREFIX_TOKEN;
657            $current_token->{value} = '';            $self->{t}->{value} = '';
658            $self->{state} = URI_BEFORE_WSP_STATE;            $self->{state} = URI_BEFORE_WSP_STATE;
659            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
660            redo A;            redo A;
661          } else {          } else {
662            $current_token->{type} = FUNCTION_TOKEN;            $self->{t}->{type} = FUNCTION_TOKEN;
663            $self->{state} = BEFORE_TOKEN_STATE;            $self->{state} = BEFORE_TOKEN_STATE;
664            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
665            return $current_token;            return $self->{t};
666            #redo A;            #redo A;
667          }          }
668        } else {        } else {
669          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
670          # reconsume          # reconsume
671          return $current_token;          return $self->{t};
672          #redo A;          #redo A;
673        }        }
674      } elsif ($self->{state} == URI_BEFORE_WSP_STATE) {      } elsif ($self->{state} == URI_BEFORE_WSP_STATE) {
# Line 546  sub get_next_token ($) { Line 682  sub get_next_token ($) {
682          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
683        }        }
684        if ($self->{c} == -1) {        if ($self->{c} == -1) {
685          $current_token->{type} = {          $self->{t}->{type} = {
686              URI_TOKEN, URI_INVALID_TOKEN,              URI_TOKEN, URI_INVALID_TOKEN,
687              URI_INVALID_TOKEN, URI_INVALID_TOKEN,              URI_INVALID_TOKEN, URI_INVALID_TOKEN,
688              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
689              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
690          }->{$current_token->{type}};                  }->{$self->{t}->{type}};        
691          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
692          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
693          return $current_token;          return $self->{t};
694          #redo A;          #redo A;
695        } elsif ($self->{c} < 0x0020 or $self->{c} == 0x0028) { # C0 or (        } elsif ($self->{c} < 0x0020 or $self->{c} == 0x0028) { # C0 or (
696          ## TODO: Should we consider matches of "(" and ")"?          ## TODO: Should we consider matches of "(" and ")"?
697          $current_token->{type} = {          $self->{t}->{type} = {
698              URI_TOKEN, URI_INVALID_TOKEN,              URI_TOKEN, URI_INVALID_TOKEN,
699              URI_INVALID_TOKEN, URI_INVALID_TOKEN,              URI_INVALID_TOKEN, URI_INVALID_TOKEN,
700              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
701              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
702          }->{$current_token->{type}};          }->{$self->{t}->{type}};
703          $self->{state} = URI_UNQUOTED_STATE;          $self->{state} = URI_UNQUOTED_STATE;
704          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
705          redo A;          redo A;
# Line 574  sub get_next_token ($) { Line 710  sub get_next_token ($) {
710        } elsif ($self->{c} == 0x0029) { # )        } elsif ($self->{c} == 0x0029) { # )
711          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
712          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
713          return $current_token;          return $self->{t};
714          #redo A;          #redo A;
715        } elsif ($self->{c} == 0x005C) { # \        } elsif ($self->{c} == 0x005C) { # \
716          $self->{state} = ESCAPE_OPEN_STATE; $q = 1;          $self->{state} = ESCAPE_OPEN_STATE; $q = 1;
717          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
718          redo A;          redo A;
719        } else {        } else {
720          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
721          $self->{state} = URI_UNQUOTED_STATE;          $self->{state} = URI_UNQUOTED_STATE;
722          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
723          redo A;          redo A;
# Line 598  sub get_next_token ($) { Line 734  sub get_next_token ($) {
734          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
735          redo A;          redo A;
736        } elsif ($self->{c} == -1) {        } elsif ($self->{c} == -1) {
737          $current_token->{type} = {          $self->{t}->{type} = {
738              URI_TOKEN, URI_INVALID_TOKEN,              URI_TOKEN, URI_INVALID_TOKEN,
739              URI_INVALID_TOKEN, URI_INVALID_TOKEN,              URI_INVALID_TOKEN, URI_INVALID_TOKEN,
740              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
741              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
742          }->{$current_token->{type}};                  }->{$self->{t}->{type}};        
743          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
744          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
745          return $current_token;          return $self->{t};
746          #redo A;          #redo A;
747        } elsif ($self->{c} < 0x0020 or {        } elsif ($self->{c} < 0x0020 or {
748            0x0022 => 1, # "            0x0022 => 1, # "
# Line 614  sub get_next_token ($) { Line 750  sub get_next_token ($) {
750            0x0028 => 1, # (            0x0028 => 1, # (
751        }->{$self->{c}}) { # C0 or (        }->{$self->{c}}) { # C0 or (
752          ## TODO: Should we consider matches of "(" and ")", '"', or "'"?          ## TODO: Should we consider matches of "(" and ")", '"', or "'"?
753          $current_token->{type} = {          $self->{t}->{type} = {
754              URI_TOKEN, URI_INVALID_TOKEN,              URI_TOKEN, URI_INVALID_TOKEN,
755              URI_INVALID_TOKEN, URI_INVALID_TOKEN,              URI_INVALID_TOKEN, URI_INVALID_TOKEN,
756              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
757              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
758          }->{$current_token->{type}};          }->{$self->{t}->{type}};
759          # stay in the state.          # stay in the state.
760          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
761          redo A;          redo A;
762        } elsif ($self->{c} == 0x0029) { # )        } elsif ($self->{c} == 0x0029) { # )
763          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
764          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
765          return $current_token;          return $self->{t};
766          #redo A;          #redo A;
767        } elsif ($self->{c} == 0x005C) { # \        } elsif ($self->{c} == 0x005C) { # \
768          $self->{state} = ESCAPE_OPEN_STATE; $q = 1;          $self->{state} = ESCAPE_OPEN_STATE; $q = 1;
769          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
770          redo A;          redo A;
771        } else {        } else {
772          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
773          # stay in the state.          # stay in the state.
774          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
775          redo A;          redo A;
# Line 650  sub get_next_token ($) { Line 786  sub get_next_token ($) {
786          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
787          redo A;          redo A;
788        } elsif ($self->{c} == -1) {        } elsif ($self->{c} == -1) {
789          $current_token->{type} = {          $self->{t}->{type} = {
790              URI_TOKEN, URI_INVALID_TOKEN,              URI_TOKEN, URI_INVALID_TOKEN,
791              URI_INVALID_TOKEN, URI_INVALID_TOKEN,              URI_INVALID_TOKEN, URI_INVALID_TOKEN,
792              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
793              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
794          }->{$current_token->{type}};                  }->{$self->{t}->{type}};        
795          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
796          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
797          return $current_token;          return $self->{t};
798          #redo A;          #redo A;
799        } elsif ($self->{c} == 0x0029) { # )        } elsif ($self->{c} == 0x0029) { # )
800          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
801          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
802          return $current_token;          return $self->{t};
803          #redo A;          #redo A;
804        } elsif ($self->{c} == 0x005C) { # \        } elsif ($self->{c} == 0x005C) { # \
805          $self->{state} = ESCAPE_OPEN_STATE; $q = 1;          $self->{state} = ESCAPE_OPEN_STATE; $q = 1;
# Line 671  sub get_next_token ($) { Line 807  sub get_next_token ($) {
807          redo A;          redo A;
808        } else {        } else {
809          ## TODO: Should we consider matches of "(" and ")", '"', or "'"?          ## TODO: Should we consider matches of "(" and ")", '"', or "'"?
810          $current_token->{type} = {          $self->{t}->{type} = {
811              URI_TOKEN, URI_INVALID_TOKEN,              URI_TOKEN, URI_INVALID_TOKEN,
812              URI_INVALID_TOKEN, URI_INVALID_TOKEN,              URI_INVALID_TOKEN, URI_INVALID_TOKEN,
813              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
814              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
815          }->{$current_token->{type}};          }->{$self->{t}->{type}};
816          # stay in the state.          # stay in the state.
817          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
818          redo A;          redo A;
819        }        }
820      } elsif ($self->{state} == ESCAPE_OPEN_STATE) {      } elsif ($self->{state} == ESCAPE_OPEN_STATE) {
821        $current_token->{has_escape} = 1;        $self->{t}->{has_escape} = 1;
822        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
823          ## NOTE: second character of |unicode| in |escape|.          ## NOTE: second character of |unicode| in |escape|.
824          $char = $self->{c} - 0x0030;          $char = $self->{c} - 0x0030;
# Line 697  sub get_next_token ($) { Line 833  sub get_next_token ($) {
833          redo A;          redo A;
834        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f
835          ## NOTE: second character of |unicode| in |escape|.          ## NOTE: second character of |unicode| in |escape|.
836          $char = $self->{c} - 0x0061 - 0xA;          $char = $self->{c} - 0x0061 + 0xA;
837          $self->{state} = ESCAPE_STATE; $i = 2;          $self->{state} = ESCAPE_STATE; $i = 2;
838          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
839          redo A;          redo A;
840        } elsif ($self->{c} == 0x000A or # \n        } elsif ($self->{c} == 0x000A or # \n
841                 $self->{c} == 0x000C) { # \f                 $self->{c} == 0x000C) { # \f
842          if ($q == 0) {          if ($q == 0) {
843            ## NOTE: In |escape| in ... in |ident|.            #
           $self->{state} = BEFORE_TOKEN_STATE;  
           unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};  
           return $current_token;  
           # reconsume  
           #redo A;  
844          } elsif ($q == 1) {          } elsif ($q == 1) {
845            ## NOTE: In |escape| in |URI|.            ## NOTE: In |escape| in |URI|.
846            $current_token->{type} = {            $self->{t}->{type} = {
847                URI_TOKEN, URI_INVALID_TOKEN,                URI_TOKEN, URI_INVALID_TOKEN,
848                URI_INVALID_TOKEN, URI_INVALID_TOKEN,                URI_INVALID_TOKEN, URI_INVALID_TOKEN,
849                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
850                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
851            }->{$current_token->{type}};            }->{$self->{t}->{type}};
852            $current_token->{value} .= chr $self->{c};            $self->{t}->{value} .= chr $self->{c};
853            $self->{state} = URI_UNQUOTED_STATE;            $self->{state} = URI_UNQUOTED_STATE;
854            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
855            redo A;            redo A;
856          } else {          } else {
857            ## Note: In |nl| in ... in |string| or |ident|.            ## Note: In |nl| in ... in |string| or |ident|.
           $current_token->{value} .= chr $self->{c};  
858            $self->{state} = STRING_STATE;            $self->{state} = STRING_STATE;
859            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
860            redo A;            redo A;
861          }          }
862        } elsif ($self->{c} == 0x000D) { # \r        } elsif ($self->{c} == 0x000D) { # \r
863          if ($q == 0) {          if ($q == 0) {
864            ## NOTE: In |escape| in ... in |ident|.            #
           $self->{state} = BEFORE_TOKEN_STATE;  
           unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};  
           return $current_token;  
           # reconsume  
           #redo A;  
865          } elsif ($q == 1) {          } elsif ($q == 1) {
866            $current_token->{type} = {            ## NOTE: In |escape| in |URI|.
867              $self->{t}->{type} = {
868                URI_TOKEN, URI_INVALID_TOKEN,                URI_TOKEN, URI_INVALID_TOKEN,
869                URI_INVALID_TOKEN, URI_INVALID_TOKEN,                URI_INVALID_TOKEN, URI_INVALID_TOKEN,
870                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
871                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
872            }->{$current_token->{type}};            }->{$self->{t}->{type}};
873            $current_token->{value} .= "\x0D\x0A";            $self->{state} = ESCAPE_BEFORE_LF_STATE;
           $self->{state} = URI_UNQUOTED_STATE;  
874            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
875            redo A;            redo A;
876          } else {          } else {
877            ## Note: In |nl| in ... in |string| or |ident|.            ## Note: In |nl| in ... in |string| or |ident|.
           $current_token->{value} .= "\x0D\x0A";  
878            $self->{state} = ESCAPE_BEFORE_LF_STATE;            $self->{state} = ESCAPE_BEFORE_LF_STATE;
879            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
880            redo A;            redo A;
881          }          }
882          } elsif ($self->{c} == -1) {
883            #
884        } else {        } else {
885          ## NOTE: second character of |escape|.          ## NOTE: second character of |escape|.
886          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
887          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
888              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
889          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
890          redo A;          redo A;
891        }        }
892    
893          if ($q == 0) {
894            if ($self->{t}->{type} == DIMENSION_TOKEN) {
895              if ($self->{t}->{hyphen} and $self->{t}->{value} eq '-') {
896                $self->{state} = BEFORE_TOKEN_STATE;
897                # reprocess
898                unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
899                unshift @{$self->{token}}, {type => MINUS_TOKEN};
900                $self->{t}->{type} = NUMBER_TOKEN;
901                $self->{t}->{value} = '';
902                return $self->{t};
903                #redo A;
904              } elsif (length $self->{t}->{value}) {
905                $self->{state} = BEFORE_TOKEN_STATE;
906                # reprocess
907                unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
908                return $self->{t};
909                #redo A;
910              } else {
911                $self->{state} = BEFORE_TOKEN_STATE;
912                # reprocess
913                unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
914                $self->{t}->{type} = NUMBER_TOKEN;
915                $self->{t}->{value} = '';
916                return $self->{t};
917                #redo A;
918              }
919            } else {
920              if ($self->{t}->{hyphen} and $self->{t}->{value} eq '-') {
921                $self->{state} = BEFORE_TOKEN_STATE;
922                # reprocess
923                unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
924                return {type => MINUS_TOKEN};
925                #redo A;
926              } elsif (length $self->{t}->{value}) {
927                $self->{state} = BEFORE_TOKEN_STATE;
928                # reprocess
929                unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
930                return $self->{t};
931                #redo A;
932              } else {
933                $self->{state} = BEFORE_TOKEN_STATE;
934                # reprocess
935                return {type => DELIM_TOKEN, value => '\\'};
936                #redo A;
937              }
938            }
939          } elsif ($q == 1) {
940            $self->{state} = URI_UNQUOTED_STATE;
941            $self->{c} = $self->{get_char}->();
942            redo A;
943          } else {
944            unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
945            $self->{t}->{type} = {
946              STRING_TOKEN, INVALID_TOKEN,
947              URI_TOKEN, URI_INVALID_TOKEN,
948              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
949            }->{$self->{t}->{type}} || $self->{t}->{type};
950            $self->{state} = BEFORE_TOKEN_STATE;
951            # reprocess
952            return $self->{t};
953            #redo A;
954          }
955      } elsif ($self->{state} == ESCAPE_STATE) {      } elsif ($self->{state} == ESCAPE_STATE) {
956        ## NOTE: third..seventh character of |unicode| in |escape|.        ## NOTE: third..seventh character of |unicode| in |escape|.
957        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
# Line 776  sub get_next_token ($) { Line 965  sub get_next_token ($) {
965          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
966          redo A;          redo A;
967        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f
968          $char = $char * 0x10 + $self->{c} - 0x0061 - 0xA;          $char = $char * 0x10 + $self->{c} - 0x0061 + 0xA;
969          $self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;          $self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
970          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
971          redo A;          redo A;
# Line 784  sub get_next_token ($) { Line 973  sub get_next_token ($) {
973                 $self->{c} == 0x000A or # \n                 $self->{c} == 0x000A or # \n
974                 $self->{c} == 0x0009 or # \t                 $self->{c} == 0x0009 or # \t
975                 $self->{c} == 0x000C) { # \f                 $self->{c} == 0x000C) { # \f
976          $current_token->{value} .= chr $char;          $self->{t}->{value} .= chr $char;
977          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
978              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
979          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
# Line 794  sub get_next_token ($) { Line 983  sub get_next_token ($) {
983          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
984          redo A;          redo A;
985        } else {        } else {
986          $current_token->{value} .= chr $char;          $self->{t}->{value} .= chr $char;
987          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
988              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
989          # reconsume          # reconsume
# Line 806  sub get_next_token ($) { Line 995  sub get_next_token ($) {
995            $self->{c} == 0x000A or # \n            $self->{c} == 0x000A or # \n
996            $self->{c} == 0x0009 or # \t            $self->{c} == 0x0009 or # \t
997            $self->{c} == 0x000C) { # \f            $self->{c} == 0x000C) { # \f
998          $current_token->{value} .= chr $char;          $self->{t}->{value} .= chr $char;
999          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
1000              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
1001          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
# Line 816  sub get_next_token ($) { Line 1005  sub get_next_token ($) {
1005          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1006          redo A;          redo A;
1007        } else {        } else {
1008          $current_token->{value} .= chr $char;          $self->{t}->{value} .= chr $char;
1009          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
1010              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
1011          # reconsume          # reconsume
1012          redo A;          redo A;
1013        }        }
1014      } elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {      } elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {
1015        ## NOTE: |\n| in |\r\n| in |unicode| in |escape|.        ## NOTE: |\n| in |\r\n| in |nl| in |escape|.
1016        if ($self->{c} == 0x000A) { # \n        if ($self->{c} == 0x000A) { # \n
         $current_token->{value} .= chr $char;  
1017          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
1018              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
1019          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1020          redo A;          redo A;
1021        } else {        } else {
         $current_token->{value} .= chr $char;  
1022          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
1023              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
1024          # reconsume          # reprocess
1025          redo A;          redo A;
1026        }        }
1027      } elsif ($self->{state} == STRING_STATE) {      } elsif ($self->{state} == STRING_STATE) {
# Line 847  sub get_next_token ($) { Line 1034  sub get_next_token ($) {
1034          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1035          redo A;          redo A;
1036        } elsif ($self->{c} == $q) { # " | '        } elsif ($self->{c} == $q) { # " | '
1037          if ($current_token->{type} == STRING_TOKEN) {          if ($self->{t}->{type} == STRING_TOKEN) {
1038            $self->{state} = BEFORE_TOKEN_STATE;            $self->{state} = BEFORE_TOKEN_STATE;
1039            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
1040            return $current_token;            return $self->{t};
1041            #redo A;            #redo A;
1042          } else {          } else {
1043            $self->{state} = URI_AFTER_WSP_STATE;            $self->{state} = URI_AFTER_WSP_STATE;
# Line 861  sub get_next_token ($) { Line 1048  sub get_next_token ($) {
1048                 $self->{c} == 0x000D or # \r                 $self->{c} == 0x000D or # \r
1049                 $self->{c} == 0x000C or # \f                 $self->{c} == 0x000C or # \f
1050                 $self->{c} == -1) {                 $self->{c} == -1) {
1051          $current_token->{type} = INVALID_TOKEN;          $self->{t}->{type} = {
1052              STRING_TOKEN, INVALID_TOKEN,
1053              INVALID_TOKEN, INVALID_TOKEN,
1054              URI_TOKEN, URI_INVALID_TOKEN,
1055              URI_INVALID_TOKEN, URI_INVALID_TOKEN,
1056              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
1057              URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
1058            }->{$self->{t}->{type}};
1059          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
1060          # reconsume          # reconsume
1061          return $current_token;          return $self->{t};
1062          #redo A;          #redo A;
1063        } else {        } else {
1064          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
1065          # stay in the state          # stay in the state
1066          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1067          redo A;          redo A;
# Line 875  sub get_next_token ($) { Line 1069  sub get_next_token ($) {
1069      } elsif ($self->{state} == NUMBER_STATE) {      } elsif ($self->{state} == NUMBER_STATE) {
1070        ## NOTE: 2nd, 3rd, or ... character in |num| before |.|.        ## NOTE: 2nd, 3rd, or ... character in |num| before |.|.
1071        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
1072          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
1073          # stay in the state          # stay in the state
1074          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1075          redo A;          redo A;
# Line 884  sub get_next_token ($) { Line 1078  sub get_next_token ($) {
1078          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1079          redo A;          redo A;
1080        } else {        } else {
1081          $current_token->{number} = $current_token->{value};          $self->{t}->{number} = $self->{t}->{value};
1082          $current_token->{value} = '';          $self->{t}->{value} = '';
1083          $self->{state} = AFTER_NUMBER_STATE;          $self->{state} = AFTER_NUMBER_STATE;
1084          # reprocess          # reprocess
1085          redo A;          redo A;
# Line 893  sub get_next_token ($) { Line 1087  sub get_next_token ($) {
1087      } elsif ($self->{state} == NUMBER_DOT_STATE) {      } elsif ($self->{state} == NUMBER_DOT_STATE) {
1088        ## NOTE: The character immediately following |.| in |num|.        ## NOTE: The character immediately following |.| in |num|.
1089        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
1090          $current_token->{value} .= '.' . chr $self->{c};          $self->{t}->{value} .= '.' . chr $self->{c};
1091          $self->{state} = NUMBER_DOT_NUMBER_STATE;          $self->{state} = NUMBER_DOT_NUMBER_STATE;
1092          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1093          redo A;          redo A;
1094        } else {        } else {
1095          unshift @{$self->{token}}, {type => DELIM_STATE, value => '.'};          unshift @{$self->{token}}, {type => DOT_TOKEN};
1096          $current_token->{number} = $current_token->{value};          $self->{t}->{number} = $self->{t}->{value};
1097          $current_token->{value} = '';          $self->{t}->{value} = '';
1098          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
1099          # reprocess          # reprocess
1100          return $current_token;          return $self->{t};
1101          #redo A;          #redo A;
1102        }        }
1103      } elsif ($self->{state} == NUMBER_FRACTION_STATE) {      } elsif ($self->{state} == NUMBER_FRACTION_STATE) {
1104        ## NOTE: The character immediately following |.| at the beginning of |num|.        ## NOTE: The character immediately following |.| at the beginning of |num|.
1105        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
1106          $current_token->{value} .= '.' . chr $self->{c};          $self->{t}->{value} .= '.' . chr $self->{c};
1107          $self->{state} = NUMBER_DOT_NUMBER_STATE;          $self->{state} = NUMBER_DOT_NUMBER_STATE;
1108          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1109          redo A;          redo A;
1110        } else {        } else {
1111          $self->{state} = BEFORE_TOKEN_STATE;          $self->{state} = BEFORE_TOKEN_STATE;
1112          $self->{c} = $self->{get_char}->();          # reprocess
1113          return {type => DELIM_TOKEN, value => '.'};          return {type => DOT_TOKEN};
1114          #redo A;          #redo A;
1115        }        }
1116      } elsif ($self->{state} == NUMBER_DOT_NUMBER_STATE) {      } elsif ($self->{state} == NUMBER_DOT_NUMBER_STATE) {
1117        ## NOTE: |[0-9]| in |num| after |.|.        ## NOTE: |[0-9]| in |num| after |.|.
1118        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) {
1119          $current_token->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
1120          # stay in the state          # stay in the state
1121          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
1122          redo A;          redo A;
1123        } else {        } else {
1124          $current_token->{number} = $current_token->{value};          $self->{t}->{number} = $self->{t}->{value};
1125          $current_token->{value} = '';          $self->{t}->{value} = '';
1126          $self->{state} = AFTER_NUMBER_STATE;          $self->{state} = AFTER_NUMBER_STATE;
1127          # reprocess          # reprocess
1128          redo A;          redo A;
# Line 937  sub get_next_token ($) { Line 1131  sub get_next_token ($) {
1131        die "$0: Unknown state |$self->{state}|";        die "$0: Unknown state |$self->{state}|";
1132      }      }
1133    } # A    } # A
1134    } # get_next_token
1135    
1136    ## TODO: |URI|, |UNICODE-RANGE|, |COMMENT|  sub serialize_token ($$) {
1137      shift;
1138      my $t = shift;
1139    
1140  } # get_next_token    ## NOTE: This function is not intended for roundtrip-able serialization.
1141    
1142      if ($t->{type} == IDENT_TOKEN) {
1143        return $t->{value};
1144      } elsif ($t->{type} == ATKEYWORD_TOKEN) {
1145        return '@' . $t->{value};
1146      } elsif ($t->{type} == HASH_TOKEN) {
1147        return '#' . $t->{value};
1148      } elsif ($t->{type} == FUNCTION_TOKEN) {
1149        return $t->{value} . '(';
1150      } elsif ($t->{type} == URI_TOKEN) {
1151        return 'url(' . $t->{value} . ')';
1152      } elsif ($t->{type} == URI_INVALID_TOKEN) {
1153        return 'url(' . $t->{value};
1154      } elsif ($t->{type} == URI_PREFIX_TOKEN) {
1155        return 'url-prefix(' . $t->{value} . ')';
1156      } elsif ($t->{type} == URI_PREFIX_INVALID_TOKEN) {
1157        return 'url-prefix(' . $t->{value};
1158      } elsif ($t->{type} == STRING_TOKEN) {
1159        return '"' . $t->{value} . '"';
1160      } elsif ($t->{type} == INVALID_TOKEN) {
1161        return '"' . $t->{value};
1162      } elsif ($t->{type} == NUMBER_TOKEN) {
1163        return $t->{number};
1164      } elsif ($t->{type} == DIMENSION_TOKEN) {
1165        return $t->{number} . $t->{value};
1166      } elsif ($t->{type} == PERCENTAGE_TOKEN) {
1167        return $t->{number} . '%';
1168      } elsif ($t->{type} == UNICODE_RANGE_TOKEN) {
1169        return 'U+' . $t->{value};
1170      } elsif ($t->{type} == DELIM_TOKEN) {
1171        return $t->{value};
1172      } elsif ($t->{type} == PLUS_TOKEN) {
1173        return '+';
1174      } elsif ($t->{type} == GREATER_TOKEN) {
1175        return '>';
1176      } elsif ($t->{type} == COMMA_TOKEN) {
1177        return ',';
1178      } elsif ($t->{type} == TILDE_TOKEN) {
1179        return '~';
1180      } elsif ($t->{type} == DASHMATCH_TOKEN) {
1181        return '|=';
1182      } elsif ($t->{type} == PREFIXMATCH_TOKEN) {
1183        return '^=';
1184      } elsif ($t->{type} == SUFFIXMATCH_TOKEN) {
1185        return '$=';
1186      } elsif ($t->{type} == SUBSTRINGMATCH_TOKEN) {
1187        return '*=';
1188      } elsif ($t->{type} == INCLUDES_TOKEN) {
1189        return '~=';
1190      } elsif ($t->{type} == SEMICOLON_TOKEN) {
1191        return ';';
1192      } elsif ($t->{type} == LBRACE_TOKEN) {
1193        return '{';
1194      } elsif ($t->{type} == RBRACE_TOKEN) {
1195        return '}';
1196      } elsif ($t->{type} == LPAREN_TOKEN) {
1197        return '(';
1198      } elsif ($t->{type} == RPAREN_TOKEN) {
1199        return ')';
1200      } elsif ($t->{type} == LBRACKET_TOKEN) {
1201        return '[';
1202      } elsif ($t->{type} == RBRACKET_TOKEN) {
1203        return ']';
1204      } elsif ($t->{type} == S_TOKEN) {
1205        return ' ';
1206      } elsif ($t->{type} == CDO_TOKEN) {
1207        return '<!--';
1208      } elsif ($t->{type} == CDC_TOKEN) {
1209        return '-->';
1210      } elsif ($t->{type} == COMMENT_TOKEN) {
1211        return '/**/';
1212      } elsif ($t->{type} == COMMENT_INVALID_TOKEN) {
1213        return '/*';
1214      } elsif ($t->{type} == EOF_TOKEN) {
1215        return '{EOF}';
1216      } elsif ($t->{type} == MINUS_TOKEN) {
1217        return '-';
1218      } elsif ($t->{type} == STAR_TOKEN) {
1219        return '*';
1220      } elsif ($t->{type} == VBAR_TOKEN) {
1221        return '|';
1222      } elsif ($t->{type} == COLON_TOKEN) {
1223        return ':';
1224      } elsif ($t->{type} == MATCH_TOKEN) {
1225        return '=';
1226      } elsif ($t->{type} == EXCLAMATION_TOKEN) {
1227        return '!';
1228      } else {
1229        return '{'.$t->{type}.'}';
1230      }
1231    } # serialize_token
1232    
1233    =head1 LICENSE
1234    
1235    Copyright 2007 Wakaba <w@suika.fam.cx>
1236    
1237    This library is free software; you can redistribute it
1238    and/or modify it under the same terms as Perl itself.
1239    
1240    =cut
1241    
1242  1;  1;
1243  # $Date$  # $Date$
Legend:
Removed from v.1.4  
changed lines
  Added in v.1.17
admin@suikawiki.org
 ViewVC Help
Powered by ViewVC 1.1.24