/[suikacvs]/markup/html/whatpm/Whatpm/CSS/Tokenizer.pm
Suika

Diff of /markup/html/whatpm/Whatpm/CSS/Tokenizer.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.13 by wakaba, Sat Sep 8 17:43:41 2007 UTC revision 1.15 by wakaba, Sun Sep 30 12:03:09 2007 UTC
# Line 1  Line 1 
1  package Whatpm::CSS::Tokenizer;  package Whatpm::CSS::Tokenizer;
2  use strict;  use strict;
3    
4    require Exporter;
5    push our @ISA, 'Exporter';
6    
7  sub BEFORE_TOKEN_STATE () { 0 }  sub BEFORE_TOKEN_STATE () { 0 }
8  sub BEFORE_NMSTART_STATE () { 1 }  sub BEFORE_NMSTART_STATE () { 1 }
9  sub NAME_STATE () { 2 }  sub NAME_STATE () { 2 }
# Line 76  our @TokenName = qw( Line 79  our @TokenName = qw(
79    COMMENT_INVALID EOF MINUS STAR VBAR DOT COLON MATCH EXCLAMATION    COMMENT_INVALID EOF MINUS STAR VBAR DOT COLON MATCH EXCLAMATION
80  );  );
81    
82    our @EXPORT_OK = qw(
83      IDENT_TOKEN ATKEYWORD_TOKEN HASH_TOKEN FUNCTION_TOKEN URI_TOKEN
84      URI_INVALID_TOKEN URI_PREFIX_TOKEN URI_PREFIX_INVALID_TOKEN
85      STRING_TOKEN INVALID_TOKEN NUMBER_TOKEN DIMENSION_TOKEN PERCENTAGE_TOKEN
86      UNICODE_RANGE_TOKEN DELIM_TOKEN PLUS_TOKEN GREATER_TOKEN COMMA_TOKEN
87      TILDE_TOKEN DASHMATCH_TOKEN PREFIXMATCH_TOKEN SUFFIXMATCH_TOKEN
88      SUBSTRINGMATCH_TOKEN INCLUDES_TOKEN SEMICOLON_TOKEN LBRACE_TOKEN
89      RBRACE_TOKEN LPAREN_TOKEN RPAREN_TOKEN LBRACKET_TOKEN RBRACKET_TOKEN
90      S_TOKEN CDO_TOKEN CDC_TOKEN COMMENT_TOKEN COMMENT_INVALID_TOKEN EOF_TOKEN
91      MINUS_TOKEN STAR_TOKEN VBAR_TOKEN DOT_TOKEN COLON_TOKEN MATCH_TOKEN
92      EXCLAMATION_TOKEN
93    );
94    
95    our %EXPORT_TAGS = ('token' => [@EXPORT_OK]);
96    
97  sub new ($) {  sub new ($) {
98    my $self = bless {token => [], get_char => sub { -1 },    my $self = bless {token => [], get_char => sub { -1 },
99                      onerror => sub { }}, shift;                      onerror => sub { }}, shift;
# Line 836  sub get_next_token ($) { Line 854  sub get_next_token ($) {
854            redo A;            redo A;
855          } else {          } else {
856            ## Note: In |nl| in ... in |string| or |ident|.            ## Note: In |nl| in ... in |string| or |ident|.
           $self->{t}->{value} .= chr $self->{c};  
857            $self->{state} = STRING_STATE;            $self->{state} = STRING_STATE;
858            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
859            redo A;            redo A;
# Line 852  sub get_next_token ($) { Line 869  sub get_next_token ($) {
869                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
870                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
871            }->{$self->{t}->{type}};            }->{$self->{t}->{type}};
           $self->{t}->{value} .= "\x0D";  
872            $self->{state} = ESCAPE_BEFORE_LF_STATE;            $self->{state} = ESCAPE_BEFORE_LF_STATE;
873            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
874            redo A;            redo A;
875          } else {          } else {
876            ## Note: In |nl| in ... in |string| or |ident|.            ## Note: In |nl| in ... in |string| or |ident|.
           $self->{t}->{value} .= "\x0D";  
877            $self->{state} = ESCAPE_BEFORE_LF_STATE;            $self->{state} = ESCAPE_BEFORE_LF_STATE;
878            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
879            redo A;            redo A;
# Line 996  sub get_next_token ($) { Line 1011  sub get_next_token ($) {
1011          redo A;          redo A;
1012        }        }
1013      } elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {      } elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {
1014        ## NOTE: |\n| in |\r\n| in |unicode| in |escape|.        ## NOTE: |\n| in |\r\n| in |nl| in |escape|.
1015        if ($self->{c} == 0x000A) { # \n        if ($self->{c} == 0x000A) { # \n
         $self->{t}->{value} .= chr $self->{c};  
1016          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
1017              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
1018          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
Legend:
Removed from v.1.13  
changed lines
  Added in v.1.15
admin@suikawiki.org
 ViewVC Help
Powered by ViewVC 1.1.24