/[suikacvs]/markup/html/whatpm/Whatpm/CSS/Tokenizer.pm
Suika

Diff of /markup/html/whatpm/Whatpm/CSS/Tokenizer.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.5 by wakaba, Sat Sep 8 03:25:05 2007 UTC revision 1.8 by wakaba, Sat Sep 8 11:09:41 2007 UTC
# Line 36  sub NUMBER_TOKEN () { 11 } Line 36  sub NUMBER_TOKEN () { 11 }
36  sub DIMENSION_TOKEN () { 12 }  sub DIMENSION_TOKEN () { 12 }
37  sub PERCENTAGE_TOKEN () { 13 }  sub PERCENTAGE_TOKEN () { 13 }
38  sub UNICODE_RANGE_TOKEN () { 14 }  sub UNICODE_RANGE_TOKEN () { 14 }
 sub UNICODE_RANGE_INVALID_TOKEN () { 15 }  
39  sub DELIM_TOKEN () { 16 }  sub DELIM_TOKEN () { 16 }
40  sub PLUS_TOKEN () { 17 }  sub PLUS_TOKEN () { 17 }
41  sub GREATER_TOKEN () { 18 }  sub GREATER_TOKEN () { 18 }
# Line 64  sub EOF_TOKEN () { 38 } Line 63  sub EOF_TOKEN () { 38 }
63  our @TokenName = qw(  our @TokenName = qw(
64    0 IDENT ATKEYWORD HASH FUNCTION URI URI_INVALID URI_PREFIX URI_PREFIX_INVALID    0 IDENT ATKEYWORD HASH FUNCTION URI URI_INVALID URI_PREFIX URI_PREFIX_INVALID
65    STRING INVALID NUMBER DIMENSION PERCENTAGE UNICODE_RANGE    STRING INVALID NUMBER DIMENSION PERCENTAGE UNICODE_RANGE
66    UNICODE_RANGE_INVALID DELIM PLUS GREATER COMMA TILDE DASHMATCH    0 DELIM PLUS GREATER COMMA TILDE DASHMATCH
67    PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH INCLUDES SEMICOLON    PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH INCLUDES SEMICOLON
68    LBRACE RBRACE LPAREN RPAREN LBRACKET RBRACKET S CDO CDC COMMENT    LBRACE RBRACE LPAREN RPAREN LBRACKET RBRACKET S CDO CDC COMMENT
69    COMMENT_INVALID EOF    COMMENT_INVALID EOF
# Line 103  sub get_next_token ($) { Line 102  sub get_next_token ($) {
102      if ($self->{state} == BEFORE_TOKEN_STATE) {      if ($self->{state} == BEFORE_TOKEN_STATE) {
103        if ($self->{c} == 0x002D) { # -        if ($self->{c} == 0x002D) { # -
104          ## NOTE: |-| in |ident| in |IDENT|          ## NOTE: |-| in |ident| in |IDENT|
105          $self->{t} = {type => IDENT_TOKEN, value => '-'};          $self->{t} = {type => IDENT_TOKEN, value => '-', hyphen => 1};
106          $self->{state} = BEFORE_NMSTART_STATE;          $self->{state} = BEFORE_NMSTART_STATE;
107          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
108          redo A;          redo A;
# Line 434  sub get_next_token ($) { Line 433  sub get_next_token ($) {
433            ## NOTE: |-| after |NUMBER|.            ## NOTE: |-| after |NUMBER|.
434            unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'};            unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'};
435            $self->{state} = BEFORE_TOKEN_STATE;            $self->{state} = BEFORE_TOKEN_STATE;
436            # reconsume            # reprocess
437            $self->{t}->{value} = $self->{t}->{number};            $self->{t}->{value} = $self->{t}->{number};
438            delete $self->{t}->{number};            delete $self->{t}->{number};
439            return $self->{t};            return $self->{t};
440          } else {          } else {
441            ## NOTE: |-| not followed by |nmstart|.            ## NOTE: |-| not followed by |nmstart|.
442            $self->{state} = BEFORE_TOKEN_STATE;            $self->{state} = BEFORE_TOKEN_STATE;
443            $self->{c} = $self->{get_char}->();            # reprocess
444            return {type => DELIM_TOKEN, value => '-'};            return {type => DELIM_TOKEN, value => '-'};
445          }          }
446        }        }
# Line 766  sub get_next_token ($) { Line 765  sub get_next_token ($) {
765          redo A;          redo A;
766        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f
767          ## NOTE: second character of |unicode| in |escape|.          ## NOTE: second character of |unicode| in |escape|.
768          $char = $self->{c} - 0x0061 - 0xA;          $char = $self->{c} - 0x0061 + 0xA;
769          $self->{state} = ESCAPE_STATE; $i = 2;          $self->{state} = ESCAPE_STATE; $i = 2;
770          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
771          redo A;          redo A;
772        } elsif ($self->{c} == 0x000A or # \n        } elsif ($self->{c} == 0x000A or # \n
773                 $self->{c} == 0x000C) { # \f                 $self->{c} == 0x000C) { # \f
774          if ($q == 0) {          if ($q == 0) {
775            ## NOTE: In |escape| in ... in |ident|.            #
           $self->{state} = BEFORE_TOKEN_STATE;  
           unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};  
           return $self->{t};  
           # reconsume  
           #redo A;  
776          } elsif ($q == 1) {          } elsif ($q == 1) {
777            ## NOTE: In |escape| in |URI|.            ## NOTE: In |escape| in |URI|.
778            $self->{t}->{type} = {            $self->{t}->{type} = {
# Line 800  sub get_next_token ($) { Line 794  sub get_next_token ($) {
794          }          }
795        } elsif ($self->{c} == 0x000D) { # \r        } elsif ($self->{c} == 0x000D) { # \r
796          if ($q == 0) {          if ($q == 0) {
797            ## NOTE: In |escape| in ... in |ident|.            #
           $self->{state} = BEFORE_TOKEN_STATE;  
           unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};  
           return $self->{t};  
           # reconsume  
           #redo A;  
798          } elsif ($q == 1) {          } elsif ($q == 1) {
799              ## NOTE: In |escape| in |URI|.
800            $self->{t}->{type} = {            $self->{t}->{type} = {
801                URI_TOKEN, URI_INVALID_TOKEN,                URI_TOKEN, URI_INVALID_TOKEN,
802                URI_INVALID_TOKEN, URI_INVALID_TOKEN,                URI_INVALID_TOKEN, URI_INVALID_TOKEN,
803                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
804                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,                URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN,
805            }->{$self->{t}->{type}};            }->{$self->{t}->{type}};
806            $self->{t}->{value} .= "\x0D\x0A";            $self->{t}->{value} .= "\x0D";
807            $self->{state} = URI_UNQUOTED_STATE;            $self->{state} = ESCAPE_BEFORE_LF_STATE;
808            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
809            redo A;            redo A;
810          } else {          } else {
811            ## Note: In |nl| in ... in |string| or |ident|.            ## Note: In |nl| in ... in |string| or |ident|.
812            $self->{t}->{value} .= "\x0D\x0A";            $self->{t}->{value} .= "\x0D";
813            $self->{state} = ESCAPE_BEFORE_LF_STATE;            $self->{state} = ESCAPE_BEFORE_LF_STATE;
814            $self->{c} = $self->{get_char}->();            $self->{c} = $self->{get_char}->();
815            redo A;            redo A;
816          }          }
817          } elsif ($self->{c} == -1) {
818            #
819        } else {        } else {
820          ## NOTE: second character of |escape|.          ## NOTE: second character of |escape|.
821          $self->{t}->{value} .= chr $self->{c};          $self->{t}->{value} .= chr $self->{c};
# Line 832  sub get_next_token ($) { Line 824  sub get_next_token ($) {
824          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
825          redo A;          redo A;
826        }        }
827    
828          if ($q == 0) {
829            $self->{state} = BEFORE_TOKEN_STATE;
830            # reprocess
831            if ($self->{t}->{hyphen} and $self->{t}->{value} eq '-') {
832              unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
833              return {type => DELIM_TOKEN, value => '-'};
834              #redo A;
835            } elsif (length $self->{t}->{value}) {
836              unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
837              return $self->{t};
838              #redo A;
839            } else {
840              return {type => DELIM_TOKEN, value => '\\'};
841              #redo A;
842            }
843          } elsif ($q == 1) {
844            $self->{state} = URI_UNQUOTED_STATE;
845            $self->{c} = $self->{get_char}->();
846            redo A;
847          } else {
848            unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'};
849            $self->{t}->{type} = {
850              STRING_TOKEN, INVALID_TOKEN,
851              URI_TOKEN, URI_INVALID_TOKEN,
852              URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN,
853            }->{$self->{t}->{type}} || $self->{t}->{type};
854            $self->{state} = BEFORE_TOKEN_STATE;
855            # reprocess
856            return $self->{t};
857            #redo A;
858          }
859      } elsif ($self->{state} == ESCAPE_STATE) {      } elsif ($self->{state} == ESCAPE_STATE) {
860        ## NOTE: third..seventh character of |unicode| in |escape|.        ## NOTE: third..seventh character of |unicode| in |escape|.
861        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9        if (0x0030 <= $self->{c} and $self->{c} <= 0x0039) { # 0..9
# Line 845  sub get_next_token ($) { Line 869  sub get_next_token ($) {
869          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
870          redo A;          redo A;
871        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f        } elsif (0x0061 <= $self->{c} and $self->{c} <= 0x0066) { # a..f
872          $char = $char * 0x10 + $self->{c} - 0x0061 - 0xA;          $char = $char * 0x10 + $self->{c} - 0x0061 + 0xA;
873          $self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;          $self->{state} = ++$i == 7 ? ESCAPE_BEFORE_NL_STATE : ESCAPE_STATE;
874          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
875          redo A;          redo A;
# Line 894  sub get_next_token ($) { Line 918  sub get_next_token ($) {
918      } elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {      } elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) {
919        ## NOTE: |\n| in |\r\n| in |unicode| in |escape|.        ## NOTE: |\n| in |\r\n| in |unicode| in |escape|.
920        if ($self->{c} == 0x000A) { # \n        if ($self->{c} == 0x000A) { # \n
921          $self->{t}->{value} .= chr $char;          $self->{t}->{value} .= chr $self->{c};
922          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
923              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
924          $self->{c} = $self->{get_char}->();          $self->{c} = $self->{get_char}->();
925          redo A;          redo A;
926        } else {        } else {
         $self->{t}->{value} .= chr $char;  
927          $self->{state} = $q == 0 ? NAME_STATE :          $self->{state} = $q == 0 ? NAME_STATE :
928              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;              $q == 1 ? URI_UNQUOTED_STATE : STRING_STATE;
929          # reconsume          # reprocess
930          redo A;          redo A;
931        }        }
932      } elsif ($self->{state} == STRING_STATE) {      } elsif ($self->{state} == STRING_STATE) {
Legend:
Removed from v.1.5  
changed lines
  Added in v.1.8
admin@suikawiki.org
 ViewVC Help
Powered by ViewVC 1.1.24