| 1 |
package Whatpm::CSS::Tokenizer; |
package Whatpm::CSS::Tokenizer; |
| 2 |
use strict; |
use strict; |
| 3 |
|
our $VERSION=do{my @r=(q$Revision$=~/\d+/g);sprintf "%d."."%02d" x $#r,@r}; |
| 4 |
|
|
| 5 |
|
require Exporter; |
| 6 |
|
push our @ISA, 'Exporter'; |
| 7 |
|
|
| 8 |
sub BEFORE_TOKEN_STATE () { 0 } |
sub BEFORE_TOKEN_STATE () { 0 } |
| 9 |
sub BEFORE_NMSTART_STATE () { 1 } |
sub BEFORE_NMSTART_STATE () { 1 } |
| 63 |
sub COMMENT_TOKEN () { 36 } |
sub COMMENT_TOKEN () { 36 } |
| 64 |
sub COMMENT_INVALID_TOKEN () { 37 } |
sub COMMENT_INVALID_TOKEN () { 37 } |
| 65 |
sub EOF_TOKEN () { 38 } |
sub EOF_TOKEN () { 38 } |
| 66 |
|
sub MINUS_TOKEN () { 39 } |
| 67 |
|
sub STAR_TOKEN () { 40 } |
| 68 |
|
sub VBAR_TOKEN () { 41 } |
| 69 |
|
sub DOT_TOKEN () { 42 } |
| 70 |
|
sub COLON_TOKEN () { 43 } |
| 71 |
|
sub MATCH_TOKEN () { 44 } |
| 72 |
|
sub EXCLAMATION_TOKEN () { 45 } |
| 73 |
|
|
| 74 |
our @TokenName = qw( |
our @TokenName = qw( |
| 75 |
0 IDENT ATKEYWORD HASH FUNCTION URI URI_INVALID URI_PREFIX URI_PREFIX_INVALID |
0 IDENT ATKEYWORD HASH FUNCTION URI URI_INVALID URI_PREFIX URI_PREFIX_INVALID |
| 77 |
0 DELIM PLUS GREATER COMMA TILDE DASHMATCH |
0 DELIM PLUS GREATER COMMA TILDE DASHMATCH |
| 78 |
PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH INCLUDES SEMICOLON |
PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH INCLUDES SEMICOLON |
| 79 |
LBRACE RBRACE LPAREN RPAREN LBRACKET RBRACKET S CDO CDC COMMENT |
LBRACE RBRACE LPAREN RPAREN LBRACKET RBRACKET S CDO CDC COMMENT |
| 80 |
COMMENT_INVALID EOF |
COMMENT_INVALID EOF MINUS STAR VBAR DOT COLON MATCH EXCLAMATION |
| 81 |
|
); |
| 82 |
|
|
| 83 |
|
our @EXPORT_OK = qw( |
| 84 |
|
IDENT_TOKEN ATKEYWORD_TOKEN HASH_TOKEN FUNCTION_TOKEN URI_TOKEN |
| 85 |
|
URI_INVALID_TOKEN URI_PREFIX_TOKEN URI_PREFIX_INVALID_TOKEN |
| 86 |
|
STRING_TOKEN INVALID_TOKEN NUMBER_TOKEN DIMENSION_TOKEN PERCENTAGE_TOKEN |
| 87 |
|
UNICODE_RANGE_TOKEN DELIM_TOKEN PLUS_TOKEN GREATER_TOKEN COMMA_TOKEN |
| 88 |
|
TILDE_TOKEN DASHMATCH_TOKEN PREFIXMATCH_TOKEN SUFFIXMATCH_TOKEN |
| 89 |
|
SUBSTRINGMATCH_TOKEN INCLUDES_TOKEN SEMICOLON_TOKEN LBRACE_TOKEN |
| 90 |
|
RBRACE_TOKEN LPAREN_TOKEN RPAREN_TOKEN LBRACKET_TOKEN RBRACKET_TOKEN |
| 91 |
|
S_TOKEN CDO_TOKEN CDC_TOKEN COMMENT_TOKEN COMMENT_INVALID_TOKEN EOF_TOKEN |
| 92 |
|
MINUS_TOKEN STAR_TOKEN VBAR_TOKEN DOT_TOKEN COLON_TOKEN MATCH_TOKEN |
| 93 |
|
EXCLAMATION_TOKEN |
| 94 |
); |
); |
| 95 |
|
|
| 96 |
|
our %EXPORT_TAGS = ('token' => [@EXPORT_OK]); |
| 97 |
|
|
| 98 |
sub new ($) { |
sub new ($) { |
| 99 |
my $self = bless {token => [], get_char => sub { -1 }, |
my $self = bless {token => [], get_char => sub { -1 }, |
| 100 |
onerror => sub { }}, shift; |
onerror => sub { }}, shift; |
| 141 |
(0x0041 <= $self->{c} and $self->{c} <= 0x0046) or # A..F |
(0x0041 <= $self->{c} and $self->{c} <= 0x0046) or # A..F |
| 142 |
(0x0061 <= $self->{c} and $self->{c} <= 0x0066) or # a..f |
(0x0061 <= $self->{c} and $self->{c} <= 0x0066) or # a..f |
| 143 |
$self->{c} == 0x003F) { # ? |
$self->{c} == 0x003F) { # ? |
| 144 |
$self->{t}->{value} .= '+' . chr $self->{c}; |
$self->{t}->{value} = chr $self->{c}; |
| 145 |
$self->{t}->{type} = UNICODE_RANGE_TOKEN; |
$self->{t}->{type} = UNICODE_RANGE_TOKEN; |
| 146 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 147 |
C: for (2..6) { |
C: for (2..6) { |
| 293 |
return {type => CDO_TOKEN}; |
return {type => CDO_TOKEN}; |
| 294 |
#redo A; |
#redo A; |
| 295 |
} else { |
} else { |
| 296 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'}; |
unshift @{$self->{token}}, {type => EXCLAMATION_TOKEN}; |
| 297 |
## NOTE: |-| in |ident| in |IDENT| |
## NOTE: |-| in |ident| in |IDENT| |
| 298 |
$self->{t} = {type => IDENT_TOKEN, value => '-'}; |
$self->{t} = {type => IDENT_TOKEN, value => '-'}; |
| 299 |
$self->{state} = BEFORE_NMSTART_STATE; |
$self->{state} = BEFORE_NMSTART_STATE; |
| 302 |
#redo A; |
#redo A; |
| 303 |
} |
} |
| 304 |
} else { |
} else { |
| 305 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '!'}; |
unshift @{$self->{token}}, {type => EXCLAMATION_TOKEN}; |
| 306 |
$self->{state} = BEFORE_TOKEN_STATE; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 307 |
#reprocess |
#reprocess |
| 308 |
return {type => DELIM_TOKEN, value => '<'}; |
return {type => DELIM_TOKEN, value => '<'}; |
| 315 |
#redo A; |
#redo A; |
| 316 |
} |
} |
| 317 |
} elsif (my $t = { |
} elsif (my $t = { |
| 318 |
0x003B => SEMICOLON_TOKEN, # ; |
0x0021 => EXCLAMATION_TOKEN, # ! |
| 319 |
0x007B => LBRACE_TOKEN, # { |
0x002D => MINUS_TOKEN, # - |
| 320 |
0x007D => RBRACE_TOKEN, # } |
0x002E => DOT_TOKEN, # . |
| 321 |
0x0028 => LPAREN_TOKEN, # ( |
0x003A => COLON_TOKEN, # : |
| 322 |
0x0029 => RPAREN_TOKEN, # ) |
0x003B => SEMICOLON_TOKEN, # ; |
| 323 |
0x005B => LBRACKET_TOKEN, # [ |
0x003D => MATCH_TOKEN, # = |
| 324 |
0x005D => RBRACKET_TOKEN, # ] |
0x007B => LBRACE_TOKEN, # { |
| 325 |
|
0x007D => RBRACE_TOKEN, # } |
| 326 |
|
0x0028 => LPAREN_TOKEN, # ( |
| 327 |
|
0x0029 => RPAREN_TOKEN, # ) |
| 328 |
|
0x005B => LBRACKET_TOKEN, # [ |
| 329 |
|
0x005D => RBRACKET_TOKEN, # ] |
| 330 |
}->{$self->{c}}) { |
}->{$self->{c}}) { |
| 331 |
# stay in the state |
# stay in the state |
| 332 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 379 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 380 |
return {type => $v}; |
return {type => $v}; |
| 381 |
#redo A; |
#redo A; |
| 382 |
|
} elsif ($v = { |
| 383 |
|
0x002A => STAR_TOKEN, # * |
| 384 |
|
0x007C => VBAR_TOKEN, # | |
| 385 |
|
}->{$c}) { |
| 386 |
|
# stay in the state. |
| 387 |
|
# reprocess |
| 388 |
|
return {type => $v}; |
| 389 |
|
#redo A; |
| 390 |
} else { |
} else { |
| 391 |
# stay in the state |
# stay in the state |
| 392 |
# reprocess |
# reprocess |
| 447 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 448 |
redo A; |
redo A; |
| 449 |
} elsif ($self->{c} == 0x005C) { # \ |
} elsif ($self->{c} == 0x005C) { # \ |
|
## TODO: 12-\X, 12-\{nl} |
|
| 450 |
$self->{state} = ESCAPE_OPEN_STATE; $q = 0; |
$self->{state} = ESCAPE_OPEN_STATE; $q = 0; |
| 451 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 452 |
redo A; |
redo A; |
| 453 |
} elsif ($self->{c} == 0x002D and # - |
} elsif ($self->{c} == 0x002D) { # - |
| 454 |
$self->{t}->{type} == IDENT_TOKEN) { |
if ($self->{t}->{type} == IDENT_TOKEN) { |
|
$self->{c} = $self->{get_char}->(); |
|
|
if ($self->{c} == 0x003E) { # > |
|
|
$self->{state} = BEFORE_TOKEN_STATE; |
|
| 455 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 456 |
return {type => CDC_TOKEN}; |
if ($self->{c} == 0x003E) { # > |
| 457 |
#redo A; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 458 |
|
$self->{c} = $self->{get_char}->(); |
| 459 |
|
return {type => CDC_TOKEN}; |
| 460 |
|
#redo A; |
| 461 |
|
} else { |
| 462 |
|
## NOTE: |-|, |-|, $self->{c} |
| 463 |
|
#$self->{t} = {type => IDENT_TOKEN, value => '-'}; |
| 464 |
|
# stay in the state |
| 465 |
|
# reconsume |
| 466 |
|
return {type => MINUS_TOKEN}; |
| 467 |
|
#redo A; |
| 468 |
|
} |
| 469 |
|
} elsif ($self->{t}->{type} == DIMENSION_TOKEN) { |
| 470 |
|
$self->{c} = $self->{get_char}->(); |
| 471 |
|
if ($self->{c} == 0x003E) { # > |
| 472 |
|
unshift @{$self->{token}}, {type => CDC_TOKEN}; |
| 473 |
|
$self->{t}->{type} = NUMBER_TOKEN; |
| 474 |
|
$self->{t}->{value} = ''; |
| 475 |
|
$self->{state} = BEFORE_TOKEN_STATE; |
| 476 |
|
$self->{c} = $self->{get_char}->(); |
| 477 |
|
return $self->{t}; |
| 478 |
|
#redo A; |
| 479 |
|
} else { |
| 480 |
|
## NOTE: |-|, |-|, $self->{c} |
| 481 |
|
my $t = $self->{t}; |
| 482 |
|
$t->{type} = NUMBER_TOKEN; |
| 483 |
|
$t->{value} = ''; |
| 484 |
|
$self->{t} = {type => IDENT_TOKEN, value => '-', hyphen => 1}; |
| 485 |
|
unshift @{$self->{token}}, {type => MINUS_TOKEN}; |
| 486 |
|
# stay in the state |
| 487 |
|
# reconsume |
| 488 |
|
return $t; |
| 489 |
|
#redo A; |
| 490 |
|
} |
| 491 |
} else { |
} else { |
| 492 |
## NOTE: |-|, |-|, $self->{c} |
# |
|
#$self->{t} = {type => IDENT_TOKEN, value => '-'}; |
|
|
# stay in the state |
|
|
# reconsume |
|
|
return {type => DELIM_TOKEN, value => '-'}; |
|
|
#redo A; |
|
| 493 |
} |
} |
| 494 |
} else { |
} else { |
| 495 |
if ($self->{t}->{type} == NUMBER_TOKEN) { |
# |
| 496 |
## NOTE: |-| after |NUMBER|. |
} |
| 497 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'}; |
|
| 498 |
$self->{state} = BEFORE_TOKEN_STATE; |
if ($self->{t}->{type} == DIMENSION_TOKEN) { |
| 499 |
# reprocess |
## NOTE: |-| after |NUMBER|. |
| 500 |
$self->{t}->{value} = $self->{t}->{number}; |
unshift @{$self->{token}}, {type => MINUS_TOKEN}; |
| 501 |
delete $self->{t}->{number}; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 502 |
return $self->{t}; |
# reprocess |
| 503 |
} else { |
$self->{t}->{type} = NUMBER_TOKEN; |
| 504 |
## NOTE: |-| not followed by |nmstart|. |
$self->{t}->{value} = ''; |
| 505 |
$self->{state} = BEFORE_TOKEN_STATE; |
return $self->{t}; |
| 506 |
# reprocess |
} else { |
| 507 |
return {type => DELIM_TOKEN, value => '-'}; |
## NOTE: |-| not followed by |nmstart|. |
| 508 |
} |
$self->{state} = BEFORE_TOKEN_STATE; |
| 509 |
|
# reprocess |
| 510 |
|
return {type => MINUS_TOKEN}; |
| 511 |
} |
} |
| 512 |
} elsif ($self->{state} == AFTER_AT_STATE) { |
} elsif ($self->{state} == AFTER_AT_STATE) { |
| 513 |
if ((0x0041 <= $self->{c} and $self->{c} <= 0x005A) or # A..Z |
if ((0x0041 <= $self->{c} and $self->{c} <= 0x005A) or # A..Z |
| 550 |
return {type => DELIM_TOKEN, value => '@'}; |
return {type => DELIM_TOKEN, value => '@'}; |
| 551 |
#redo A; |
#redo A; |
| 552 |
} else { |
} else { |
| 553 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'}; |
unshift @{$self->{token}}, {type => MINUS_TOKEN}; |
| 554 |
$self->{t} = {type => IDENT_TOKEN, value => '-'}; |
$self->{t} = {type => IDENT_TOKEN, value => '-'}; |
| 555 |
$self->{state} = BEFORE_NMSTART_STATE; |
$self->{state} = BEFORE_NMSTART_STATE; |
| 556 |
# reprocess |
# reprocess |
| 563 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 564 |
redo A; |
redo A; |
| 565 |
} else { |
} else { |
| 566 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '-'}; |
unshift @{$self->{token}}, {type => MINUS_TOKEN}; |
| 567 |
$self->{state} = BEFORE_TOKEN_STATE; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 568 |
# reprocess |
# reprocess |
| 569 |
return {type => DELIM_TOKEN, value => '@'}; |
return {type => DELIM_TOKEN, value => '@'}; |
| 571 |
} elsif ($self->{state} == AFTER_NUMBER_STATE) { |
} elsif ($self->{state} == AFTER_NUMBER_STATE) { |
| 572 |
if ($self->{c} == 0x002D) { # - |
if ($self->{c} == 0x002D) { # - |
| 573 |
## NOTE: |-| in |ident|. |
## NOTE: |-| in |ident|. |
| 574 |
|
$self->{t}->{hyphen} = 1; |
| 575 |
$self->{t}->{value} = '-'; |
$self->{t}->{value} = '-'; |
| 576 |
|
$self->{t}->{type} = DIMENSION_TOKEN; |
| 577 |
$self->{state} = BEFORE_NMSTART_STATE; |
$self->{state} = BEFORE_NMSTART_STATE; |
| 578 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 579 |
redo A; |
redo A; |
| 590 |
} elsif ($self->{c} == 0x005C) { # \ |
} elsif ($self->{c} == 0x005C) { # \ |
| 591 |
## NOTE: |nmstart| in |ident| in |IDENT| |
## NOTE: |nmstart| in |ident| in |IDENT| |
| 592 |
$self->{t}->{value} = ''; |
$self->{t}->{value} = ''; |
| 593 |
|
$self->{t}->{type} = DIMENSION_TOKEN; |
| 594 |
$self->{state} = ESCAPE_OPEN_STATE; $q = 0; |
$self->{state} = ESCAPE_OPEN_STATE; $q = 0; |
| 595 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 596 |
redo A; |
redo A; |
| 855 |
redo A; |
redo A; |
| 856 |
} else { |
} else { |
| 857 |
## Note: In |nl| in ... in |string| or |ident|. |
## Note: In |nl| in ... in |string| or |ident|. |
|
$self->{t}->{value} .= chr $self->{c}; |
|
| 858 |
$self->{state} = STRING_STATE; |
$self->{state} = STRING_STATE; |
| 859 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 860 |
redo A; |
redo A; |
| 870 |
URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN, |
URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN, |
| 871 |
URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN, |
URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN, |
| 872 |
}->{$self->{t}->{type}}; |
}->{$self->{t}->{type}}; |
|
$self->{t}->{value} .= "\x0D"; |
|
| 873 |
$self->{state} = ESCAPE_BEFORE_LF_STATE; |
$self->{state} = ESCAPE_BEFORE_LF_STATE; |
| 874 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 875 |
redo A; |
redo A; |
| 876 |
} else { |
} else { |
| 877 |
## Note: In |nl| in ... in |string| or |ident|. |
## Note: In |nl| in ... in |string| or |ident|. |
|
$self->{t}->{value} .= "\x0D"; |
|
| 878 |
$self->{state} = ESCAPE_BEFORE_LF_STATE; |
$self->{state} = ESCAPE_BEFORE_LF_STATE; |
| 879 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 880 |
redo A; |
redo A; |
| 891 |
} |
} |
| 892 |
|
|
| 893 |
if ($q == 0) { |
if ($q == 0) { |
| 894 |
$self->{state} = BEFORE_TOKEN_STATE; |
if ($self->{t}->{type} == DIMENSION_TOKEN) { |
| 895 |
# reprocess |
if ($self->{t}->{hyphen} and $self->{t}->{value} eq '-') { |
| 896 |
if ($self->{t}->{hyphen} and $self->{t}->{value} eq '-') { |
$self->{state} = BEFORE_TOKEN_STATE; |
| 897 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'}; |
# reprocess |
| 898 |
return {type => DELIM_TOKEN, value => '-'}; |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'}; |
| 899 |
#redo A; |
unshift @{$self->{token}}, {type => MINUS_TOKEN}; |
| 900 |
} elsif (length $self->{t}->{value}) { |
$self->{t}->{type} = NUMBER_TOKEN; |
| 901 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'}; |
$self->{t}->{value} = ''; |
| 902 |
return $self->{t}; |
return $self->{t}; |
| 903 |
#redo A; |
#redo A; |
| 904 |
|
} elsif (length $self->{t}->{value}) { |
| 905 |
|
$self->{state} = BEFORE_TOKEN_STATE; |
| 906 |
|
# reprocess |
| 907 |
|
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'}; |
| 908 |
|
return $self->{t}; |
| 909 |
|
#redo A; |
| 910 |
|
} else { |
| 911 |
|
$self->{state} = BEFORE_TOKEN_STATE; |
| 912 |
|
# reprocess |
| 913 |
|
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'}; |
| 914 |
|
$self->{t}->{type} = NUMBER_TOKEN; |
| 915 |
|
$self->{t}->{value} = ''; |
| 916 |
|
return $self->{t}; |
| 917 |
|
#redo A; |
| 918 |
|
} |
| 919 |
} else { |
} else { |
| 920 |
return {type => DELIM_TOKEN, value => '\\'}; |
if ($self->{t}->{hyphen} and $self->{t}->{value} eq '-') { |
| 921 |
#redo A; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 922 |
|
# reprocess |
| 923 |
|
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'}; |
| 924 |
|
return {type => MINUS_TOKEN}; |
| 925 |
|
#redo A; |
| 926 |
|
} elsif (length $self->{t}->{value}) { |
| 927 |
|
$self->{state} = BEFORE_TOKEN_STATE; |
| 928 |
|
# reprocess |
| 929 |
|
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '\\'}; |
| 930 |
|
return $self->{t}; |
| 931 |
|
#redo A; |
| 932 |
|
} else { |
| 933 |
|
$self->{state} = BEFORE_TOKEN_STATE; |
| 934 |
|
# reprocess |
| 935 |
|
return {type => DELIM_TOKEN, value => '\\'}; |
| 936 |
|
#redo A; |
| 937 |
|
} |
| 938 |
} |
} |
| 939 |
} elsif ($q == 1) { |
} elsif ($q == 1) { |
| 940 |
$self->{state} = URI_UNQUOTED_STATE; |
$self->{state} = URI_UNQUOTED_STATE; |
| 1012 |
redo A; |
redo A; |
| 1013 |
} |
} |
| 1014 |
} elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) { |
} elsif ($self->{state} == ESCAPE_BEFORE_LF_STATE) { |
| 1015 |
## NOTE: |\n| in |\r\n| in |unicode| in |escape|. |
## NOTE: |\n| in |\r\n| in |nl| in |escape|. |
| 1016 |
if ($self->{c} == 0x000A) { # \n |
if ($self->{c} == 0x000A) { # \n |
|
$self->{t}->{value} .= chr $self->{c}; |
|
| 1017 |
$self->{state} = $q == 0 ? NAME_STATE : |
$self->{state} = $q == 0 ? NAME_STATE : |
| 1018 |
$q == 1 ? URI_UNQUOTED_STATE : STRING_STATE; |
$q == 1 ? URI_UNQUOTED_STATE : STRING_STATE; |
| 1019 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 1048 |
$self->{c} == 0x000D or # \r |
$self->{c} == 0x000D or # \r |
| 1049 |
$self->{c} == 0x000C or # \f |
$self->{c} == 0x000C or # \f |
| 1050 |
$self->{c} == -1) { |
$self->{c} == -1) { |
| 1051 |
$self->{t}->{type} = INVALID_TOKEN; |
$self->{t}->{type} = { |
| 1052 |
|
STRING_TOKEN, INVALID_TOKEN, |
| 1053 |
|
INVALID_TOKEN, INVALID_TOKEN, |
| 1054 |
|
URI_TOKEN, URI_INVALID_TOKEN, |
| 1055 |
|
URI_INVALID_TOKEN, URI_INVALID_TOKEN, |
| 1056 |
|
URI_PREFIX_TOKEN, URI_PREFIX_INVALID_TOKEN, |
| 1057 |
|
URI_PREFIX_INVALID_TOKEN, URI_PREFIX_INVALID_TOKEN, |
| 1058 |
|
}->{$self->{t}->{type}}; |
| 1059 |
$self->{state} = BEFORE_TOKEN_STATE; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 1060 |
# reconsume |
# reconsume |
| 1061 |
return $self->{t}; |
return $self->{t}; |
| 1092 |
$self->{c} = $self->{get_char}->(); |
$self->{c} = $self->{get_char}->(); |
| 1093 |
redo A; |
redo A; |
| 1094 |
} else { |
} else { |
| 1095 |
unshift @{$self->{token}}, {type => DELIM_TOKEN, value => '.'}; |
unshift @{$self->{token}}, {type => DOT_TOKEN}; |
| 1096 |
$self->{t}->{number} = $self->{t}->{value}; |
$self->{t}->{number} = $self->{t}->{value}; |
| 1097 |
$self->{t}->{value} = ''; |
$self->{t}->{value} = ''; |
| 1098 |
$self->{state} = BEFORE_TOKEN_STATE; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 1110 |
} else { |
} else { |
| 1111 |
$self->{state} = BEFORE_TOKEN_STATE; |
$self->{state} = BEFORE_TOKEN_STATE; |
| 1112 |
# reprocess |
# reprocess |
| 1113 |
return {type => DELIM_TOKEN, value => '.'}; |
return {type => DOT_TOKEN}; |
| 1114 |
#redo A; |
#redo A; |
| 1115 |
} |
} |
| 1116 |
} elsif ($self->{state} == NUMBER_DOT_NUMBER_STATE) { |
} elsif ($self->{state} == NUMBER_DOT_NUMBER_STATE) { |
| 1133 |
} # A |
} # A |
| 1134 |
} # get_next_token |
} # get_next_token |
| 1135 |
|
|
| 1136 |
|
=head1 LICENSE |
| 1137 |
|
|
| 1138 |
|
Copyright 2007 Wakaba <w@suika.fam.cx> |
| 1139 |
|
|
| 1140 |
|
This library is free software; you can redistribute it |
| 1141 |
|
and/or modify it under the same terms as Perl itself. |
| 1142 |
|
|
| 1143 |
|
=cut |
| 1144 |
|
|
| 1145 |
1; |
1; |
| 1146 |
# $Date$ |
# $Date$ |