/[suikacvs]/markup/html/whatpm/Whatpm/HTML/Tokenizer.pm.src
Suika

Contents of /markup/html/whatpm/Whatpm/HTML/Tokenizer.pm.src

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1.1 - (hide annotations) (download) (as text)
Tue Oct 14 02:27:58 2008 UTC (16 years ago) by wakaba
Branch: MAIN
File MIME type: application/x-wais-source 
++ whatpm/Whatpm/ChangeLog	14 Oct 2008 02:26:16 -0000
2008-10-14  Wakaba  <wakaba@suika.fam.cx>

	* Makefile: New rule to make HTML/Tokenizer.pm is added.

	* HTML.pm.src: Tokenizer part moved to another file.

++ whatpm/Whatpm/HTML/ChangeLog	14 Oct 2008 02:25:46 -0000
2008-10-14  Wakaba  <wakaba@suika.fam.cx>

	* Tokenizer.pm.src: New file.
1 wakaba 1.1 package Whatpm::HTML::Tokenizer;
2     use strict;
3     our $VERSION=do{my @r=(q$Revision: 1.207 $=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
4    
5     package Whatpm::HTML;
6    
7     ## Content model flags
8    
9     sub CM_ENTITY () { 0b001 } # & markup in data
10     sub CM_LIMITED_MARKUP () { 0b010 } # < markup in data (limited)
11     sub CM_FULL_MARKUP () { 0b100 } # < markup in data (any)
12    
13     sub PLAINTEXT_CONTENT_MODEL () { 0 }
14     sub CDATA_CONTENT_MODEL () { CM_LIMITED_MARKUP }
15     sub RCDATA_CONTENT_MODEL () { CM_ENTITY | CM_LIMITED_MARKUP }
16     sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }
17    
18     ## Tokenizer states
19    
20     sub DATA_STATE () { 0 }
21     #sub ENTITY_DATA_STATE () { 1 }
22     sub TAG_OPEN_STATE () { 2 }
23     sub CLOSE_TAG_OPEN_STATE () { 3 }
24     sub TAG_NAME_STATE () { 4 }
25     sub BEFORE_ATTRIBUTE_NAME_STATE () { 5 }
26     sub ATTRIBUTE_NAME_STATE () { 6 }
27     sub AFTER_ATTRIBUTE_NAME_STATE () { 7 }
28     sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8 }
29     sub ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE () { 9 }
30     sub ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE () { 10 }
31     sub ATTRIBUTE_VALUE_UNQUOTED_STATE () { 11 }
32     #sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }
33     sub MARKUP_DECLARATION_OPEN_STATE () { 13 }
34     sub COMMENT_START_STATE () { 14 }
35     sub COMMENT_START_DASH_STATE () { 15 }
36     sub COMMENT_STATE () { 16 }
37     sub COMMENT_END_STATE () { 17 }
38     sub COMMENT_END_DASH_STATE () { 18 }
39     sub BOGUS_COMMENT_STATE () { 19 }
40     sub DOCTYPE_STATE () { 20 }
41     sub BEFORE_DOCTYPE_NAME_STATE () { 21 }
42     sub DOCTYPE_NAME_STATE () { 22 }
43     sub AFTER_DOCTYPE_NAME_STATE () { 23 }
44     sub BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 24 }
45     sub DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE () { 25 }
46     sub DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE () { 26 }
47     sub AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 27 }
48     sub BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 28 }
49     sub DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE () { 29 }
50     sub DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE () { 30 }
51     sub AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 31 }
52     sub BOGUS_DOCTYPE_STATE () { 32 }
53     sub AFTER_ATTRIBUTE_VALUE_QUOTED_STATE () { 33 }
54     sub SELF_CLOSING_START_TAG_STATE () { 34 }
55     sub CDATA_SECTION_STATE () { 35 }
56     sub MD_HYPHEN_STATE () { 36 } # "markup declaration open state" in the spec
57     sub MD_DOCTYPE_STATE () { 37 } # "markup declaration open state" in the spec
58     sub MD_CDATA_STATE () { 38 } # "markup declaration open state" in the spec
59     sub CDATA_RCDATA_CLOSE_TAG_STATE () { 39 } # "close tag open state" in the spec
60     sub CDATA_SECTION_MSE1_STATE () { 40 } # "CDATA section state" in the spec
61     sub CDATA_SECTION_MSE2_STATE () { 41 } # "CDATA section state" in the spec
62     sub PUBLIC_STATE () { 42 } # "after DOCTYPE name state" in the spec
63     sub SYSTEM_STATE () { 43 } # "after DOCTYPE name state" in the spec
64     ## NOTE: "Entity data state", "entity in attribute value state", and
65     ## "consume a character reference" algorithm are jointly implemented
66     ## using the following six states:
67     sub ENTITY_STATE () { 44 }
68     sub ENTITY_HASH_STATE () { 45 }
69     sub NCR_NUM_STATE () { 46 }
70     sub HEXREF_X_STATE () { 47 }
71     sub HEXREF_HEX_STATE () { 48 }
72     sub ENTITY_NAME_STATE () { 49 }
73     sub PCDATA_STATE () { 50 } # "data state" in the spec
74    
75     ## Token types
76    
77     sub DOCTYPE_TOKEN () { 1 }
78     sub COMMENT_TOKEN () { 2 }
79     sub START_TAG_TOKEN () { 3 }
80     sub END_TAG_TOKEN () { 4 }
81     sub END_OF_FILE_TOKEN () { 5 }
82     sub CHARACTER_TOKEN () { 6 }
83    
84     ## Tree constructor state constants (see Whatpm::HTML for the full
85     ## list and descriptions)
86    
87     sub IN_FOREIGN_CONTENT_IM () { 0b100000000000 }
88     sub FOREIGN_EL () { 0b1_00000000000 }
89    
90     ## Character reference mappings
91    
92     my $charref_map = {
93     0x0D => 0x000A,
94     0x80 => 0x20AC,
95     0x81 => 0xFFFD,
96     0x82 => 0x201A,
97     0x83 => 0x0192,
98     0x84 => 0x201E,
99     0x85 => 0x2026,
100     0x86 => 0x2020,
101     0x87 => 0x2021,
102     0x88 => 0x02C6,
103     0x89 => 0x2030,
104     0x8A => 0x0160,
105     0x8B => 0x2039,
106     0x8C => 0x0152,
107     0x8D => 0xFFFD,
108     0x8E => 0x017D,
109     0x8F => 0xFFFD,
110     0x90 => 0xFFFD,
111     0x91 => 0x2018,
112     0x92 => 0x2019,
113     0x93 => 0x201C,
114     0x94 => 0x201D,
115     0x95 => 0x2022,
116     0x96 => 0x2013,
117     0x97 => 0x2014,
118     0x98 => 0x02DC,
119     0x99 => 0x2122,
120     0x9A => 0x0161,
121     0x9B => 0x203A,
122     0x9C => 0x0153,
123     0x9D => 0xFFFD,
124     0x9E => 0x017E,
125     0x9F => 0x0178,
126     }; # $charref_map
127     $charref_map->{$_} = 0xFFFD
128     for 0x0000..0x0008, 0x000B, 0x000E..0x001F, 0x007F,
129     0xD800..0xDFFF, 0xFDD0..0xFDDF, ## ISSUE: 0xFDEF
130     0xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF,
131     0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE,
132     0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
133     0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, 0xDFFFF, 0xEFFFE,
134     0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF;
135    
136     ## Implementations MUST act as if state machine in the spec
137    
138     sub _initialize_tokenizer ($) {
139     my $self = shift;
140    
141     ## NOTE: Fields set by |new| constructor:
142     #$self->{level}
143     #$self->{set_nc}
144     #$self->{parse_error}
145    
146     $self->{state} = DATA_STATE; # MUST
147     #$self->{s_kwd}; # state keyword - initialized when used
148     #$self->{entity__value}; # initialized when used
149     #$self->{entity__match}; # initialized when used
150     $self->{content_model} = PCDATA_CONTENT_MODEL; # be
151     undef $self->{ct}; # current token
152     undef $self->{ca}; # current attribute
153     undef $self->{last_stag_name}; # last emitted start tag name
154     #$self->{prev_state}; # initialized when used
155     delete $self->{self_closing};
156     $self->{char_buffer} = '';
157     $self->{char_buffer_pos} = 0;
158     $self->{nc} = -1; # next input character
159     #$self->{next_nc}
160     !!!next-input-character;
161     $self->{token} = [];
162     # $self->{escape}
163     } # _initialize_tokenizer
164    
165     ## A token has:
166     ## ->{type} == DOCTYPE_TOKEN, START_TAG_TOKEN, END_TAG_TOKEN, COMMENT_TOKEN,
167     ## CHARACTER_TOKEN, or END_OF_FILE_TOKEN
168     ## ->{name} (DOCTYPE_TOKEN)
169     ## ->{tag_name} (START_TAG_TOKEN, END_TAG_TOKEN)
170     ## ->{pubid} (DOCTYPE_TOKEN)
171     ## ->{sysid} (DOCTYPE_TOKEN)
172     ## ->{quirks} == 1 or 0 (DOCTYPE_TOKEN): "force-quirks" flag
173     ## ->{attributes} isa HASH (START_TAG_TOKEN, END_TAG_TOKEN)
174     ## ->{name}
175     ## ->{value}
176     ## ->{has_reference} == 1 or 0
177     ## ->{data} (COMMENT_TOKEN, CHARACTER_TOKEN)
178     ## NOTE: The "self-closing flag" is hold as |$self->{self_closing}|.
179     ## |->{self_closing}| is used to save the value of |$self->{self_closing}|
180     ## while the token is pushed back to the stack.
181    
182     ## Emitted token MUST immediately be handled by the tree construction state.
183    
184     ## Before each step, UA MAY check to see if either one of the scripts in
185     ## "list of scripts that will execute as soon as possible" or the first
186     ## script in the "list of scripts that will execute asynchronously",
187     ## has completed loading. If one has, then it MUST be executed
188     ## and removed from the list.
189    
190     ## TODO: Polytheistic slash SHOULD NOT be used. (Applied only to atheists.)
191     ## (This requirement was dropped from HTML5 spec, unfortunately.)
192    
193     my $is_space = {
194     0x0009 => 1, # CHARACTER TABULATION (HT)
195     0x000A => 1, # LINE FEED (LF)
196     #0x000B => 0, # LINE TABULATION (VT)
197     0x000C => 1, # FORM FEED (FF)
198     #0x000D => 1, # CARRIAGE RETURN (CR)
199     0x0020 => 1, # SPACE (SP)
200     };
201    
202     sub _get_next_token ($) {
203     my $self = shift;
204    
205     if ($self->{self_closing}) {
206     !!!parse-error (type => 'nestc', token => $self->{ct});
207     ## NOTE: The |self_closing| flag is only set by start tag token.
208     ## In addition, when a start tag token is emitted, it is always set to
209     ## |ct|.
210     delete $self->{self_closing};
211     }
212    
213     if (@{$self->{token}}) {
214     $self->{self_closing} = $self->{token}->[0]->{self_closing};
215     return shift @{$self->{token}};
216     }
217    
218     A: {
219     if ($self->{state} == PCDATA_STATE) {
220     ## NOTE: Same as |DATA_STATE|, but only for |PCDATA| content model.
221    
222     if ($self->{nc} == 0x0026) { # &
223     !!!cp (0.1);
224     ## NOTE: In the spec, the tokenizer is switched to the
225     ## "entity data state". In this implementation, the tokenizer
226     ## is switched to the |ENTITY_STATE|, which is an implementation
227     ## of the "consume a character reference" algorithm.
228     $self->{entity_add} = -1;
229     $self->{prev_state} = DATA_STATE;
230     $self->{state} = ENTITY_STATE;
231     !!!next-input-character;
232     redo A;
233     } elsif ($self->{nc} == 0x003C) { # <
234     !!!cp (0.2);
235     $self->{state} = TAG_OPEN_STATE;
236     !!!next-input-character;
237     redo A;
238     } elsif ($self->{nc} == -1) {
239     !!!cp (0.3);
240     !!!emit ({type => END_OF_FILE_TOKEN,
241     line => $self->{line}, column => $self->{column}});
242     last A; ## TODO: ok?
243     } else {
244     !!!cp (0.4);
245     #
246     }
247    
248     # Anything else
249     my $token = {type => CHARACTER_TOKEN,
250     data => chr $self->{nc},
251     line => $self->{line}, column => $self->{column},
252     };
253     $self->{read_until}->($token->{data}, q[<&], length $token->{data});
254    
255     ## Stay in the state.
256     !!!next-input-character;
257     !!!emit ($token);
258     redo A;
259     } elsif ($self->{state} == DATA_STATE) {
260     $self->{s_kwd} = '' unless defined $self->{s_kwd};
261     if ($self->{nc} == 0x0026) { # &
262     $self->{s_kwd} = '';
263     if ($self->{content_model} & CM_ENTITY and # PCDATA | RCDATA
264     not $self->{escape}) {
265     !!!cp (1);
266     ## NOTE: In the spec, the tokenizer is switched to the
267     ## "entity data state". In this implementation, the tokenizer
268     ## is switched to the |ENTITY_STATE|, which is an implementation
269     ## of the "consume a character reference" algorithm.
270     $self->{entity_add} = -1;
271     $self->{prev_state} = DATA_STATE;
272     $self->{state} = ENTITY_STATE;
273     !!!next-input-character;
274     redo A;
275     } else {
276     !!!cp (2);
277     #
278     }
279     } elsif ($self->{nc} == 0x002D) { # -
280     if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
281     $self->{s_kwd} .= '-';
282    
283     if ($self->{s_kwd} eq '<!--') {
284     !!!cp (3);
285     $self->{escape} = 1; # unless $self->{escape};
286     $self->{s_kwd} = '--';
287     #
288     } elsif ($self->{s_kwd} eq '---') {
289     !!!cp (4);
290     $self->{s_kwd} = '--';
291     #
292     } else {
293     !!!cp (5);
294     #
295     }
296     }
297    
298     #
299     } elsif ($self->{nc} == 0x0021) { # !
300     if (length $self->{s_kwd}) {
301     !!!cp (5.1);
302     $self->{s_kwd} .= '!';
303     #
304     } else {
305     !!!cp (5.2);
306     #$self->{s_kwd} = '';
307     #
308     }
309     #
310     } elsif ($self->{nc} == 0x003C) { # <
311     if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA
312     (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA
313     not $self->{escape})) {
314     !!!cp (6);
315     $self->{state} = TAG_OPEN_STATE;
316     !!!next-input-character;
317     redo A;
318     } else {
319     !!!cp (7);
320     $self->{s_kwd} = '';
321     #
322     }
323     } elsif ($self->{nc} == 0x003E) { # >
324     if ($self->{escape} and
325     ($self->{content_model} & CM_LIMITED_MARKUP)) { # RCDATA | CDATA
326     if ($self->{s_kwd} eq '--') {
327     !!!cp (8);
328     delete $self->{escape};
329     } else {
330     !!!cp (9);
331     }
332     } else {
333     !!!cp (10);
334     }
335    
336     $self->{s_kwd} = '';
337     #
338     } elsif ($self->{nc} == -1) {
339     !!!cp (11);
340     $self->{s_kwd} = '';
341     !!!emit ({type => END_OF_FILE_TOKEN,
342     line => $self->{line}, column => $self->{column}});
343     last A; ## TODO: ok?
344     } else {
345     !!!cp (12);
346     $self->{s_kwd} = '';
347     #
348     }
349    
350     # Anything else
351     my $token = {type => CHARACTER_TOKEN,
352     data => chr $self->{nc},
353     line => $self->{line}, column => $self->{column},
354     };
355     if ($self->{read_until}->($token->{data}, q[-!<>&],
356     length $token->{data})) {
357     $self->{s_kwd} = '';
358     }
359    
360     ## Stay in the data state.
361     if ($self->{content_model} == PCDATA_CONTENT_MODEL) {
362     !!!cp (13);
363     $self->{state} = PCDATA_STATE;
364     } else {
365     !!!cp (14);
366     ## Stay in the state.
367     }
368     !!!next-input-character;
369     !!!emit ($token);
370     redo A;
371     } elsif ($self->{state} == TAG_OPEN_STATE) {
372     if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
373     if ($self->{nc} == 0x002F) { # /
374     !!!cp (15);
375     !!!next-input-character;
376     $self->{state} = CLOSE_TAG_OPEN_STATE;
377     redo A;
378     } elsif ($self->{nc} == 0x0021) { # !
379     !!!cp (15.1);
380     $self->{s_kwd} = '<' unless $self->{escape};
381     #
382     } else {
383     !!!cp (16);
384     #
385     }
386    
387     ## reconsume
388     $self->{state} = DATA_STATE;
389     !!!emit ({type => CHARACTER_TOKEN, data => '<',
390     line => $self->{line_prev},
391     column => $self->{column_prev},
392     });
393     redo A;
394     } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA
395     if ($self->{nc} == 0x0021) { # !
396     !!!cp (17);
397     $self->{state} = MARKUP_DECLARATION_OPEN_STATE;
398     !!!next-input-character;
399     redo A;
400     } elsif ($self->{nc} == 0x002F) { # /
401     !!!cp (18);
402     $self->{state} = CLOSE_TAG_OPEN_STATE;
403     !!!next-input-character;
404     redo A;
405     } elsif (0x0041 <= $self->{nc} and
406     $self->{nc} <= 0x005A) { # A..Z
407     !!!cp (19);
408     $self->{ct}
409     = {type => START_TAG_TOKEN,
410     tag_name => chr ($self->{nc} + 0x0020),
411     line => $self->{line_prev},
412     column => $self->{column_prev}};
413     $self->{state} = TAG_NAME_STATE;
414     !!!next-input-character;
415     redo A;
416     } elsif (0x0061 <= $self->{nc} and
417     $self->{nc} <= 0x007A) { # a..z
418     !!!cp (20);
419     $self->{ct} = {type => START_TAG_TOKEN,
420     tag_name => chr ($self->{nc}),
421     line => $self->{line_prev},
422     column => $self->{column_prev}};
423     $self->{state} = TAG_NAME_STATE;
424     !!!next-input-character;
425     redo A;
426     } elsif ($self->{nc} == 0x003E) { # >
427     !!!cp (21);
428     !!!parse-error (type => 'empty start tag',
429     line => $self->{line_prev},
430     column => $self->{column_prev});
431     $self->{state} = DATA_STATE;
432     !!!next-input-character;
433    
434     !!!emit ({type => CHARACTER_TOKEN, data => '<>',
435     line => $self->{line_prev},
436     column => $self->{column_prev},
437     });
438    
439     redo A;
440     } elsif ($self->{nc} == 0x003F) { # ?
441     !!!cp (22);
442     !!!parse-error (type => 'pio',
443     line => $self->{line_prev},
444     column => $self->{column_prev});
445     $self->{state} = BOGUS_COMMENT_STATE;
446     $self->{ct} = {type => COMMENT_TOKEN, data => '',
447     line => $self->{line_prev},
448     column => $self->{column_prev},
449     };
450     ## $self->{nc} is intentionally left as is
451     redo A;
452     } else {
453     !!!cp (23);
454     !!!parse-error (type => 'bare stago',
455     line => $self->{line_prev},
456     column => $self->{column_prev});
457     $self->{state} = DATA_STATE;
458     ## reconsume
459    
460     !!!emit ({type => CHARACTER_TOKEN, data => '<',
461     line => $self->{line_prev},
462     column => $self->{column_prev},
463     });
464    
465     redo A;
466     }
467     } else {
468     die "$0: $self->{content_model} in tag open";
469     }
470     } elsif ($self->{state} == CLOSE_TAG_OPEN_STATE) {
471     ## NOTE: The "close tag open state" in the spec is implemented as
472     ## |CLOSE_TAG_OPEN_STATE| and |CDATA_RCDATA_CLOSE_TAG_STATE|.
473    
474     my ($l, $c) = ($self->{line_prev}, $self->{column_prev} - 1); # "<"of"</"
475     if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
476     if (defined $self->{last_stag_name}) {
477     $self->{state} = CDATA_RCDATA_CLOSE_TAG_STATE;
478     $self->{s_kwd} = '';
479     ## Reconsume.
480     redo A;
481     } else {
482     ## No start tag token has ever been emitted
483     ## NOTE: See <http://krijnhoetmer.nl/irc-logs/whatwg/20070626#l-564>.
484     !!!cp (28);
485     $self->{state} = DATA_STATE;
486     ## Reconsume.
487     !!!emit ({type => CHARACTER_TOKEN, data => '</',
488     line => $l, column => $c,
489     });
490     redo A;
491     }
492     }
493    
494     if (0x0041 <= $self->{nc} and
495     $self->{nc} <= 0x005A) { # A..Z
496     !!!cp (29);
497     $self->{ct}
498     = {type => END_TAG_TOKEN,
499     tag_name => chr ($self->{nc} + 0x0020),
500     line => $l, column => $c};
501     $self->{state} = TAG_NAME_STATE;
502     !!!next-input-character;
503     redo A;
504     } elsif (0x0061 <= $self->{nc} and
505     $self->{nc} <= 0x007A) { # a..z
506     !!!cp (30);
507     $self->{ct} = {type => END_TAG_TOKEN,
508     tag_name => chr ($self->{nc}),
509     line => $l, column => $c};
510     $self->{state} = TAG_NAME_STATE;
511     !!!next-input-character;
512     redo A;
513     } elsif ($self->{nc} == 0x003E) { # >
514     !!!cp (31);
515     !!!parse-error (type => 'empty end tag',
516     line => $self->{line_prev}, ## "<" in "</>"
517     column => $self->{column_prev} - 1);
518     $self->{state} = DATA_STATE;
519     !!!next-input-character;
520     redo A;
521     } elsif ($self->{nc} == -1) {
522     !!!cp (32);
523     !!!parse-error (type => 'bare etago');
524     $self->{state} = DATA_STATE;
525     # reconsume
526    
527     !!!emit ({type => CHARACTER_TOKEN, data => '</',
528     line => $l, column => $c,
529     });
530    
531     redo A;
532     } else {
533     !!!cp (33);
534     !!!parse-error (type => 'bogus end tag');
535     $self->{state} = BOGUS_COMMENT_STATE;
536     $self->{ct} = {type => COMMENT_TOKEN, data => '',
537     line => $self->{line_prev}, # "<" of "</"
538     column => $self->{column_prev} - 1,
539     };
540     ## NOTE: $self->{nc} is intentionally left as is.
541     ## Although the "anything else" case of the spec not explicitly
542     ## states that the next input character is to be reconsumed,
543     ## it will be included to the |data| of the comment token
544     ## generated from the bogus end tag, as defined in the
545     ## "bogus comment state" entry.
546     redo A;
547     }
548     } elsif ($self->{state} == CDATA_RCDATA_CLOSE_TAG_STATE) {
549     my $ch = substr $self->{last_stag_name}, length $self->{s_kwd}, 1;
550     if (length $ch) {
551     my $CH = $ch;
552     $ch =~ tr/a-z/A-Z/;
553     my $nch = chr $self->{nc};
554     if ($nch eq $ch or $nch eq $CH) {
555     !!!cp (24);
556     ## Stay in the state.
557     $self->{s_kwd} .= $nch;
558     !!!next-input-character;
559     redo A;
560     } else {
561     !!!cp (25);
562     $self->{state} = DATA_STATE;
563     ## Reconsume.
564     !!!emit ({type => CHARACTER_TOKEN,
565     data => '</' . $self->{s_kwd},
566     line => $self->{line_prev},
567     column => $self->{column_prev} - 1 - length $self->{s_kwd},
568     });
569     redo A;
570     }
571     } else { # after "<{tag-name}"
572     unless ($is_space->{$self->{nc}} or
573     {
574     0x003E => 1, # >
575     0x002F => 1, # /
576     -1 => 1, # EOF
577     }->{$self->{nc}}) {
578     !!!cp (26);
579     ## Reconsume.
580     $self->{state} = DATA_STATE;
581     !!!emit ({type => CHARACTER_TOKEN,
582     data => '</' . $self->{s_kwd},
583     line => $self->{line_prev},
584     column => $self->{column_prev} - 1 - length $self->{s_kwd},
585     });
586     redo A;
587     } else {
588     !!!cp (27);
589     $self->{ct}
590     = {type => END_TAG_TOKEN,
591     tag_name => $self->{last_stag_name},
592     line => $self->{line_prev},
593     column => $self->{column_prev} - 1 - length $self->{s_kwd}};
594     $self->{state} = TAG_NAME_STATE;
595     ## Reconsume.
596     redo A;
597     }
598     }
599     } elsif ($self->{state} == TAG_NAME_STATE) {
600     if ($is_space->{$self->{nc}}) {
601     !!!cp (34);
602     $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
603     !!!next-input-character;
604     redo A;
605     } elsif ($self->{nc} == 0x003E) { # >
606     if ($self->{ct}->{type} == START_TAG_TOKEN) {
607     !!!cp (35);
608     $self->{last_stag_name} = $self->{ct}->{tag_name};
609     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
610     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
611     #if ($self->{ct}->{attributes}) {
612     # ## NOTE: This should never be reached.
613     # !!! cp (36);
614     # !!! parse-error (type => 'end tag attribute');
615     #} else {
616     !!!cp (37);
617     #}
618     } else {
619     die "$0: $self->{ct}->{type}: Unknown token type";
620     }
621     $self->{state} = DATA_STATE;
622     !!!next-input-character;
623    
624     !!!emit ($self->{ct}); # start tag or end tag
625    
626     redo A;
627     } elsif (0x0041 <= $self->{nc} and
628     $self->{nc} <= 0x005A) { # A..Z
629     !!!cp (38);
630     $self->{ct}->{tag_name} .= chr ($self->{nc} + 0x0020);
631     # start tag or end tag
632     ## Stay in this state
633     !!!next-input-character;
634     redo A;
635     } elsif ($self->{nc} == -1) {
636     !!!parse-error (type => 'unclosed tag');
637     if ($self->{ct}->{type} == START_TAG_TOKEN) {
638     !!!cp (39);
639     $self->{last_stag_name} = $self->{ct}->{tag_name};
640     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
641     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
642     #if ($self->{ct}->{attributes}) {
643     # ## NOTE: This state should never be reached.
644     # !!! cp (40);
645     # !!! parse-error (type => 'end tag attribute');
646     #} else {
647     !!!cp (41);
648     #}
649     } else {
650     die "$0: $self->{ct}->{type}: Unknown token type";
651     }
652     $self->{state} = DATA_STATE;
653     # reconsume
654    
655     !!!emit ($self->{ct}); # start tag or end tag
656    
657     redo A;
658     } elsif ($self->{nc} == 0x002F) { # /
659     !!!cp (42);
660     $self->{state} = SELF_CLOSING_START_TAG_STATE;
661     !!!next-input-character;
662     redo A;
663     } else {
664     !!!cp (44);
665     $self->{ct}->{tag_name} .= chr $self->{nc};
666     # start tag or end tag
667     ## Stay in the state
668     !!!next-input-character;
669     redo A;
670     }
671     } elsif ($self->{state} == BEFORE_ATTRIBUTE_NAME_STATE) {
672     if ($is_space->{$self->{nc}}) {
673     !!!cp (45);
674     ## Stay in the state
675     !!!next-input-character;
676     redo A;
677     } elsif ($self->{nc} == 0x003E) { # >
678     if ($self->{ct}->{type} == START_TAG_TOKEN) {
679     !!!cp (46);
680     $self->{last_stag_name} = $self->{ct}->{tag_name};
681     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
682     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
683     if ($self->{ct}->{attributes}) {
684     !!!cp (47);
685     !!!parse-error (type => 'end tag attribute');
686     } else {
687     !!!cp (48);
688     }
689     } else {
690     die "$0: $self->{ct}->{type}: Unknown token type";
691     }
692     $self->{state} = DATA_STATE;
693     !!!next-input-character;
694    
695     !!!emit ($self->{ct}); # start tag or end tag
696    
697     redo A;
698     } elsif (0x0041 <= $self->{nc} and
699     $self->{nc} <= 0x005A) { # A..Z
700     !!!cp (49);
701     $self->{ca}
702     = {name => chr ($self->{nc} + 0x0020),
703     value => '',
704     line => $self->{line}, column => $self->{column}};
705     $self->{state} = ATTRIBUTE_NAME_STATE;
706     !!!next-input-character;
707     redo A;
708     } elsif ($self->{nc} == 0x002F) { # /
709     !!!cp (50);
710     $self->{state} = SELF_CLOSING_START_TAG_STATE;
711     !!!next-input-character;
712     redo A;
713     } elsif ($self->{nc} == -1) {
714     !!!parse-error (type => 'unclosed tag');
715     if ($self->{ct}->{type} == START_TAG_TOKEN) {
716     !!!cp (52);
717     $self->{last_stag_name} = $self->{ct}->{tag_name};
718     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
719     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
720     if ($self->{ct}->{attributes}) {
721     !!!cp (53);
722     !!!parse-error (type => 'end tag attribute');
723     } else {
724     !!!cp (54);
725     }
726     } else {
727     die "$0: $self->{ct}->{type}: Unknown token type";
728     }
729     $self->{state} = DATA_STATE;
730     # reconsume
731    
732     !!!emit ($self->{ct}); # start tag or end tag
733    
734     redo A;
735     } else {
736     if ({
737     0x0022 => 1, # "
738     0x0027 => 1, # '
739     0x003D => 1, # =
740     }->{$self->{nc}}) {
741     !!!cp (55);
742     !!!parse-error (type => 'bad attribute name');
743     } else {
744     !!!cp (56);
745     }
746     $self->{ca}
747     = {name => chr ($self->{nc}),
748     value => '',
749     line => $self->{line}, column => $self->{column}};
750     $self->{state} = ATTRIBUTE_NAME_STATE;
751     !!!next-input-character;
752     redo A;
753     }
754     } elsif ($self->{state} == ATTRIBUTE_NAME_STATE) {
755     my $before_leave = sub {
756     if (exists $self->{ct}->{attributes} # start tag or end tag
757     ->{$self->{ca}->{name}}) { # MUST
758     !!!cp (57);
759     !!!parse-error (type => 'duplicate attribute', text => $self->{ca}->{name}, line => $self->{ca}->{line}, column => $self->{ca}->{column});
760     ## Discard $self->{ca} # MUST
761     } else {
762     !!!cp (58);
763     $self->{ct}->{attributes}->{$self->{ca}->{name}}
764     = $self->{ca};
765     }
766     }; # $before_leave
767    
768     if ($is_space->{$self->{nc}}) {
769     !!!cp (59);
770     $before_leave->();
771     $self->{state} = AFTER_ATTRIBUTE_NAME_STATE;
772     !!!next-input-character;
773     redo A;
774     } elsif ($self->{nc} == 0x003D) { # =
775     !!!cp (60);
776     $before_leave->();
777     $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
778     !!!next-input-character;
779     redo A;
780     } elsif ($self->{nc} == 0x003E) { # >
781     $before_leave->();
782     if ($self->{ct}->{type} == START_TAG_TOKEN) {
783     !!!cp (61);
784     $self->{last_stag_name} = $self->{ct}->{tag_name};
785     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
786     !!!cp (62);
787     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
788     if ($self->{ct}->{attributes}) {
789     !!!parse-error (type => 'end tag attribute');
790     }
791     } else {
792     die "$0: $self->{ct}->{type}: Unknown token type";
793     }
794     $self->{state} = DATA_STATE;
795     !!!next-input-character;
796    
797     !!!emit ($self->{ct}); # start tag or end tag
798    
799     redo A;
800     } elsif (0x0041 <= $self->{nc} and
801     $self->{nc} <= 0x005A) { # A..Z
802     !!!cp (63);
803     $self->{ca}->{name} .= chr ($self->{nc} + 0x0020);
804     ## Stay in the state
805     !!!next-input-character;
806     redo A;
807     } elsif ($self->{nc} == 0x002F) { # /
808     !!!cp (64);
809     $before_leave->();
810     $self->{state} = SELF_CLOSING_START_TAG_STATE;
811     !!!next-input-character;
812     redo A;
813     } elsif ($self->{nc} == -1) {
814     !!!parse-error (type => 'unclosed tag');
815     $before_leave->();
816     if ($self->{ct}->{type} == START_TAG_TOKEN) {
817     !!!cp (66);
818     $self->{last_stag_name} = $self->{ct}->{tag_name};
819     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
820     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
821     if ($self->{ct}->{attributes}) {
822     !!!cp (67);
823     !!!parse-error (type => 'end tag attribute');
824     } else {
825     ## NOTE: This state should never be reached.
826     !!!cp (68);
827     }
828     } else {
829     die "$0: $self->{ct}->{type}: Unknown token type";
830     }
831     $self->{state} = DATA_STATE;
832     # reconsume
833    
834     !!!emit ($self->{ct}); # start tag or end tag
835    
836     redo A;
837     } else {
838     if ($self->{nc} == 0x0022 or # "
839     $self->{nc} == 0x0027) { # '
840     !!!cp (69);
841     !!!parse-error (type => 'bad attribute name');
842     } else {
843     !!!cp (70);
844     }
845     $self->{ca}->{name} .= chr ($self->{nc});
846     ## Stay in the state
847     !!!next-input-character;
848     redo A;
849     }
850     } elsif ($self->{state} == AFTER_ATTRIBUTE_NAME_STATE) {
851     if ($is_space->{$self->{nc}}) {
852     !!!cp (71);
853     ## Stay in the state
854     !!!next-input-character;
855     redo A;
856     } elsif ($self->{nc} == 0x003D) { # =
857     !!!cp (72);
858     $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
859     !!!next-input-character;
860     redo A;
861     } elsif ($self->{nc} == 0x003E) { # >
862     if ($self->{ct}->{type} == START_TAG_TOKEN) {
863     !!!cp (73);
864     $self->{last_stag_name} = $self->{ct}->{tag_name};
865     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
866     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
867     if ($self->{ct}->{attributes}) {
868     !!!cp (74);
869     !!!parse-error (type => 'end tag attribute');
870     } else {
871     ## NOTE: This state should never be reached.
872     !!!cp (75);
873     }
874     } else {
875     die "$0: $self->{ct}->{type}: Unknown token type";
876     }
877     $self->{state} = DATA_STATE;
878     !!!next-input-character;
879    
880     !!!emit ($self->{ct}); # start tag or end tag
881    
882     redo A;
883     } elsif (0x0041 <= $self->{nc} and
884     $self->{nc} <= 0x005A) { # A..Z
885     !!!cp (76);
886     $self->{ca}
887     = {name => chr ($self->{nc} + 0x0020),
888     value => '',
889     line => $self->{line}, column => $self->{column}};
890     $self->{state} = ATTRIBUTE_NAME_STATE;
891     !!!next-input-character;
892     redo A;
893     } elsif ($self->{nc} == 0x002F) { # /
894     !!!cp (77);
895     $self->{state} = SELF_CLOSING_START_TAG_STATE;
896     !!!next-input-character;
897     redo A;
898     } elsif ($self->{nc} == -1) {
899     !!!parse-error (type => 'unclosed tag');
900     if ($self->{ct}->{type} == START_TAG_TOKEN) {
901     !!!cp (79);
902     $self->{last_stag_name} = $self->{ct}->{tag_name};
903     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
904     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
905     if ($self->{ct}->{attributes}) {
906     !!!cp (80);
907     !!!parse-error (type => 'end tag attribute');
908     } else {
909     ## NOTE: This state should never be reached.
910     !!!cp (81);
911     }
912     } else {
913     die "$0: $self->{ct}->{type}: Unknown token type";
914     }
915     $self->{state} = DATA_STATE;
916     # reconsume
917    
918     !!!emit ($self->{ct}); # start tag or end tag
919    
920     redo A;
921     } else {
922     if ($self->{nc} == 0x0022 or # "
923     $self->{nc} == 0x0027) { # '
924     !!!cp (78);
925     !!!parse-error (type => 'bad attribute name');
926     } else {
927     !!!cp (82);
928     }
929     $self->{ca}
930     = {name => chr ($self->{nc}),
931     value => '',
932     line => $self->{line}, column => $self->{column}};
933     $self->{state} = ATTRIBUTE_NAME_STATE;
934     !!!next-input-character;
935     redo A;
936     }
937     } elsif ($self->{state} == BEFORE_ATTRIBUTE_VALUE_STATE) {
938     if ($is_space->{$self->{nc}}) {
939     !!!cp (83);
940     ## Stay in the state
941     !!!next-input-character;
942     redo A;
943     } elsif ($self->{nc} == 0x0022) { # "
944     !!!cp (84);
945     $self->{state} = ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE;
946     !!!next-input-character;
947     redo A;
948     } elsif ($self->{nc} == 0x0026) { # &
949     !!!cp (85);
950     $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
951     ## reconsume
952     redo A;
953     } elsif ($self->{nc} == 0x0027) { # '
954     !!!cp (86);
955     $self->{state} = ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE;
956     !!!next-input-character;
957     redo A;
958     } elsif ($self->{nc} == 0x003E) { # >
959     !!!parse-error (type => 'empty unquoted attribute value');
960     if ($self->{ct}->{type} == START_TAG_TOKEN) {
961     !!!cp (87);
962     $self->{last_stag_name} = $self->{ct}->{tag_name};
963     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
964     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
965     if ($self->{ct}->{attributes}) {
966     !!!cp (88);
967     !!!parse-error (type => 'end tag attribute');
968     } else {
969     ## NOTE: This state should never be reached.
970     !!!cp (89);
971     }
972     } else {
973     die "$0: $self->{ct}->{type}: Unknown token type";
974     }
975     $self->{state} = DATA_STATE;
976     !!!next-input-character;
977    
978     !!!emit ($self->{ct}); # start tag or end tag
979    
980     redo A;
981     } elsif ($self->{nc} == -1) {
982     !!!parse-error (type => 'unclosed tag');
983     if ($self->{ct}->{type} == START_TAG_TOKEN) {
984     !!!cp (90);
985     $self->{last_stag_name} = $self->{ct}->{tag_name};
986     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
987     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
988     if ($self->{ct}->{attributes}) {
989     !!!cp (91);
990     !!!parse-error (type => 'end tag attribute');
991     } else {
992     ## NOTE: This state should never be reached.
993     !!!cp (92);
994     }
995     } else {
996     die "$0: $self->{ct}->{type}: Unknown token type";
997     }
998     $self->{state} = DATA_STATE;
999     ## reconsume
1000    
1001     !!!emit ($self->{ct}); # start tag or end tag
1002    
1003     redo A;
1004     } else {
1005     if ($self->{nc} == 0x003D) { # =
1006     !!!cp (93);
1007     !!!parse-error (type => 'bad attribute value');
1008     } else {
1009     !!!cp (94);
1010     }
1011     $self->{ca}->{value} .= chr ($self->{nc});
1012     $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
1013     !!!next-input-character;
1014     redo A;
1015     }
1016     } elsif ($self->{state} == ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE) {
1017     if ($self->{nc} == 0x0022) { # "
1018     !!!cp (95);
1019     $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;
1020     !!!next-input-character;
1021     redo A;
1022     } elsif ($self->{nc} == 0x0026) { # &
1023     !!!cp (96);
1024     ## NOTE: In the spec, the tokenizer is switched to the
1025     ## "entity in attribute value state". In this implementation, the
1026     ## tokenizer is switched to the |ENTITY_STATE|, which is an
1027     ## implementation of the "consume a character reference" algorithm.
1028     $self->{prev_state} = $self->{state};
1029     $self->{entity_add} = 0x0022; # "
1030     $self->{state} = ENTITY_STATE;
1031     !!!next-input-character;
1032     redo A;
1033     } elsif ($self->{nc} == -1) {
1034     !!!parse-error (type => 'unclosed attribute value');
1035     if ($self->{ct}->{type} == START_TAG_TOKEN) {
1036     !!!cp (97);
1037     $self->{last_stag_name} = $self->{ct}->{tag_name};
1038     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1039     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1040     if ($self->{ct}->{attributes}) {
1041     !!!cp (98);
1042     !!!parse-error (type => 'end tag attribute');
1043     } else {
1044     ## NOTE: This state should never be reached.
1045     !!!cp (99);
1046     }
1047     } else {
1048     die "$0: $self->{ct}->{type}: Unknown token type";
1049     }
1050     $self->{state} = DATA_STATE;
1051     ## reconsume
1052    
1053     !!!emit ($self->{ct}); # start tag or end tag
1054    
1055     redo A;
1056     } else {
1057     !!!cp (100);
1058     $self->{ca}->{value} .= chr ($self->{nc});
1059     $self->{read_until}->($self->{ca}->{value},
1060     q["&],
1061     length $self->{ca}->{value});
1062    
1063     ## Stay in the state
1064     !!!next-input-character;
1065     redo A;
1066     }
1067     } elsif ($self->{state} == ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE) {
1068     if ($self->{nc} == 0x0027) { # '
1069     !!!cp (101);
1070     $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;
1071     !!!next-input-character;
1072     redo A;
1073     } elsif ($self->{nc} == 0x0026) { # &
1074     !!!cp (102);
1075     ## NOTE: In the spec, the tokenizer is switched to the
1076     ## "entity in attribute value state". In this implementation, the
1077     ## tokenizer is switched to the |ENTITY_STATE|, which is an
1078     ## implementation of the "consume a character reference" algorithm.
1079     $self->{entity_add} = 0x0027; # '
1080     $self->{prev_state} = $self->{state};
1081     $self->{state} = ENTITY_STATE;
1082     !!!next-input-character;
1083     redo A;
1084     } elsif ($self->{nc} == -1) {
1085     !!!parse-error (type => 'unclosed attribute value');
1086     if ($self->{ct}->{type} == START_TAG_TOKEN) {
1087     !!!cp (103);
1088     $self->{last_stag_name} = $self->{ct}->{tag_name};
1089     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1090     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1091     if ($self->{ct}->{attributes}) {
1092     !!!cp (104);
1093     !!!parse-error (type => 'end tag attribute');
1094     } else {
1095     ## NOTE: This state should never be reached.
1096     !!!cp (105);
1097     }
1098     } else {
1099     die "$0: $self->{ct}->{type}: Unknown token type";
1100     }
1101     $self->{state} = DATA_STATE;
1102     ## reconsume
1103    
1104     !!!emit ($self->{ct}); # start tag or end tag
1105    
1106     redo A;
1107     } else {
1108     !!!cp (106);
1109     $self->{ca}->{value} .= chr ($self->{nc});
1110     $self->{read_until}->($self->{ca}->{value},
1111     q['&],
1112     length $self->{ca}->{value});
1113    
1114     ## Stay in the state
1115     !!!next-input-character;
1116     redo A;
1117     }
1118     } elsif ($self->{state} == ATTRIBUTE_VALUE_UNQUOTED_STATE) {
1119     if ($is_space->{$self->{nc}}) {
1120     !!!cp (107);
1121     $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1122     !!!next-input-character;
1123     redo A;
1124     } elsif ($self->{nc} == 0x0026) { # &
1125     !!!cp (108);
1126     ## NOTE: In the spec, the tokenizer is switched to the
1127     ## "entity in attribute value state". In this implementation, the
1128     ## tokenizer is switched to the |ENTITY_STATE|, which is an
1129     ## implementation of the "consume a character reference" algorithm.
1130     $self->{entity_add} = -1;
1131     $self->{prev_state} = $self->{state};
1132     $self->{state} = ENTITY_STATE;
1133     !!!next-input-character;
1134     redo A;
1135     } elsif ($self->{nc} == 0x003E) { # >
1136     if ($self->{ct}->{type} == START_TAG_TOKEN) {
1137     !!!cp (109);
1138     $self->{last_stag_name} = $self->{ct}->{tag_name};
1139     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1140     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1141     if ($self->{ct}->{attributes}) {
1142     !!!cp (110);
1143     !!!parse-error (type => 'end tag attribute');
1144     } else {
1145     ## NOTE: This state should never be reached.
1146     !!!cp (111);
1147     }
1148     } else {
1149     die "$0: $self->{ct}->{type}: Unknown token type";
1150     }
1151     $self->{state} = DATA_STATE;
1152     !!!next-input-character;
1153    
1154     !!!emit ($self->{ct}); # start tag or end tag
1155    
1156     redo A;
1157     } elsif ($self->{nc} == -1) {
1158     !!!parse-error (type => 'unclosed tag');
1159     if ($self->{ct}->{type} == START_TAG_TOKEN) {
1160     !!!cp (112);
1161     $self->{last_stag_name} = $self->{ct}->{tag_name};
1162     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1163     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1164     if ($self->{ct}->{attributes}) {
1165     !!!cp (113);
1166     !!!parse-error (type => 'end tag attribute');
1167     } else {
1168     ## NOTE: This state should never be reached.
1169     !!!cp (114);
1170     }
1171     } else {
1172     die "$0: $self->{ct}->{type}: Unknown token type";
1173     }
1174     $self->{state} = DATA_STATE;
1175     ## reconsume
1176    
1177     !!!emit ($self->{ct}); # start tag or end tag
1178    
1179     redo A;
1180     } else {
1181     if ({
1182     0x0022 => 1, # "
1183     0x0027 => 1, # '
1184     0x003D => 1, # =
1185     }->{$self->{nc}}) {
1186     !!!cp (115);
1187     !!!parse-error (type => 'bad attribute value');
1188     } else {
1189     !!!cp (116);
1190     }
1191     $self->{ca}->{value} .= chr ($self->{nc});
1192     $self->{read_until}->($self->{ca}->{value},
1193     q["'=& >],
1194     length $self->{ca}->{value});
1195    
1196     ## Stay in the state
1197     !!!next-input-character;
1198     redo A;
1199     }
1200     } elsif ($self->{state} == AFTER_ATTRIBUTE_VALUE_QUOTED_STATE) {
1201     if ($is_space->{$self->{nc}}) {
1202     !!!cp (118);
1203     $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1204     !!!next-input-character;
1205     redo A;
1206     } elsif ($self->{nc} == 0x003E) { # >
1207     if ($self->{ct}->{type} == START_TAG_TOKEN) {
1208     !!!cp (119);
1209     $self->{last_stag_name} = $self->{ct}->{tag_name};
1210     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1211     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1212     if ($self->{ct}->{attributes}) {
1213     !!!cp (120);
1214     !!!parse-error (type => 'end tag attribute');
1215     } else {
1216     ## NOTE: This state should never be reached.
1217     !!!cp (121);
1218     }
1219     } else {
1220     die "$0: $self->{ct}->{type}: Unknown token type";
1221     }
1222     $self->{state} = DATA_STATE;
1223     !!!next-input-character;
1224    
1225     !!!emit ($self->{ct}); # start tag or end tag
1226    
1227     redo A;
1228     } elsif ($self->{nc} == 0x002F) { # /
1229     !!!cp (122);
1230     $self->{state} = SELF_CLOSING_START_TAG_STATE;
1231     !!!next-input-character;
1232     redo A;
1233     } elsif ($self->{nc} == -1) {
1234     !!!parse-error (type => 'unclosed tag');
1235     if ($self->{ct}->{type} == START_TAG_TOKEN) {
1236     !!!cp (122.3);
1237     $self->{last_stag_name} = $self->{ct}->{tag_name};
1238     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1239     if ($self->{ct}->{attributes}) {
1240     !!!cp (122.1);
1241     !!!parse-error (type => 'end tag attribute');
1242     } else {
1243     ## NOTE: This state should never be reached.
1244     !!!cp (122.2);
1245     }
1246     } else {
1247     die "$0: $self->{ct}->{type}: Unknown token type";
1248     }
1249     $self->{state} = DATA_STATE;
1250     ## Reconsume.
1251     !!!emit ($self->{ct}); # start tag or end tag
1252     redo A;
1253     } else {
1254     !!!cp ('124.1');
1255     !!!parse-error (type => 'no space between attributes');
1256     $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1257     ## reconsume
1258     redo A;
1259     }
1260     } elsif ($self->{state} == SELF_CLOSING_START_TAG_STATE) {
1261     if ($self->{nc} == 0x003E) { # >
1262     if ($self->{ct}->{type} == END_TAG_TOKEN) {
1263     !!!cp ('124.2');
1264     !!!parse-error (type => 'nestc', token => $self->{ct});
1265     ## TODO: Different type than slash in start tag
1266     $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1267     if ($self->{ct}->{attributes}) {
1268     !!!cp ('124.4');
1269     !!!parse-error (type => 'end tag attribute');
1270     } else {
1271     !!!cp ('124.5');
1272     }
1273     ## TODO: Test |<title></title/>|
1274     } else {
1275     !!!cp ('124.3');
1276     $self->{self_closing} = 1;
1277     }
1278    
1279     $self->{state} = DATA_STATE;
1280     !!!next-input-character;
1281    
1282     !!!emit ($self->{ct}); # start tag or end tag
1283    
1284     redo A;
1285     } elsif ($self->{nc} == -1) {
1286     !!!parse-error (type => 'unclosed tag');
1287     if ($self->{ct}->{type} == START_TAG_TOKEN) {
1288     !!!cp (124.7);
1289     $self->{last_stag_name} = $self->{ct}->{tag_name};
1290     } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1291     if ($self->{ct}->{attributes}) {
1292     !!!cp (124.5);
1293     !!!parse-error (type => 'end tag attribute');
1294     } else {
1295     ## NOTE: This state should never be reached.
1296     !!!cp (124.6);
1297     }
1298     } else {
1299     die "$0: $self->{ct}->{type}: Unknown token type";
1300     }
1301     $self->{state} = DATA_STATE;
1302     ## Reconsume.
1303     !!!emit ($self->{ct}); # start tag or end tag
1304     redo A;
1305     } else {
1306     !!!cp ('124.4');
1307     !!!parse-error (type => 'nestc');
1308     ## TODO: This error type is wrong.
1309     $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1310     ## Reconsume.
1311     redo A;
1312     }
1313     } elsif ($self->{state} == BOGUS_COMMENT_STATE) {
1314     ## (only happen if PCDATA state)
1315    
1316     ## NOTE: Unlike spec's "bogus comment state", this implementation
1317     ## consumes characters one-by-one basis.
1318    
1319     if ($self->{nc} == 0x003E) { # >
1320     !!!cp (124);
1321     $self->{state} = DATA_STATE;
1322     !!!next-input-character;
1323    
1324     !!!emit ($self->{ct}); # comment
1325     redo A;
1326     } elsif ($self->{nc} == -1) {
1327     !!!cp (125);
1328     $self->{state} = DATA_STATE;
1329     ## reconsume
1330    
1331     !!!emit ($self->{ct}); # comment
1332     redo A;
1333     } else {
1334     !!!cp (126);
1335     $self->{ct}->{data} .= chr ($self->{nc}); # comment
1336     $self->{read_until}->($self->{ct}->{data},
1337     q[>],
1338     length $self->{ct}->{data});
1339    
1340     ## Stay in the state.
1341     !!!next-input-character;
1342     redo A;
1343     }
1344     } elsif ($self->{state} == MARKUP_DECLARATION_OPEN_STATE) {
1345     ## (only happen if PCDATA state)
1346    
1347     if ($self->{nc} == 0x002D) { # -
1348     !!!cp (133);
1349     $self->{state} = MD_HYPHEN_STATE;
1350     !!!next-input-character;
1351     redo A;
1352     } elsif ($self->{nc} == 0x0044 or # D
1353     $self->{nc} == 0x0064) { # d
1354     ## ASCII case-insensitive.
1355     !!!cp (130);
1356     $self->{state} = MD_DOCTYPE_STATE;
1357     $self->{s_kwd} = chr $self->{nc};
1358     !!!next-input-character;
1359     redo A;
1360     } elsif ($self->{insertion_mode} & IN_FOREIGN_CONTENT_IM and
1361     $self->{open_elements}->[-1]->[1] & FOREIGN_EL and
1362     $self->{nc} == 0x005B) { # [
1363     !!!cp (135.4);
1364     $self->{state} = MD_CDATA_STATE;
1365     $self->{s_kwd} = '[';
1366     !!!next-input-character;
1367     redo A;
1368     } else {
1369     !!!cp (136);
1370     }
1371    
1372     !!!parse-error (type => 'bogus comment',
1373     line => $self->{line_prev},
1374     column => $self->{column_prev} - 1);
1375     ## Reconsume.
1376     $self->{state} = BOGUS_COMMENT_STATE;
1377     $self->{ct} = {type => COMMENT_TOKEN, data => '',
1378     line => $self->{line_prev},
1379     column => $self->{column_prev} - 1,
1380     };
1381     redo A;
1382     } elsif ($self->{state} == MD_HYPHEN_STATE) {
1383     if ($self->{nc} == 0x002D) { # -
1384     !!!cp (127);
1385     $self->{ct} = {type => COMMENT_TOKEN, data => '',
1386     line => $self->{line_prev},
1387     column => $self->{column_prev} - 2,
1388     };
1389     $self->{state} = COMMENT_START_STATE;
1390     !!!next-input-character;
1391     redo A;
1392     } else {
1393     !!!cp (128);
1394     !!!parse-error (type => 'bogus comment',
1395     line => $self->{line_prev},
1396     column => $self->{column_prev} - 2);
1397     $self->{state} = BOGUS_COMMENT_STATE;
1398     ## Reconsume.
1399     $self->{ct} = {type => COMMENT_TOKEN,
1400     data => '-',
1401     line => $self->{line_prev},
1402     column => $self->{column_prev} - 2,
1403     };
1404     redo A;
1405     }
1406     } elsif ($self->{state} == MD_DOCTYPE_STATE) {
1407     ## ASCII case-insensitive.
1408     if ($self->{nc} == [
1409     undef,
1410     0x004F, # O
1411     0x0043, # C
1412     0x0054, # T
1413     0x0059, # Y
1414     0x0050, # P
1415     ]->[length $self->{s_kwd}] or
1416     $self->{nc} == [
1417     undef,
1418     0x006F, # o
1419     0x0063, # c
1420     0x0074, # t
1421     0x0079, # y
1422     0x0070, # p
1423     ]->[length $self->{s_kwd}]) {
1424     !!!cp (131);
1425     ## Stay in the state.
1426     $self->{s_kwd} .= chr $self->{nc};
1427     !!!next-input-character;
1428     redo A;
1429     } elsif ((length $self->{s_kwd}) == 6 and
1430     ($self->{nc} == 0x0045 or # E
1431     $self->{nc} == 0x0065)) { # e
1432     !!!cp (129);
1433     $self->{state} = DOCTYPE_STATE;
1434     $self->{ct} = {type => DOCTYPE_TOKEN,
1435     quirks => 1,
1436     line => $self->{line_prev},
1437     column => $self->{column_prev} - 7,
1438     };
1439     !!!next-input-character;
1440     redo A;
1441     } else {
1442     !!!cp (132);
1443     !!!parse-error (type => 'bogus comment',
1444     line => $self->{line_prev},
1445     column => $self->{column_prev} - 1 - length $self->{s_kwd});
1446     $self->{state} = BOGUS_COMMENT_STATE;
1447     ## Reconsume.
1448     $self->{ct} = {type => COMMENT_TOKEN,
1449     data => $self->{s_kwd},
1450     line => $self->{line_prev},
1451     column => $self->{column_prev} - 1 - length $self->{s_kwd},
1452     };
1453     redo A;
1454     }
1455     } elsif ($self->{state} == MD_CDATA_STATE) {
1456     if ($self->{nc} == {
1457     '[' => 0x0043, # C
1458     '[C' => 0x0044, # D
1459     '[CD' => 0x0041, # A
1460     '[CDA' => 0x0054, # T
1461     '[CDAT' => 0x0041, # A
1462     }->{$self->{s_kwd}}) {
1463     !!!cp (135.1);
1464     ## Stay in the state.
1465     $self->{s_kwd} .= chr $self->{nc};
1466     !!!next-input-character;
1467     redo A;
1468     } elsif ($self->{s_kwd} eq '[CDATA' and
1469     $self->{nc} == 0x005B) { # [
1470     !!!cp (135.2);
1471     $self->{ct} = {type => CHARACTER_TOKEN,
1472     data => '',
1473     line => $self->{line_prev},
1474     column => $self->{column_prev} - 7};
1475     $self->{state} = CDATA_SECTION_STATE;
1476     !!!next-input-character;
1477     redo A;
1478     } else {
1479     !!!cp (135.3);
1480     !!!parse-error (type => 'bogus comment',
1481     line => $self->{line_prev},
1482     column => $self->{column_prev} - 1 - length $self->{s_kwd});
1483     $self->{state} = BOGUS_COMMENT_STATE;
1484     ## Reconsume.
1485     $self->{ct} = {type => COMMENT_TOKEN,
1486     data => $self->{s_kwd},
1487     line => $self->{line_prev},
1488     column => $self->{column_prev} - 1 - length $self->{s_kwd},
1489     };
1490     redo A;
1491     }
1492     } elsif ($self->{state} == COMMENT_START_STATE) {
1493     if ($self->{nc} == 0x002D) { # -
1494     !!!cp (137);
1495     $self->{state} = COMMENT_START_DASH_STATE;
1496     !!!next-input-character;
1497     redo A;
1498     } elsif ($self->{nc} == 0x003E) { # >
1499     !!!cp (138);
1500     !!!parse-error (type => 'bogus comment');
1501     $self->{state} = DATA_STATE;
1502     !!!next-input-character;
1503    
1504     !!!emit ($self->{ct}); # comment
1505    
1506     redo A;
1507     } elsif ($self->{nc} == -1) {
1508     !!!cp (139);
1509     !!!parse-error (type => 'unclosed comment');
1510     $self->{state} = DATA_STATE;
1511     ## reconsume
1512    
1513     !!!emit ($self->{ct}); # comment
1514    
1515     redo A;
1516     } else {
1517     !!!cp (140);
1518     $self->{ct}->{data} # comment
1519     .= chr ($self->{nc});
1520     $self->{state} = COMMENT_STATE;
1521     !!!next-input-character;
1522     redo A;
1523     }
1524     } elsif ($self->{state} == COMMENT_START_DASH_STATE) {
1525     if ($self->{nc} == 0x002D) { # -
1526     !!!cp (141);
1527     $self->{state} = COMMENT_END_STATE;
1528     !!!next-input-character;
1529     redo A;
1530     } elsif ($self->{nc} == 0x003E) { # >
1531     !!!cp (142);
1532     !!!parse-error (type => 'bogus comment');
1533     $self->{state} = DATA_STATE;
1534     !!!next-input-character;
1535    
1536     !!!emit ($self->{ct}); # comment
1537    
1538     redo A;
1539     } elsif ($self->{nc} == -1) {
1540     !!!cp (143);
1541     !!!parse-error (type => 'unclosed comment');
1542     $self->{state} = DATA_STATE;
1543     ## reconsume
1544    
1545     !!!emit ($self->{ct}); # comment
1546    
1547     redo A;
1548     } else {
1549     !!!cp (144);
1550     $self->{ct}->{data} # comment
1551     .= '-' . chr ($self->{nc});
1552     $self->{state} = COMMENT_STATE;
1553     !!!next-input-character;
1554     redo A;
1555     }
1556     } elsif ($self->{state} == COMMENT_STATE) {
1557     if ($self->{nc} == 0x002D) { # -
1558     !!!cp (145);
1559     $self->{state} = COMMENT_END_DASH_STATE;
1560     !!!next-input-character;
1561     redo A;
1562     } elsif ($self->{nc} == -1) {
1563     !!!cp (146);
1564     !!!parse-error (type => 'unclosed comment');
1565     $self->{state} = DATA_STATE;
1566     ## reconsume
1567    
1568     !!!emit ($self->{ct}); # comment
1569    
1570     redo A;
1571     } else {
1572     !!!cp (147);
1573     $self->{ct}->{data} .= chr ($self->{nc}); # comment
1574     $self->{read_until}->($self->{ct}->{data},
1575     q[-],
1576     length $self->{ct}->{data});
1577    
1578     ## Stay in the state
1579     !!!next-input-character;
1580     redo A;
1581     }
1582     } elsif ($self->{state} == COMMENT_END_DASH_STATE) {
1583     if ($self->{nc} == 0x002D) { # -
1584     !!!cp (148);
1585     $self->{state} = COMMENT_END_STATE;
1586     !!!next-input-character;
1587     redo A;
1588     } elsif ($self->{nc} == -1) {
1589     !!!cp (149);
1590     !!!parse-error (type => 'unclosed comment');
1591     $self->{state} = DATA_STATE;
1592     ## reconsume
1593    
1594     !!!emit ($self->{ct}); # comment
1595    
1596     redo A;
1597     } else {
1598     !!!cp (150);
1599     $self->{ct}->{data} .= '-' . chr ($self->{nc}); # comment
1600     $self->{state} = COMMENT_STATE;
1601     !!!next-input-character;
1602     redo A;
1603     }
1604     } elsif ($self->{state} == COMMENT_END_STATE) {
1605     if ($self->{nc} == 0x003E) { # >
1606     !!!cp (151);
1607     $self->{state} = DATA_STATE;
1608     !!!next-input-character;
1609    
1610     !!!emit ($self->{ct}); # comment
1611    
1612     redo A;
1613     } elsif ($self->{nc} == 0x002D) { # -
1614     !!!cp (152);
1615     !!!parse-error (type => 'dash in comment',
1616     line => $self->{line_prev},
1617     column => $self->{column_prev});
1618     $self->{ct}->{data} .= '-'; # comment
1619     ## Stay in the state
1620     !!!next-input-character;
1621     redo A;
1622     } elsif ($self->{nc} == -1) {
1623     !!!cp (153);
1624     !!!parse-error (type => 'unclosed comment');
1625     $self->{state} = DATA_STATE;
1626     ## reconsume
1627    
1628     !!!emit ($self->{ct}); # comment
1629    
1630     redo A;
1631     } else {
1632     !!!cp (154);
1633     !!!parse-error (type => 'dash in comment',
1634     line => $self->{line_prev},
1635     column => $self->{column_prev});
1636     $self->{ct}->{data} .= '--' . chr ($self->{nc}); # comment
1637     $self->{state} = COMMENT_STATE;
1638     !!!next-input-character;
1639     redo A;
1640     }
1641     } elsif ($self->{state} == DOCTYPE_STATE) {
1642     if ($is_space->{$self->{nc}}) {
1643     !!!cp (155);
1644     $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
1645     !!!next-input-character;
1646     redo A;
1647     } else {
1648     !!!cp (156);
1649     !!!parse-error (type => 'no space before DOCTYPE name');
1650     $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
1651     ## reconsume
1652     redo A;
1653     }
1654     } elsif ($self->{state} == BEFORE_DOCTYPE_NAME_STATE) {
1655     if ($is_space->{$self->{nc}}) {
1656     !!!cp (157);
1657     ## Stay in the state
1658     !!!next-input-character;
1659     redo A;
1660     } elsif ($self->{nc} == 0x003E) { # >
1661     !!!cp (158);
1662     !!!parse-error (type => 'no DOCTYPE name');
1663     $self->{state} = DATA_STATE;
1664     !!!next-input-character;
1665    
1666     !!!emit ($self->{ct}); # DOCTYPE (quirks)
1667    
1668     redo A;
1669     } elsif ($self->{nc} == -1) {
1670     !!!cp (159);
1671     !!!parse-error (type => 'no DOCTYPE name');
1672     $self->{state} = DATA_STATE;
1673     ## reconsume
1674    
1675     !!!emit ($self->{ct}); # DOCTYPE (quirks)
1676    
1677     redo A;
1678     } else {
1679     !!!cp (160);
1680     $self->{ct}->{name} = chr $self->{nc};
1681     delete $self->{ct}->{quirks};
1682     $self->{state} = DOCTYPE_NAME_STATE;
1683     !!!next-input-character;
1684     redo A;
1685     }
1686     } elsif ($self->{state} == DOCTYPE_NAME_STATE) {
1687     ## ISSUE: Redundant "First," in the spec.
1688     if ($is_space->{$self->{nc}}) {
1689     !!!cp (161);
1690     $self->{state} = AFTER_DOCTYPE_NAME_STATE;
1691     !!!next-input-character;
1692     redo A;
1693     } elsif ($self->{nc} == 0x003E) { # >
1694     !!!cp (162);
1695     $self->{state} = DATA_STATE;
1696     !!!next-input-character;
1697    
1698     !!!emit ($self->{ct}); # DOCTYPE
1699    
1700     redo A;
1701     } elsif ($self->{nc} == -1) {
1702     !!!cp (163);
1703     !!!parse-error (type => 'unclosed DOCTYPE');
1704     $self->{state} = DATA_STATE;
1705     ## reconsume
1706    
1707     $self->{ct}->{quirks} = 1;
1708     !!!emit ($self->{ct}); # DOCTYPE
1709    
1710     redo A;
1711     } else {
1712     !!!cp (164);
1713     $self->{ct}->{name}
1714     .= chr ($self->{nc}); # DOCTYPE
1715     ## Stay in the state
1716     !!!next-input-character;
1717     redo A;
1718     }
1719     } elsif ($self->{state} == AFTER_DOCTYPE_NAME_STATE) {
1720     if ($is_space->{$self->{nc}}) {
1721     !!!cp (165);
1722     ## Stay in the state
1723     !!!next-input-character;
1724     redo A;
1725     } elsif ($self->{nc} == 0x003E) { # >
1726     !!!cp (166);
1727     $self->{state} = DATA_STATE;
1728     !!!next-input-character;
1729    
1730     !!!emit ($self->{ct}); # DOCTYPE
1731    
1732     redo A;
1733     } elsif ($self->{nc} == -1) {
1734     !!!cp (167);
1735     !!!parse-error (type => 'unclosed DOCTYPE');
1736     $self->{state} = DATA_STATE;
1737     ## reconsume
1738    
1739     $self->{ct}->{quirks} = 1;
1740     !!!emit ($self->{ct}); # DOCTYPE
1741    
1742     redo A;
1743     } elsif ($self->{nc} == 0x0050 or # P
1744     $self->{nc} == 0x0070) { # p
1745     $self->{state} = PUBLIC_STATE;
1746     $self->{s_kwd} = chr $self->{nc};
1747     !!!next-input-character;
1748     redo A;
1749     } elsif ($self->{nc} == 0x0053 or # S
1750     $self->{nc} == 0x0073) { # s
1751     $self->{state} = SYSTEM_STATE;
1752     $self->{s_kwd} = chr $self->{nc};
1753     !!!next-input-character;
1754     redo A;
1755     } else {
1756     !!!cp (180);
1757     !!!parse-error (type => 'string after DOCTYPE name');
1758     $self->{ct}->{quirks} = 1;
1759    
1760     $self->{state} = BOGUS_DOCTYPE_STATE;
1761     !!!next-input-character;
1762     redo A;
1763     }
1764     } elsif ($self->{state} == PUBLIC_STATE) {
1765     ## ASCII case-insensitive
1766     if ($self->{nc} == [
1767     undef,
1768     0x0055, # U
1769     0x0042, # B
1770     0x004C, # L
1771     0x0049, # I
1772     ]->[length $self->{s_kwd}] or
1773     $self->{nc} == [
1774     undef,
1775     0x0075, # u
1776     0x0062, # b
1777     0x006C, # l
1778     0x0069, # i
1779     ]->[length $self->{s_kwd}]) {
1780     !!!cp (175);
1781     ## Stay in the state.
1782     $self->{s_kwd} .= chr $self->{nc};
1783     !!!next-input-character;
1784     redo A;
1785     } elsif ((length $self->{s_kwd}) == 5 and
1786     ($self->{nc} == 0x0043 or # C
1787     $self->{nc} == 0x0063)) { # c
1788     !!!cp (168);
1789     $self->{state} = BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1790     !!!next-input-character;
1791     redo A;
1792     } else {
1793     !!!cp (169);
1794     !!!parse-error (type => 'string after DOCTYPE name',
1795     line => $self->{line_prev},
1796     column => $self->{column_prev} + 1 - length $self->{s_kwd});
1797     $self->{ct}->{quirks} = 1;
1798    
1799     $self->{state} = BOGUS_DOCTYPE_STATE;
1800     ## Reconsume.
1801     redo A;
1802     }
1803     } elsif ($self->{state} == SYSTEM_STATE) {
1804     ## ASCII case-insensitive
1805     if ($self->{nc} == [
1806     undef,
1807     0x0059, # Y
1808     0x0053, # S
1809     0x0054, # T
1810     0x0045, # E
1811     ]->[length $self->{s_kwd}] or
1812     $self->{nc} == [
1813     undef,
1814     0x0079, # y
1815     0x0073, # s
1816     0x0074, # t
1817     0x0065, # e
1818     ]->[length $self->{s_kwd}]) {
1819     !!!cp (170);
1820     ## Stay in the state.
1821     $self->{s_kwd} .= chr $self->{nc};
1822     !!!next-input-character;
1823     redo A;
1824     } elsif ((length $self->{s_kwd}) == 5 and
1825     ($self->{nc} == 0x004D or # M
1826     $self->{nc} == 0x006D)) { # m
1827     !!!cp (171);
1828     $self->{state} = BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
1829     !!!next-input-character;
1830     redo A;
1831     } else {
1832     !!!cp (172);
1833     !!!parse-error (type => 'string after DOCTYPE name',
1834     line => $self->{line_prev},
1835     column => $self->{column_prev} + 1 - length $self->{s_kwd});
1836     $self->{ct}->{quirks} = 1;
1837    
1838     $self->{state} = BOGUS_DOCTYPE_STATE;
1839     ## Reconsume.
1840     redo A;
1841     }
1842     } elsif ($self->{state} == BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
1843     if ($is_space->{$self->{nc}}) {
1844     !!!cp (181);
1845     ## Stay in the state
1846     !!!next-input-character;
1847     redo A;
1848     } elsif ($self->{nc} eq 0x0022) { # "
1849     !!!cp (182);
1850     $self->{ct}->{pubid} = ''; # DOCTYPE
1851     $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE;
1852     !!!next-input-character;
1853     redo A;
1854     } elsif ($self->{nc} eq 0x0027) { # '
1855     !!!cp (183);
1856     $self->{ct}->{pubid} = ''; # DOCTYPE
1857     $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE;
1858     !!!next-input-character;
1859     redo A;
1860     } elsif ($self->{nc} eq 0x003E) { # >
1861     !!!cp (184);
1862     !!!parse-error (type => 'no PUBLIC literal');
1863    
1864     $self->{state} = DATA_STATE;
1865     !!!next-input-character;
1866    
1867     $self->{ct}->{quirks} = 1;
1868     !!!emit ($self->{ct}); # DOCTYPE
1869    
1870     redo A;
1871     } elsif ($self->{nc} == -1) {
1872     !!!cp (185);
1873     !!!parse-error (type => 'unclosed DOCTYPE');
1874    
1875     $self->{state} = DATA_STATE;
1876     ## reconsume
1877    
1878     $self->{ct}->{quirks} = 1;
1879     !!!emit ($self->{ct}); # DOCTYPE
1880    
1881     redo A;
1882     } else {
1883     !!!cp (186);
1884     !!!parse-error (type => 'string after PUBLIC');
1885     $self->{ct}->{quirks} = 1;
1886    
1887     $self->{state} = BOGUS_DOCTYPE_STATE;
1888     !!!next-input-character;
1889     redo A;
1890     }
1891     } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE) {
1892     if ($self->{nc} == 0x0022) { # "
1893     !!!cp (187);
1894     $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1895     !!!next-input-character;
1896     redo A;
1897     } elsif ($self->{nc} == 0x003E) { # >
1898     !!!cp (188);
1899     !!!parse-error (type => 'unclosed PUBLIC literal');
1900    
1901     $self->{state} = DATA_STATE;
1902     !!!next-input-character;
1903    
1904     $self->{ct}->{quirks} = 1;
1905     !!!emit ($self->{ct}); # DOCTYPE
1906    
1907     redo A;
1908     } elsif ($self->{nc} == -1) {
1909     !!!cp (189);
1910     !!!parse-error (type => 'unclosed PUBLIC literal');
1911    
1912     $self->{state} = DATA_STATE;
1913     ## reconsume
1914    
1915     $self->{ct}->{quirks} = 1;
1916     !!!emit ($self->{ct}); # DOCTYPE
1917    
1918     redo A;
1919     } else {
1920     !!!cp (190);
1921     $self->{ct}->{pubid} # DOCTYPE
1922     .= chr $self->{nc};
1923     $self->{read_until}->($self->{ct}->{pubid}, q[">],
1924     length $self->{ct}->{pubid});
1925    
1926     ## Stay in the state
1927     !!!next-input-character;
1928     redo A;
1929     }
1930     } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE) {
1931     if ($self->{nc} == 0x0027) { # '
1932     !!!cp (191);
1933     $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
1934     !!!next-input-character;
1935     redo A;
1936     } elsif ($self->{nc} == 0x003E) { # >
1937     !!!cp (192);
1938     !!!parse-error (type => 'unclosed PUBLIC literal');
1939    
1940     $self->{state} = DATA_STATE;
1941     !!!next-input-character;
1942    
1943     $self->{ct}->{quirks} = 1;
1944     !!!emit ($self->{ct}); # DOCTYPE
1945    
1946     redo A;
1947     } elsif ($self->{nc} == -1) {
1948     !!!cp (193);
1949     !!!parse-error (type => 'unclosed PUBLIC literal');
1950    
1951     $self->{state} = DATA_STATE;
1952     ## reconsume
1953    
1954     $self->{ct}->{quirks} = 1;
1955     !!!emit ($self->{ct}); # DOCTYPE
1956    
1957     redo A;
1958     } else {
1959     !!!cp (194);
1960     $self->{ct}->{pubid} # DOCTYPE
1961     .= chr $self->{nc};
1962     $self->{read_until}->($self->{ct}->{pubid}, q['>],
1963     length $self->{ct}->{pubid});
1964    
1965     ## Stay in the state
1966     !!!next-input-character;
1967     redo A;
1968     }
1969     } elsif ($self->{state} == AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
1970     if ($is_space->{$self->{nc}}) {
1971     !!!cp (195);
1972     ## Stay in the state
1973     !!!next-input-character;
1974     redo A;
1975     } elsif ($self->{nc} == 0x0022) { # "
1976     !!!cp (196);
1977     $self->{ct}->{sysid} = ''; # DOCTYPE
1978     $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
1979     !!!next-input-character;
1980     redo A;
1981     } elsif ($self->{nc} == 0x0027) { # '
1982     !!!cp (197);
1983     $self->{ct}->{sysid} = ''; # DOCTYPE
1984     $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
1985     !!!next-input-character;
1986     redo A;
1987     } elsif ($self->{nc} == 0x003E) { # >
1988     !!!cp (198);
1989     $self->{state} = DATA_STATE;
1990     !!!next-input-character;
1991    
1992     !!!emit ($self->{ct}); # DOCTYPE
1993    
1994     redo A;
1995     } elsif ($self->{nc} == -1) {
1996     !!!cp (199);
1997     !!!parse-error (type => 'unclosed DOCTYPE');
1998    
1999     $self->{state} = DATA_STATE;
2000     ## reconsume
2001    
2002     $self->{ct}->{quirks} = 1;
2003     !!!emit ($self->{ct}); # DOCTYPE
2004    
2005     redo A;
2006     } else {
2007     !!!cp (200);
2008     !!!parse-error (type => 'string after PUBLIC literal');
2009     $self->{ct}->{quirks} = 1;
2010    
2011     $self->{state} = BOGUS_DOCTYPE_STATE;
2012     !!!next-input-character;
2013     redo A;
2014     }
2015     } elsif ($self->{state} == BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
2016     if ($is_space->{$self->{nc}}) {
2017     !!!cp (201);
2018     ## Stay in the state
2019     !!!next-input-character;
2020     redo A;
2021     } elsif ($self->{nc} == 0x0022) { # "
2022     !!!cp (202);
2023     $self->{ct}->{sysid} = ''; # DOCTYPE
2024     $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
2025     !!!next-input-character;
2026     redo A;
2027     } elsif ($self->{nc} == 0x0027) { # '
2028     !!!cp (203);
2029     $self->{ct}->{sysid} = ''; # DOCTYPE
2030     $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
2031     !!!next-input-character;
2032     redo A;
2033     } elsif ($self->{nc} == 0x003E) { # >
2034     !!!cp (204);
2035     !!!parse-error (type => 'no SYSTEM literal');
2036     $self->{state} = DATA_STATE;
2037     !!!next-input-character;
2038    
2039     $self->{ct}->{quirks} = 1;
2040     !!!emit ($self->{ct}); # DOCTYPE
2041    
2042     redo A;
2043     } elsif ($self->{nc} == -1) {
2044     !!!cp (205);
2045     !!!parse-error (type => 'unclosed DOCTYPE');
2046    
2047     $self->{state} = DATA_STATE;
2048     ## reconsume
2049    
2050     $self->{ct}->{quirks} = 1;
2051     !!!emit ($self->{ct}); # DOCTYPE
2052    
2053     redo A;
2054     } else {
2055     !!!cp (206);
2056     !!!parse-error (type => 'string after SYSTEM');
2057     $self->{ct}->{quirks} = 1;
2058    
2059     $self->{state} = BOGUS_DOCTYPE_STATE;
2060     !!!next-input-character;
2061     redo A;
2062     }
2063     } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE) {
2064     if ($self->{nc} == 0x0022) { # "
2065     !!!cp (207);
2066     $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
2067     !!!next-input-character;
2068     redo A;
2069     } elsif ($self->{nc} == 0x003E) { # >
2070     !!!cp (208);
2071     !!!parse-error (type => 'unclosed SYSTEM literal');
2072    
2073     $self->{state} = DATA_STATE;
2074     !!!next-input-character;
2075    
2076     $self->{ct}->{quirks} = 1;
2077     !!!emit ($self->{ct}); # DOCTYPE
2078    
2079     redo A;
2080     } elsif ($self->{nc} == -1) {
2081     !!!cp (209);
2082     !!!parse-error (type => 'unclosed SYSTEM literal');
2083    
2084     $self->{state} = DATA_STATE;
2085     ## reconsume
2086    
2087     $self->{ct}->{quirks} = 1;
2088     !!!emit ($self->{ct}); # DOCTYPE
2089    
2090     redo A;
2091     } else {
2092     !!!cp (210);
2093     $self->{ct}->{sysid} # DOCTYPE
2094     .= chr $self->{nc};
2095     $self->{read_until}->($self->{ct}->{sysid}, q[">],
2096     length $self->{ct}->{sysid});
2097    
2098     ## Stay in the state
2099     !!!next-input-character;
2100     redo A;
2101     }
2102     } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE) {
2103     if ($self->{nc} == 0x0027) { # '
2104     !!!cp (211);
2105     $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
2106     !!!next-input-character;
2107     redo A;
2108     } elsif ($self->{nc} == 0x003E) { # >
2109     !!!cp (212);
2110     !!!parse-error (type => 'unclosed SYSTEM literal');
2111    
2112     $self->{state} = DATA_STATE;
2113     !!!next-input-character;
2114    
2115     $self->{ct}->{quirks} = 1;
2116     !!!emit ($self->{ct}); # DOCTYPE
2117    
2118     redo A;
2119     } elsif ($self->{nc} == -1) {
2120     !!!cp (213);
2121     !!!parse-error (type => 'unclosed SYSTEM literal');
2122    
2123     $self->{state} = DATA_STATE;
2124     ## reconsume
2125    
2126     $self->{ct}->{quirks} = 1;
2127     !!!emit ($self->{ct}); # DOCTYPE
2128    
2129     redo A;
2130     } else {
2131     !!!cp (214);
2132     $self->{ct}->{sysid} # DOCTYPE
2133     .= chr $self->{nc};
2134     $self->{read_until}->($self->{ct}->{sysid}, q['>],
2135     length $self->{ct}->{sysid});
2136    
2137     ## Stay in the state
2138     !!!next-input-character;
2139     redo A;
2140     }
2141     } elsif ($self->{state} == AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
2142     if ($is_space->{$self->{nc}}) {
2143     !!!cp (215);
2144     ## Stay in the state
2145     !!!next-input-character;
2146     redo A;
2147     } elsif ($self->{nc} == 0x003E) { # >
2148     !!!cp (216);
2149     $self->{state} = DATA_STATE;
2150     !!!next-input-character;
2151    
2152     !!!emit ($self->{ct}); # DOCTYPE
2153    
2154     redo A;
2155     } elsif ($self->{nc} == -1) {
2156     !!!cp (217);
2157     !!!parse-error (type => 'unclosed DOCTYPE');
2158     $self->{state} = DATA_STATE;
2159     ## reconsume
2160    
2161     $self->{ct}->{quirks} = 1;
2162     !!!emit ($self->{ct}); # DOCTYPE
2163    
2164     redo A;
2165     } else {
2166     !!!cp (218);
2167     !!!parse-error (type => 'string after SYSTEM literal');
2168     #$self->{ct}->{quirks} = 1;
2169    
2170     $self->{state} = BOGUS_DOCTYPE_STATE;
2171     !!!next-input-character;
2172     redo A;
2173     }
2174     } elsif ($self->{state} == BOGUS_DOCTYPE_STATE) {
2175     if ($self->{nc} == 0x003E) { # >
2176     !!!cp (219);
2177     $self->{state} = DATA_STATE;
2178     !!!next-input-character;
2179    
2180     !!!emit ($self->{ct}); # DOCTYPE
2181    
2182     redo A;
2183     } elsif ($self->{nc} == -1) {
2184     !!!cp (220);
2185     $self->{state} = DATA_STATE;
2186     ## reconsume
2187    
2188     !!!emit ($self->{ct}); # DOCTYPE
2189    
2190     redo A;
2191     } else {
2192     !!!cp (221);
2193     my $s = '';
2194     $self->{read_until}->($s, q[>], 0);
2195    
2196     ## Stay in the state
2197     !!!next-input-character;
2198     redo A;
2199     }
2200     } elsif ($self->{state} == CDATA_SECTION_STATE) {
2201     ## NOTE: "CDATA section state" in the state is jointly implemented
2202     ## by three states, |CDATA_SECTION_STATE|, |CDATA_SECTION_MSE1_STATE|,
2203     ## and |CDATA_SECTION_MSE2_STATE|.
2204    
2205     if ($self->{nc} == 0x005D) { # ]
2206     !!!cp (221.1);
2207     $self->{state} = CDATA_SECTION_MSE1_STATE;
2208     !!!next-input-character;
2209     redo A;
2210     } elsif ($self->{nc} == -1) {
2211     $self->{state} = DATA_STATE;
2212     !!!next-input-character;
2213     if (length $self->{ct}->{data}) { # character
2214     !!!cp (221.2);
2215     !!!emit ($self->{ct}); # character
2216     } else {
2217     !!!cp (221.3);
2218     ## No token to emit. $self->{ct} is discarded.
2219     }
2220     redo A;
2221     } else {
2222     !!!cp (221.4);
2223     $self->{ct}->{data} .= chr $self->{nc};
2224     $self->{read_until}->($self->{ct}->{data},
2225     q<]>,
2226     length $self->{ct}->{data});
2227    
2228     ## Stay in the state.
2229     !!!next-input-character;
2230     redo A;
2231     }
2232    
2233     ## ISSUE: "text tokens" in spec.
2234     } elsif ($self->{state} == CDATA_SECTION_MSE1_STATE) {
2235     if ($self->{nc} == 0x005D) { # ]
2236     !!!cp (221.5);
2237     $self->{state} = CDATA_SECTION_MSE2_STATE;
2238     !!!next-input-character;
2239     redo A;
2240     } else {
2241     !!!cp (221.6);
2242     $self->{ct}->{data} .= ']';
2243     $self->{state} = CDATA_SECTION_STATE;
2244     ## Reconsume.
2245     redo A;
2246     }
2247     } elsif ($self->{state} == CDATA_SECTION_MSE2_STATE) {
2248     if ($self->{nc} == 0x003E) { # >
2249     $self->{state} = DATA_STATE;
2250     !!!next-input-character;
2251     if (length $self->{ct}->{data}) { # character
2252     !!!cp (221.7);
2253     !!!emit ($self->{ct}); # character
2254     } else {
2255     !!!cp (221.8);
2256     ## No token to emit. $self->{ct} is discarded.
2257     }
2258     redo A;
2259     } elsif ($self->{nc} == 0x005D) { # ]
2260     !!!cp (221.9); # character
2261     $self->{ct}->{data} .= ']'; ## Add first "]" of "]]]".
2262     ## Stay in the state.
2263     !!!next-input-character;
2264     redo A;
2265     } else {
2266     !!!cp (221.11);
2267     $self->{ct}->{data} .= ']]'; # character
2268     $self->{state} = CDATA_SECTION_STATE;
2269     ## Reconsume.
2270     redo A;
2271     }
2272     } elsif ($self->{state} == ENTITY_STATE) {
2273     if ($is_space->{$self->{nc}} or
2274     {
2275     0x003C => 1, 0x0026 => 1, -1 => 1, # <, &
2276     $self->{entity_add} => 1,
2277     }->{$self->{nc}}) {
2278     !!!cp (1001);
2279     ## Don't consume
2280     ## No error
2281     ## Return nothing.
2282     #
2283     } elsif ($self->{nc} == 0x0023) { # #
2284     !!!cp (999);
2285     $self->{state} = ENTITY_HASH_STATE;
2286     $self->{s_kwd} = '#';
2287     !!!next-input-character;
2288     redo A;
2289     } elsif ((0x0041 <= $self->{nc} and
2290     $self->{nc} <= 0x005A) or # A..Z
2291     (0x0061 <= $self->{nc} and
2292     $self->{nc} <= 0x007A)) { # a..z
2293     !!!cp (998);
2294     require Whatpm::_NamedEntityList;
2295     $self->{state} = ENTITY_NAME_STATE;
2296     $self->{s_kwd} = chr $self->{nc};
2297     $self->{entity__value} = $self->{s_kwd};
2298     $self->{entity__match} = 0;
2299     !!!next-input-character;
2300     redo A;
2301     } else {
2302     !!!cp (1027);
2303     !!!parse-error (type => 'bare ero');
2304     ## Return nothing.
2305     #
2306     }
2307    
2308     ## NOTE: No character is consumed by the "consume a character
2309     ## reference" algorithm. In other word, there is an "&" character
2310     ## that does not introduce a character reference, which would be
2311     ## appended to the parent element or the attribute value in later
2312     ## process of the tokenizer.
2313    
2314     if ($self->{prev_state} == DATA_STATE) {
2315     !!!cp (997);
2316     $self->{state} = $self->{prev_state};
2317     ## Reconsume.
2318     !!!emit ({type => CHARACTER_TOKEN, data => '&',
2319     line => $self->{line_prev},
2320     column => $self->{column_prev},
2321     });
2322     redo A;
2323     } else {
2324     !!!cp (996);
2325     $self->{ca}->{value} .= '&';
2326     $self->{state} = $self->{prev_state};
2327     ## Reconsume.
2328     redo A;
2329     }
2330     } elsif ($self->{state} == ENTITY_HASH_STATE) {
2331     if ($self->{nc} == 0x0078 or # x
2332     $self->{nc} == 0x0058) { # X
2333     !!!cp (995);
2334     $self->{state} = HEXREF_X_STATE;
2335     $self->{s_kwd} .= chr $self->{nc};
2336     !!!next-input-character;
2337     redo A;
2338     } elsif (0x0030 <= $self->{nc} and
2339     $self->{nc} <= 0x0039) { # 0..9
2340     !!!cp (994);
2341     $self->{state} = NCR_NUM_STATE;
2342     $self->{s_kwd} = $self->{nc} - 0x0030;
2343     !!!next-input-character;
2344     redo A;
2345     } else {
2346     !!!parse-error (type => 'bare nero',
2347     line => $self->{line_prev},
2348     column => $self->{column_prev} - 1);
2349    
2350     ## NOTE: According to the spec algorithm, nothing is returned,
2351     ## and then "&#" is appended to the parent element or the attribute
2352     ## value in the later processing.
2353    
2354     if ($self->{prev_state} == DATA_STATE) {
2355     !!!cp (1019);
2356     $self->{state} = $self->{prev_state};
2357     ## Reconsume.
2358     !!!emit ({type => CHARACTER_TOKEN,
2359     data => '&#',
2360     line => $self->{line_prev},
2361     column => $self->{column_prev} - 1,
2362     });
2363     redo A;
2364     } else {
2365     !!!cp (993);
2366     $self->{ca}->{value} .= '&#';
2367     $self->{state} = $self->{prev_state};
2368     ## Reconsume.
2369     redo A;
2370     }
2371     }
2372     } elsif ($self->{state} == NCR_NUM_STATE) {
2373     if (0x0030 <= $self->{nc} and
2374     $self->{nc} <= 0x0039) { # 0..9
2375     !!!cp (1012);
2376     $self->{s_kwd} *= 10;
2377     $self->{s_kwd} += $self->{nc} - 0x0030;
2378    
2379     ## Stay in the state.
2380     !!!next-input-character;
2381     redo A;
2382     } elsif ($self->{nc} == 0x003B) { # ;
2383     !!!cp (1013);
2384     !!!next-input-character;
2385     #
2386     } else {
2387     !!!cp (1014);
2388     !!!parse-error (type => 'no refc');
2389     ## Reconsume.
2390     #
2391     }
2392    
2393     my $code = $self->{s_kwd};
2394     my $l = $self->{line_prev};
2395     my $c = $self->{column_prev};
2396     if ($charref_map->{$code}) {
2397     !!!cp (1015);
2398     !!!parse-error (type => 'invalid character reference',
2399     text => (sprintf 'U+%04X', $code),
2400     line => $l, column => $c);
2401     $code = $charref_map->{$code};
2402     } elsif ($code > 0x10FFFF) {
2403     !!!cp (1016);
2404     !!!parse-error (type => 'invalid character reference',
2405     text => (sprintf 'U-%08X', $code),
2406     line => $l, column => $c);
2407     $code = 0xFFFD;
2408     }
2409    
2410     if ($self->{prev_state} == DATA_STATE) {
2411     !!!cp (992);
2412     $self->{state} = $self->{prev_state};
2413     ## Reconsume.
2414     !!!emit ({type => CHARACTER_TOKEN, data => chr $code,
2415     line => $l, column => $c,
2416     });
2417     redo A;
2418     } else {
2419     !!!cp (991);
2420     $self->{ca}->{value} .= chr $code;
2421     $self->{ca}->{has_reference} = 1;
2422     $self->{state} = $self->{prev_state};
2423     ## Reconsume.
2424     redo A;
2425     }
2426     } elsif ($self->{state} == HEXREF_X_STATE) {
2427     if ((0x0030 <= $self->{nc} and $self->{nc} <= 0x0039) or
2428     (0x0041 <= $self->{nc} and $self->{nc} <= 0x0046) or
2429     (0x0061 <= $self->{nc} and $self->{nc} <= 0x0066)) {
2430     # 0..9, A..F, a..f
2431     !!!cp (990);
2432     $self->{state} = HEXREF_HEX_STATE;
2433     $self->{s_kwd} = 0;
2434     ## Reconsume.
2435     redo A;
2436     } else {
2437     !!!parse-error (type => 'bare hcro',
2438     line => $self->{line_prev},
2439     column => $self->{column_prev} - 2);
2440    
2441     ## NOTE: According to the spec algorithm, nothing is returned,
2442     ## and then "&#" followed by "X" or "x" is appended to the parent
2443     ## element or the attribute value in the later processing.
2444    
2445     if ($self->{prev_state} == DATA_STATE) {
2446     !!!cp (1005);
2447     $self->{state} = $self->{prev_state};
2448     ## Reconsume.
2449     !!!emit ({type => CHARACTER_TOKEN,
2450     data => '&' . $self->{s_kwd},
2451     line => $self->{line_prev},
2452     column => $self->{column_prev} - length $self->{s_kwd},
2453     });
2454     redo A;
2455     } else {
2456     !!!cp (989);
2457     $self->{ca}->{value} .= '&' . $self->{s_kwd};
2458     $self->{state} = $self->{prev_state};
2459     ## Reconsume.
2460     redo A;
2461     }
2462     }
2463     } elsif ($self->{state} == HEXREF_HEX_STATE) {
2464     if (0x0030 <= $self->{nc} and $self->{nc} <= 0x0039) {
2465     # 0..9
2466     !!!cp (1002);
2467     $self->{s_kwd} *= 0x10;
2468     $self->{s_kwd} += $self->{nc} - 0x0030;
2469     ## Stay in the state.
2470     !!!next-input-character;
2471     redo A;
2472     } elsif (0x0061 <= $self->{nc} and
2473     $self->{nc} <= 0x0066) { # a..f
2474     !!!cp (1003);
2475     $self->{s_kwd} *= 0x10;
2476     $self->{s_kwd} += $self->{nc} - 0x0060 + 9;
2477     ## Stay in the state.
2478     !!!next-input-character;
2479     redo A;
2480     } elsif (0x0041 <= $self->{nc} and
2481     $self->{nc} <= 0x0046) { # A..F
2482     !!!cp (1004);
2483     $self->{s_kwd} *= 0x10;
2484     $self->{s_kwd} += $self->{nc} - 0x0040 + 9;
2485     ## Stay in the state.
2486     !!!next-input-character;
2487     redo A;
2488     } elsif ($self->{nc} == 0x003B) { # ;
2489     !!!cp (1006);
2490     !!!next-input-character;
2491     #
2492     } else {
2493     !!!cp (1007);
2494     !!!parse-error (type => 'no refc',
2495     line => $self->{line},
2496     column => $self->{column});
2497     ## Reconsume.
2498     #
2499     }
2500    
2501     my $code = $self->{s_kwd};
2502     my $l = $self->{line_prev};
2503     my $c = $self->{column_prev};
2504     if ($charref_map->{$code}) {
2505     !!!cp (1008);
2506     !!!parse-error (type => 'invalid character reference',
2507     text => (sprintf 'U+%04X', $code),
2508     line => $l, column => $c);
2509     $code = $charref_map->{$code};
2510     } elsif ($code > 0x10FFFF) {
2511     !!!cp (1009);
2512     !!!parse-error (type => 'invalid character reference',
2513     text => (sprintf 'U-%08X', $code),
2514     line => $l, column => $c);
2515     $code = 0xFFFD;
2516     }
2517    
2518     if ($self->{prev_state} == DATA_STATE) {
2519     !!!cp (988);
2520     $self->{state} = $self->{prev_state};
2521     ## Reconsume.
2522     !!!emit ({type => CHARACTER_TOKEN, data => chr $code,
2523     line => $l, column => $c,
2524     });
2525     redo A;
2526     } else {
2527     !!!cp (987);
2528     $self->{ca}->{value} .= chr $code;
2529     $self->{ca}->{has_reference} = 1;
2530     $self->{state} = $self->{prev_state};
2531     ## Reconsume.
2532     redo A;
2533     }
2534     } elsif ($self->{state} == ENTITY_NAME_STATE) {
2535     if (length $self->{s_kwd} < 30 and
2536     ## NOTE: Some number greater than the maximum length of entity name
2537     ((0x0041 <= $self->{nc} and # a
2538     $self->{nc} <= 0x005A) or # x
2539     (0x0061 <= $self->{nc} and # a
2540     $self->{nc} <= 0x007A) or # z
2541     (0x0030 <= $self->{nc} and # 0
2542     $self->{nc} <= 0x0039) or # 9
2543     $self->{nc} == 0x003B)) { # ;
2544     our $EntityChar;
2545     $self->{s_kwd} .= chr $self->{nc};
2546     if (defined $EntityChar->{$self->{s_kwd}}) {
2547     if ($self->{nc} == 0x003B) { # ;
2548     !!!cp (1020);
2549     $self->{entity__value} = $EntityChar->{$self->{s_kwd}};
2550     $self->{entity__match} = 1;
2551     !!!next-input-character;
2552     #
2553     } else {
2554     !!!cp (1021);
2555     $self->{entity__value} = $EntityChar->{$self->{s_kwd}};
2556     $self->{entity__match} = -1;
2557     ## Stay in the state.
2558     !!!next-input-character;
2559     redo A;
2560     }
2561     } else {
2562     !!!cp (1022);
2563     $self->{entity__value} .= chr $self->{nc};
2564     $self->{entity__match} *= 2;
2565     ## Stay in the state.
2566     !!!next-input-character;
2567     redo A;
2568     }
2569     }
2570    
2571     my $data;
2572     my $has_ref;
2573     if ($self->{entity__match} > 0) {
2574     !!!cp (1023);
2575     $data = $self->{entity__value};
2576     $has_ref = 1;
2577     #
2578     } elsif ($self->{entity__match} < 0) {
2579     !!!parse-error (type => 'no refc');
2580     if ($self->{prev_state} != DATA_STATE and # in attribute
2581     $self->{entity__match} < -1) {
2582     !!!cp (1024);
2583     $data = '&' . $self->{s_kwd};
2584     #
2585     } else {
2586     !!!cp (1025);
2587     $data = $self->{entity__value};
2588     $has_ref = 1;
2589     #
2590     }
2591     } else {
2592     !!!cp (1026);
2593     !!!parse-error (type => 'bare ero',
2594     line => $self->{line_prev},
2595     column => $self->{column_prev} - length $self->{s_kwd});
2596     $data = '&' . $self->{s_kwd};
2597     #
2598     }
2599    
2600     ## NOTE: In these cases, when a character reference is found,
2601     ## it is consumed and a character token is returned, or, otherwise,
2602     ## nothing is consumed and returned, according to the spec algorithm.
2603     ## In this implementation, anything that has been examined by the
2604     ## tokenizer is appended to the parent element or the attribute value
2605     ## as string, either literal string when no character reference or
2606     ## entity-replaced string otherwise, in this stage, since any characters
2607     ## that would not be consumed are appended in the data state or in an
2608     ## appropriate attribute value state anyway.
2609    
2610     if ($self->{prev_state} == DATA_STATE) {
2611     !!!cp (986);
2612     $self->{state} = $self->{prev_state};
2613     ## Reconsume.
2614     !!!emit ({type => CHARACTER_TOKEN,
2615     data => $data,
2616     line => $self->{line_prev},
2617     column => $self->{column_prev} + 1 - length $self->{s_kwd},
2618     });
2619     redo A;
2620     } else {
2621     !!!cp (985);
2622     $self->{ca}->{value} .= $data;
2623     $self->{ca}->{has_reference} = 1 if $has_ref;
2624     $self->{state} = $self->{prev_state};
2625     ## Reconsume.
2626     redo A;
2627     }
2628     } else {
2629     die "$0: $self->{state}: Unknown state";
2630     }
2631     } # A
2632    
2633     die "$0: _get_next_token: unexpected case";
2634     } # _get_next_token
2635    
2636     1;
2637     ## $Date:$
admin@suikawiki.org
 ViewVC Help
Powered by ViewVC 1.1.24