/[suikacvs]/markup/html/whatpm/Whatpm/HTML/Tokenizer.pm
Suika

Contents of /markup/html/whatpm/Whatpm/HTML/Tokenizer.pm

Parent Directory Parent Directory | Revision Log Revision Log

Revision 1.3 - (show annotations) (download)
Tue Oct 14 05:34:05 2008 UTC (17 years, 5 months ago) by wakaba
Branch: MAIN
Changes since 1.2: +6 -4 lines 
++ whatpm/Whatpm/HTML/ChangeLog	14 Oct 2008 05:33:48 -0000
	* Tokenizer.pm.src: Introduced "in_xml" flag for CDATA section
	support in XML.

2008-10-14  Wakaba  <wakaba@suika.fam.cx>

++ whatpm/Whatpm/XML/ChangeLog	14 Oct 2008 05:34:00 -0000
	* Parser.pm.src: Set |in_xml| flag for tokenizer.

2008-10-14  Wakaba  <wakaba@suika.fam.cx>

	* Parser.pm.src: A bug on end tag handling fixed.

2008-10-14  Wakaba  <wakaba@suika.fam.cx>
1 package Whatpm::HTML::Tokenizer;
2 use strict;
3 our $VERSION=do{my @r=(q$Revision: 1.2 $=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
4
5 BEGIN {
6 require Exporter;
7 push our @ISA, 'Exporter';
8
9 our @EXPORT_OK = qw(
10 DOCTYPE_TOKEN
11 COMMENT_TOKEN
12 START_TAG_TOKEN
13 END_TAG_TOKEN
14 END_OF_FILE_TOKEN
15 CHARACTER_TOKEN
16 PI_TOKEN
17 ABORT_TOKEN
18 );
19
20 our %EXPORT_TAGS = (
21 token => [qw(
22 DOCTYPE_TOKEN
23 COMMENT_TOKEN
24 START_TAG_TOKEN
25 END_TAG_TOKEN
26 END_OF_FILE_TOKEN
27 CHARACTER_TOKEN
28 PI_TOKEN
29 ABORT_TOKEN
30 )],
31 );
32 }
33
34 ## Token types
35
36 sub DOCTYPE_TOKEN () { 1 }
37 sub COMMENT_TOKEN () { 2 }
38 sub START_TAG_TOKEN () { 3 }
39 sub END_TAG_TOKEN () { 4 }
40 sub END_OF_FILE_TOKEN () { 5 }
41 sub CHARACTER_TOKEN () { 6 }
42 sub PI_TOKEN () { 7 } # XML5
43 sub ABORT_TOKEN () { 8 } # Not a token actually
44
45 package Whatpm::HTML;
46
47 BEGIN { Whatpm::HTML::Tokenizer->import (':token') }
48
49 ## Content model flags
50
51 sub CM_ENTITY () { 0b001 } # & markup in data
52 sub CM_LIMITED_MARKUP () { 0b010 } # < markup in data (limited)
53 sub CM_FULL_MARKUP () { 0b100 } # < markup in data (any)
54
55 sub PLAINTEXT_CONTENT_MODEL () { 0 }
56 sub CDATA_CONTENT_MODEL () { CM_LIMITED_MARKUP }
57 sub RCDATA_CONTENT_MODEL () { CM_ENTITY | CM_LIMITED_MARKUP }
58 sub PCDATA_CONTENT_MODEL () { CM_ENTITY | CM_FULL_MARKUP }
59
60 ## Tokenizer states
61
62 sub DATA_STATE () { 0 }
63 #sub ENTITY_DATA_STATE () { 1 }
64 sub TAG_OPEN_STATE () { 2 }
65 sub CLOSE_TAG_OPEN_STATE () { 3 }
66 sub TAG_NAME_STATE () { 4 }
67 sub BEFORE_ATTRIBUTE_NAME_STATE () { 5 }
68 sub ATTRIBUTE_NAME_STATE () { 6 }
69 sub AFTER_ATTRIBUTE_NAME_STATE () { 7 }
70 sub BEFORE_ATTRIBUTE_VALUE_STATE () { 8 }
71 sub ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE () { 9 }
72 sub ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE () { 10 }
73 sub ATTRIBUTE_VALUE_UNQUOTED_STATE () { 11 }
74 #sub ENTITY_IN_ATTRIBUTE_VALUE_STATE () { 12 }
75 sub MARKUP_DECLARATION_OPEN_STATE () { 13 }
76 sub COMMENT_START_STATE () { 14 }
77 sub COMMENT_START_DASH_STATE () { 15 }
78 sub COMMENT_STATE () { 16 }
79 sub COMMENT_END_STATE () { 17 }
80 sub COMMENT_END_DASH_STATE () { 18 }
81 sub BOGUS_COMMENT_STATE () { 19 }
82 sub DOCTYPE_STATE () { 20 }
83 sub BEFORE_DOCTYPE_NAME_STATE () { 21 }
84 sub DOCTYPE_NAME_STATE () { 22 }
85 sub AFTER_DOCTYPE_NAME_STATE () { 23 }
86 sub BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 24 }
87 sub DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE () { 25 }
88 sub DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE () { 26 }
89 sub AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE () { 27 }
90 sub BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 28 }
91 sub DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE () { 29 }
92 sub DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE () { 30 }
93 sub AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE () { 31 }
94 sub BOGUS_DOCTYPE_STATE () { 32 }
95 sub AFTER_ATTRIBUTE_VALUE_QUOTED_STATE () { 33 }
96 sub SELF_CLOSING_START_TAG_STATE () { 34 }
97 sub CDATA_SECTION_STATE () { 35 }
98 sub MD_HYPHEN_STATE () { 36 } # "markup declaration open state" in the spec
99 sub MD_DOCTYPE_STATE () { 37 } # "markup declaration open state" in the spec
100 sub MD_CDATA_STATE () { 38 } # "markup declaration open state" in the spec
101 sub CDATA_RCDATA_CLOSE_TAG_STATE () { 39 } # "close tag open state" in the spec
102 sub CDATA_SECTION_MSE1_STATE () { 40 } # "CDATA section state" in the spec
103 sub CDATA_SECTION_MSE2_STATE () { 41 } # "CDATA section state" in the spec
104 sub PUBLIC_STATE () { 42 } # "after DOCTYPE name state" in the spec
105 sub SYSTEM_STATE () { 43 } # "after DOCTYPE name state" in the spec
106 ## NOTE: "Entity data state", "entity in attribute value state", and
107 ## "consume a character reference" algorithm are jointly implemented
108 ## using the following six states:
109 sub ENTITY_STATE () { 44 }
110 sub ENTITY_HASH_STATE () { 45 }
111 sub NCR_NUM_STATE () { 46 }
112 sub HEXREF_X_STATE () { 47 }
113 sub HEXREF_HEX_STATE () { 48 }
114 sub ENTITY_NAME_STATE () { 49 }
115 sub PCDATA_STATE () { 50 } # "data state" in the spec
116
117 ## Tree constructor state constants (see Whatpm::HTML for the full
118 ## list and descriptions)
119
120 sub IN_FOREIGN_CONTENT_IM () { 0b100000000000 }
121 sub FOREIGN_EL () { 0b1_00000000000 }
122
123 ## Character reference mappings
124
125 my $charref_map = {
126 0x0D => 0x000A,
127 0x80 => 0x20AC,
128 0x81 => 0xFFFD,
129 0x82 => 0x201A,
130 0x83 => 0x0192,
131 0x84 => 0x201E,
132 0x85 => 0x2026,
133 0x86 => 0x2020,
134 0x87 => 0x2021,
135 0x88 => 0x02C6,
136 0x89 => 0x2030,
137 0x8A => 0x0160,
138 0x8B => 0x2039,
139 0x8C => 0x0152,
140 0x8D => 0xFFFD,
141 0x8E => 0x017D,
142 0x8F => 0xFFFD,
143 0x90 => 0xFFFD,
144 0x91 => 0x2018,
145 0x92 => 0x2019,
146 0x93 => 0x201C,
147 0x94 => 0x201D,
148 0x95 => 0x2022,
149 0x96 => 0x2013,
150 0x97 => 0x2014,
151 0x98 => 0x02DC,
152 0x99 => 0x2122,
153 0x9A => 0x0161,
154 0x9B => 0x203A,
155 0x9C => 0x0153,
156 0x9D => 0xFFFD,
157 0x9E => 0x017E,
158 0x9F => 0x0178,
159 }; # $charref_map
160 $charref_map->{$_} = 0xFFFD
161 for 0x0000..0x0008, 0x000B, 0x000E..0x001F, 0x007F,
162 0xD800..0xDFFF, 0xFDD0..0xFDDF, ## ISSUE: 0xFDEF
163 0xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF,
164 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE,
165 0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
166 0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, 0xDFFFF, 0xEFFFE,
167 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF;
168
169 ## Implementations MUST act as if state machine in the spec
170
171 sub _initialize_tokenizer ($) {
172 my $self = shift;
173
174 ## NOTE: Fields set by |new| constructor:
175 #$self->{level}
176 #$self->{set_nc}
177 #$self->{parse_error}
178 #$self->{is_xml} (if XML)
179
180 $self->{state} = DATA_STATE; # MUST
181 #$self->{s_kwd}; # state keyword - initialized when used
182 #$self->{entity__value}; # initialized when used
183 #$self->{entity__match}; # initialized when used
184 $self->{content_model} = PCDATA_CONTENT_MODEL; # be
185 undef $self->{ct}; # current token
186 undef $self->{ca}; # current attribute
187 undef $self->{last_stag_name}; # last emitted start tag name
188 #$self->{prev_state}; # initialized when used
189 delete $self->{self_closing};
190 $self->{char_buffer} = '';
191 $self->{char_buffer_pos} = 0;
192 $self->{nc} = -1; # next input character
193 #$self->{next_nc}
194
195 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
196 $self->{line_prev} = $self->{line};
197 $self->{column_prev} = $self->{column};
198 $self->{column}++;
199 $self->{nc}
200 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
201 } else {
202 $self->{set_nc}->($self);
203 }
204
205 $self->{token} = [];
206 # $self->{escape}
207 } # _initialize_tokenizer
208
209 ## A token has:
210 ## ->{type} == DOCTYPE_TOKEN, START_TAG_TOKEN, END_TAG_TOKEN, COMMENT_TOKEN,
211 ## CHARACTER_TOKEN, or END_OF_FILE_TOKEN
212 ## ->{name} (DOCTYPE_TOKEN)
213 ## ->{tag_name} (START_TAG_TOKEN, END_TAG_TOKEN)
214 ## ->{pubid} (DOCTYPE_TOKEN)
215 ## ->{sysid} (DOCTYPE_TOKEN)
216 ## ->{quirks} == 1 or 0 (DOCTYPE_TOKEN): "force-quirks" flag
217 ## ->{attributes} isa HASH (START_TAG_TOKEN, END_TAG_TOKEN)
218 ## ->{name}
219 ## ->{value}
220 ## ->{has_reference} == 1 or 0
221 ## ->{data} (COMMENT_TOKEN, CHARACTER_TOKEN)
222 ## NOTE: The "self-closing flag" is hold as |$self->{self_closing}|.
223 ## |->{self_closing}| is used to save the value of |$self->{self_closing}|
224 ## while the token is pushed back to the stack.
225
226 ## Emitted token MUST immediately be handled by the tree construction state.
227
228 ## Before each step, UA MAY check to see if either one of the scripts in
229 ## "list of scripts that will execute as soon as possible" or the first
230 ## script in the "list of scripts that will execute asynchronously",
231 ## has completed loading. If one has, then it MUST be executed
232 ## and removed from the list.
233
234 ## TODO: Polytheistic slash SHOULD NOT be used. (Applied only to atheists.)
235 ## (This requirement was dropped from HTML5 spec, unfortunately.)
236
237 my $is_space = {
238 0x0009 => 1, # CHARACTER TABULATION (HT)
239 0x000A => 1, # LINE FEED (LF)
240 #0x000B => 0, # LINE TABULATION (VT)
241 0x000C => 1, # FORM FEED (FF)
242 #0x000D => 1, # CARRIAGE RETURN (CR)
243 0x0020 => 1, # SPACE (SP)
244 };
245
246 sub _get_next_token ($) {
247 my $self = shift;
248
249 if ($self->{self_closing}) {
250 $self->{parse_error}->(level => $self->{level}->{must}, type => 'nestc', token => $self->{ct});
251 ## NOTE: The |self_closing| flag is only set by start tag token.
252 ## In addition, when a start tag token is emitted, it is always set to
253 ## |ct|.
254 delete $self->{self_closing};
255 }
256
257 if (@{$self->{token}}) {
258 $self->{self_closing} = $self->{token}->[0]->{self_closing};
259 return shift @{$self->{token}};
260 }
261
262 A: {
263 if ($self->{state} == PCDATA_STATE) {
264 ## NOTE: Same as |DATA_STATE|, but only for |PCDATA| content model.
265
266 if ($self->{nc} == 0x0026) { # &
267
268 ## NOTE: In the spec, the tokenizer is switched to the
269 ## "entity data state". In this implementation, the tokenizer
270 ## is switched to the |ENTITY_STATE|, which is an implementation
271 ## of the "consume a character reference" algorithm.
272 $self->{entity_add} = -1;
273 $self->{prev_state} = DATA_STATE;
274 $self->{state} = ENTITY_STATE;
275
276 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
277 $self->{line_prev} = $self->{line};
278 $self->{column_prev} = $self->{column};
279 $self->{column}++;
280 $self->{nc}
281 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
282 } else {
283 $self->{set_nc}->($self);
284 }
285
286 redo A;
287 } elsif ($self->{nc} == 0x003C) { # <
288
289 $self->{state} = TAG_OPEN_STATE;
290
291 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
292 $self->{line_prev} = $self->{line};
293 $self->{column_prev} = $self->{column};
294 $self->{column}++;
295 $self->{nc}
296 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
297 } else {
298 $self->{set_nc}->($self);
299 }
300
301 redo A;
302 } elsif ($self->{nc} == -1) {
303
304 return ({type => END_OF_FILE_TOKEN,
305 line => $self->{line}, column => $self->{column}});
306 last A; ## TODO: ok?
307 } else {
308
309 #
310 }
311
312 # Anything else
313 my $token = {type => CHARACTER_TOKEN,
314 data => chr $self->{nc},
315 line => $self->{line}, column => $self->{column},
316 };
317 $self->{read_until}->($token->{data}, q[<&], length $token->{data});
318
319 ## Stay in the state.
320
321 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
322 $self->{line_prev} = $self->{line};
323 $self->{column_prev} = $self->{column};
324 $self->{column}++;
325 $self->{nc}
326 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
327 } else {
328 $self->{set_nc}->($self);
329 }
330
331 return ($token);
332 redo A;
333 } elsif ($self->{state} == DATA_STATE) {
334 $self->{s_kwd} = '' unless defined $self->{s_kwd};
335 if ($self->{nc} == 0x0026) { # &
336 $self->{s_kwd} = '';
337 if ($self->{content_model} & CM_ENTITY and # PCDATA | RCDATA
338 not $self->{escape}) {
339
340 ## NOTE: In the spec, the tokenizer is switched to the
341 ## "entity data state". In this implementation, the tokenizer
342 ## is switched to the |ENTITY_STATE|, which is an implementation
343 ## of the "consume a character reference" algorithm.
344 $self->{entity_add} = -1;
345 $self->{prev_state} = DATA_STATE;
346 $self->{state} = ENTITY_STATE;
347
348 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
349 $self->{line_prev} = $self->{line};
350 $self->{column_prev} = $self->{column};
351 $self->{column}++;
352 $self->{nc}
353 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
354 } else {
355 $self->{set_nc}->($self);
356 }
357
358 redo A;
359 } else {
360
361 #
362 }
363 } elsif ($self->{nc} == 0x002D) { # -
364 if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
365 $self->{s_kwd} .= '-';
366
367 if ($self->{s_kwd} eq '<!--') {
368
369 $self->{escape} = 1; # unless $self->{escape};
370 $self->{s_kwd} = '--';
371 #
372 } elsif ($self->{s_kwd} eq '---') {
373
374 $self->{s_kwd} = '--';
375 #
376 } else {
377
378 #
379 }
380 }
381
382 #
383 } elsif ($self->{nc} == 0x0021) { # !
384 if (length $self->{s_kwd}) {
385
386 $self->{s_kwd} .= '!';
387 #
388 } else {
389
390 #$self->{s_kwd} = '';
391 #
392 }
393 #
394 } elsif ($self->{nc} == 0x003C) { # <
395 if ($self->{content_model} & CM_FULL_MARKUP or # PCDATA
396 (($self->{content_model} & CM_LIMITED_MARKUP) and # CDATA | RCDATA
397 not $self->{escape})) {
398
399 $self->{state} = TAG_OPEN_STATE;
400
401 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
402 $self->{line_prev} = $self->{line};
403 $self->{column_prev} = $self->{column};
404 $self->{column}++;
405 $self->{nc}
406 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
407 } else {
408 $self->{set_nc}->($self);
409 }
410
411 redo A;
412 } else {
413
414 $self->{s_kwd} = '';
415 #
416 }
417 } elsif ($self->{nc} == 0x003E) { # >
418 if ($self->{escape} and
419 ($self->{content_model} & CM_LIMITED_MARKUP)) { # RCDATA | CDATA
420 if ($self->{s_kwd} eq '--') {
421
422 delete $self->{escape};
423 } else {
424
425 }
426 } else {
427
428 }
429
430 $self->{s_kwd} = '';
431 #
432 } elsif ($self->{nc} == -1) {
433
434 $self->{s_kwd} = '';
435 return ({type => END_OF_FILE_TOKEN,
436 line => $self->{line}, column => $self->{column}});
437 last A; ## TODO: ok?
438 } else {
439
440 $self->{s_kwd} = '';
441 #
442 }
443
444 # Anything else
445 my $token = {type => CHARACTER_TOKEN,
446 data => chr $self->{nc},
447 line => $self->{line}, column => $self->{column},
448 };
449 if ($self->{read_until}->($token->{data}, q[-!<>&],
450 length $token->{data})) {
451 $self->{s_kwd} = '';
452 }
453
454 ## Stay in the data state.
455 if ($self->{content_model} == PCDATA_CONTENT_MODEL) {
456
457 $self->{state} = PCDATA_STATE;
458 } else {
459
460 ## Stay in the state.
461 }
462
463 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
464 $self->{line_prev} = $self->{line};
465 $self->{column_prev} = $self->{column};
466 $self->{column}++;
467 $self->{nc}
468 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
469 } else {
470 $self->{set_nc}->($self);
471 }
472
473 return ($token);
474 redo A;
475 } elsif ($self->{state} == TAG_OPEN_STATE) {
476 if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
477 if ($self->{nc} == 0x002F) { # /
478
479
480 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
481 $self->{line_prev} = $self->{line};
482 $self->{column_prev} = $self->{column};
483 $self->{column}++;
484 $self->{nc}
485 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
486 } else {
487 $self->{set_nc}->($self);
488 }
489
490 $self->{state} = CLOSE_TAG_OPEN_STATE;
491 redo A;
492 } elsif ($self->{nc} == 0x0021) { # !
493
494 $self->{s_kwd} = '<' unless $self->{escape};
495 #
496 } else {
497
498 #
499 }
500
501 ## reconsume
502 $self->{state} = DATA_STATE;
503 return ({type => CHARACTER_TOKEN, data => '<',
504 line => $self->{line_prev},
505 column => $self->{column_prev},
506 });
507 redo A;
508 } elsif ($self->{content_model} & CM_FULL_MARKUP) { # PCDATA
509 if ($self->{nc} == 0x0021) { # !
510
511 $self->{state} = MARKUP_DECLARATION_OPEN_STATE;
512
513 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
514 $self->{line_prev} = $self->{line};
515 $self->{column_prev} = $self->{column};
516 $self->{column}++;
517 $self->{nc}
518 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
519 } else {
520 $self->{set_nc}->($self);
521 }
522
523 redo A;
524 } elsif ($self->{nc} == 0x002F) { # /
525
526 $self->{state} = CLOSE_TAG_OPEN_STATE;
527
528 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
529 $self->{line_prev} = $self->{line};
530 $self->{column_prev} = $self->{column};
531 $self->{column}++;
532 $self->{nc}
533 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
534 } else {
535 $self->{set_nc}->($self);
536 }
537
538 redo A;
539 } elsif (0x0041 <= $self->{nc} and
540 $self->{nc} <= 0x005A) { # A..Z
541
542 $self->{ct}
543 = {type => START_TAG_TOKEN,
544 tag_name => chr ($self->{nc} + 0x0020),
545 line => $self->{line_prev},
546 column => $self->{column_prev}};
547 $self->{state} = TAG_NAME_STATE;
548
549 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
550 $self->{line_prev} = $self->{line};
551 $self->{column_prev} = $self->{column};
552 $self->{column}++;
553 $self->{nc}
554 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
555 } else {
556 $self->{set_nc}->($self);
557 }
558
559 redo A;
560 } elsif (0x0061 <= $self->{nc} and
561 $self->{nc} <= 0x007A) { # a..z
562
563 $self->{ct} = {type => START_TAG_TOKEN,
564 tag_name => chr ($self->{nc}),
565 line => $self->{line_prev},
566 column => $self->{column_prev}};
567 $self->{state} = TAG_NAME_STATE;
568
569 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
570 $self->{line_prev} = $self->{line};
571 $self->{column_prev} = $self->{column};
572 $self->{column}++;
573 $self->{nc}
574 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
575 } else {
576 $self->{set_nc}->($self);
577 }
578
579 redo A;
580 } elsif ($self->{nc} == 0x003E) { # >
581
582 $self->{parse_error}->(level => $self->{level}->{must}, type => 'empty start tag',
583 line => $self->{line_prev},
584 column => $self->{column_prev});
585 $self->{state} = DATA_STATE;
586
587 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
588 $self->{line_prev} = $self->{line};
589 $self->{column_prev} = $self->{column};
590 $self->{column}++;
591 $self->{nc}
592 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
593 } else {
594 $self->{set_nc}->($self);
595 }
596
597
598 return ({type => CHARACTER_TOKEN, data => '<>',
599 line => $self->{line_prev},
600 column => $self->{column_prev},
601 });
602
603 redo A;
604 } elsif ($self->{nc} == 0x003F) { # ?
605
606 $self->{parse_error}->(level => $self->{level}->{must}, type => 'pio',
607 line => $self->{line_prev},
608 column => $self->{column_prev});
609 $self->{state} = BOGUS_COMMENT_STATE;
610 $self->{ct} = {type => COMMENT_TOKEN, data => '',
611 line => $self->{line_prev},
612 column => $self->{column_prev},
613 };
614 ## $self->{nc} is intentionally left as is
615 redo A;
616 } else {
617
618 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bare stago',
619 line => $self->{line_prev},
620 column => $self->{column_prev});
621 $self->{state} = DATA_STATE;
622 ## reconsume
623
624 return ({type => CHARACTER_TOKEN, data => '<',
625 line => $self->{line_prev},
626 column => $self->{column_prev},
627 });
628
629 redo A;
630 }
631 } else {
632 die "$0: $self->{content_model} in tag open";
633 }
634 } elsif ($self->{state} == CLOSE_TAG_OPEN_STATE) {
635 ## NOTE: The "close tag open state" in the spec is implemented as
636 ## |CLOSE_TAG_OPEN_STATE| and |CDATA_RCDATA_CLOSE_TAG_STATE|.
637
638 my ($l, $c) = ($self->{line_prev}, $self->{column_prev} - 1); # "<"of"</"
639 if ($self->{content_model} & CM_LIMITED_MARKUP) { # RCDATA | CDATA
640 if (defined $self->{last_stag_name}) {
641 $self->{state} = CDATA_RCDATA_CLOSE_TAG_STATE;
642 $self->{s_kwd} = '';
643 ## Reconsume.
644 redo A;
645 } else {
646 ## No start tag token has ever been emitted
647 ## NOTE: See <http://krijnhoetmer.nl/irc-logs/whatwg/20070626#l-564>.
648
649 $self->{state} = DATA_STATE;
650 ## Reconsume.
651 return ({type => CHARACTER_TOKEN, data => '</',
652 line => $l, column => $c,
653 });
654 redo A;
655 }
656 }
657
658 if (0x0041 <= $self->{nc} and
659 $self->{nc} <= 0x005A) { # A..Z
660
661 $self->{ct}
662 = {type => END_TAG_TOKEN,
663 tag_name => chr ($self->{nc} + 0x0020),
664 line => $l, column => $c};
665 $self->{state} = TAG_NAME_STATE;
666
667 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
668 $self->{line_prev} = $self->{line};
669 $self->{column_prev} = $self->{column};
670 $self->{column}++;
671 $self->{nc}
672 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
673 } else {
674 $self->{set_nc}->($self);
675 }
676
677 redo A;
678 } elsif (0x0061 <= $self->{nc} and
679 $self->{nc} <= 0x007A) { # a..z
680
681 $self->{ct} = {type => END_TAG_TOKEN,
682 tag_name => chr ($self->{nc}),
683 line => $l, column => $c};
684 $self->{state} = TAG_NAME_STATE;
685
686 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
687 $self->{line_prev} = $self->{line};
688 $self->{column_prev} = $self->{column};
689 $self->{column}++;
690 $self->{nc}
691 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
692 } else {
693 $self->{set_nc}->($self);
694 }
695
696 redo A;
697 } elsif ($self->{nc} == 0x003E) { # >
698
699 $self->{parse_error}->(level => $self->{level}->{must}, type => 'empty end tag',
700 line => $self->{line_prev}, ## "<" in "</>"
701 column => $self->{column_prev} - 1);
702 $self->{state} = DATA_STATE;
703
704 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
705 $self->{line_prev} = $self->{line};
706 $self->{column_prev} = $self->{column};
707 $self->{column}++;
708 $self->{nc}
709 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
710 } else {
711 $self->{set_nc}->($self);
712 }
713
714 redo A;
715 } elsif ($self->{nc} == -1) {
716
717 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bare etago');
718 $self->{state} = DATA_STATE;
719 # reconsume
720
721 return ({type => CHARACTER_TOKEN, data => '</',
722 line => $l, column => $c,
723 });
724
725 redo A;
726 } else {
727
728 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bogus end tag');
729 $self->{state} = BOGUS_COMMENT_STATE;
730 $self->{ct} = {type => COMMENT_TOKEN, data => '',
731 line => $self->{line_prev}, # "<" of "</"
732 column => $self->{column_prev} - 1,
733 };
734 ## NOTE: $self->{nc} is intentionally left as is.
735 ## Although the "anything else" case of the spec not explicitly
736 ## states that the next input character is to be reconsumed,
737 ## it will be included to the |data| of the comment token
738 ## generated from the bogus end tag, as defined in the
739 ## "bogus comment state" entry.
740 redo A;
741 }
742 } elsif ($self->{state} == CDATA_RCDATA_CLOSE_TAG_STATE) {
743 my $ch = substr $self->{last_stag_name}, length $self->{s_kwd}, 1;
744 if (length $ch) {
745 my $CH = $ch;
746 $ch =~ tr/a-z/A-Z/;
747 my $nch = chr $self->{nc};
748 if ($nch eq $ch or $nch eq $CH) {
749
750 ## Stay in the state.
751 $self->{s_kwd} .= $nch;
752
753 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
754 $self->{line_prev} = $self->{line};
755 $self->{column_prev} = $self->{column};
756 $self->{column}++;
757 $self->{nc}
758 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
759 } else {
760 $self->{set_nc}->($self);
761 }
762
763 redo A;
764 } else {
765
766 $self->{state} = DATA_STATE;
767 ## Reconsume.
768 return ({type => CHARACTER_TOKEN,
769 data => '</' . $self->{s_kwd},
770 line => $self->{line_prev},
771 column => $self->{column_prev} - 1 - length $self->{s_kwd},
772 });
773 redo A;
774 }
775 } else { # after "<{tag-name}"
776 unless ($is_space->{$self->{nc}} or
777 {
778 0x003E => 1, # >
779 0x002F => 1, # /
780 -1 => 1, # EOF
781 }->{$self->{nc}}) {
782
783 ## Reconsume.
784 $self->{state} = DATA_STATE;
785 return ({type => CHARACTER_TOKEN,
786 data => '</' . $self->{s_kwd},
787 line => $self->{line_prev},
788 column => $self->{column_prev} - 1 - length $self->{s_kwd},
789 });
790 redo A;
791 } else {
792
793 $self->{ct}
794 = {type => END_TAG_TOKEN,
795 tag_name => $self->{last_stag_name},
796 line => $self->{line_prev},
797 column => $self->{column_prev} - 1 - length $self->{s_kwd}};
798 $self->{state} = TAG_NAME_STATE;
799 ## Reconsume.
800 redo A;
801 }
802 }
803 } elsif ($self->{state} == TAG_NAME_STATE) {
804 if ($is_space->{$self->{nc}}) {
805
806 $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
807
808 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
809 $self->{line_prev} = $self->{line};
810 $self->{column_prev} = $self->{column};
811 $self->{column}++;
812 $self->{nc}
813 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
814 } else {
815 $self->{set_nc}->($self);
816 }
817
818 redo A;
819 } elsif ($self->{nc} == 0x003E) { # >
820 if ($self->{ct}->{type} == START_TAG_TOKEN) {
821
822 $self->{last_stag_name} = $self->{ct}->{tag_name};
823 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
824 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
825 #if ($self->{ct}->{attributes}) {
826 # ## NOTE: This should never be reached.
827 # !!! cp (36);
828 # !!! parse-error (type => 'end tag attribute');
829 #} else {
830
831 #}
832 } else {
833 die "$0: $self->{ct}->{type}: Unknown token type";
834 }
835 $self->{state} = DATA_STATE;
836
837 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
838 $self->{line_prev} = $self->{line};
839 $self->{column_prev} = $self->{column};
840 $self->{column}++;
841 $self->{nc}
842 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
843 } else {
844 $self->{set_nc}->($self);
845 }
846
847
848 return ($self->{ct}); # start tag or end tag
849
850 redo A;
851 } elsif (0x0041 <= $self->{nc} and
852 $self->{nc} <= 0x005A) { # A..Z
853
854 $self->{ct}->{tag_name} .= chr ($self->{nc} + 0x0020);
855 # start tag or end tag
856 ## Stay in this state
857
858 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
859 $self->{line_prev} = $self->{line};
860 $self->{column_prev} = $self->{column};
861 $self->{column}++;
862 $self->{nc}
863 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
864 } else {
865 $self->{set_nc}->($self);
866 }
867
868 redo A;
869 } elsif ($self->{nc} == -1) {
870 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
871 if ($self->{ct}->{type} == START_TAG_TOKEN) {
872
873 $self->{last_stag_name} = $self->{ct}->{tag_name};
874 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
875 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
876 #if ($self->{ct}->{attributes}) {
877 # ## NOTE: This state should never be reached.
878 # !!! cp (40);
879 # !!! parse-error (type => 'end tag attribute');
880 #} else {
881
882 #}
883 } else {
884 die "$0: $self->{ct}->{type}: Unknown token type";
885 }
886 $self->{state} = DATA_STATE;
887 # reconsume
888
889 return ($self->{ct}); # start tag or end tag
890
891 redo A;
892 } elsif ($self->{nc} == 0x002F) { # /
893
894 $self->{state} = SELF_CLOSING_START_TAG_STATE;
895
896 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
897 $self->{line_prev} = $self->{line};
898 $self->{column_prev} = $self->{column};
899 $self->{column}++;
900 $self->{nc}
901 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
902 } else {
903 $self->{set_nc}->($self);
904 }
905
906 redo A;
907 } else {
908
909 $self->{ct}->{tag_name} .= chr $self->{nc};
910 # start tag or end tag
911 ## Stay in the state
912
913 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
914 $self->{line_prev} = $self->{line};
915 $self->{column_prev} = $self->{column};
916 $self->{column}++;
917 $self->{nc}
918 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
919 } else {
920 $self->{set_nc}->($self);
921 }
922
923 redo A;
924 }
925 } elsif ($self->{state} == BEFORE_ATTRIBUTE_NAME_STATE) {
926 if ($is_space->{$self->{nc}}) {
927
928 ## Stay in the state
929
930 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
931 $self->{line_prev} = $self->{line};
932 $self->{column_prev} = $self->{column};
933 $self->{column}++;
934 $self->{nc}
935 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
936 } else {
937 $self->{set_nc}->($self);
938 }
939
940 redo A;
941 } elsif ($self->{nc} == 0x003E) { # >
942 if ($self->{ct}->{type} == START_TAG_TOKEN) {
943
944 $self->{last_stag_name} = $self->{ct}->{tag_name};
945 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
946 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
947 if ($self->{ct}->{attributes}) {
948
949 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
950 } else {
951
952 }
953 } else {
954 die "$0: $self->{ct}->{type}: Unknown token type";
955 }
956 $self->{state} = DATA_STATE;
957
958 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
959 $self->{line_prev} = $self->{line};
960 $self->{column_prev} = $self->{column};
961 $self->{column}++;
962 $self->{nc}
963 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
964 } else {
965 $self->{set_nc}->($self);
966 }
967
968
969 return ($self->{ct}); # start tag or end tag
970
971 redo A;
972 } elsif (0x0041 <= $self->{nc} and
973 $self->{nc} <= 0x005A) { # A..Z
974
975 $self->{ca}
976 = {name => chr ($self->{nc} + 0x0020),
977 value => '',
978 line => $self->{line}, column => $self->{column}};
979 $self->{state} = ATTRIBUTE_NAME_STATE;
980
981 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
982 $self->{line_prev} = $self->{line};
983 $self->{column_prev} = $self->{column};
984 $self->{column}++;
985 $self->{nc}
986 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
987 } else {
988 $self->{set_nc}->($self);
989 }
990
991 redo A;
992 } elsif ($self->{nc} == 0x002F) { # /
993
994 $self->{state} = SELF_CLOSING_START_TAG_STATE;
995
996 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
997 $self->{line_prev} = $self->{line};
998 $self->{column_prev} = $self->{column};
999 $self->{column}++;
1000 $self->{nc}
1001 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1002 } else {
1003 $self->{set_nc}->($self);
1004 }
1005
1006 redo A;
1007 } elsif ($self->{nc} == -1) {
1008 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
1009 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1010
1011 $self->{last_stag_name} = $self->{ct}->{tag_name};
1012 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1013 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1014 if ($self->{ct}->{attributes}) {
1015
1016 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1017 } else {
1018
1019 }
1020 } else {
1021 die "$0: $self->{ct}->{type}: Unknown token type";
1022 }
1023 $self->{state} = DATA_STATE;
1024 # reconsume
1025
1026 return ($self->{ct}); # start tag or end tag
1027
1028 redo A;
1029 } else {
1030 if ({
1031 0x0022 => 1, # "
1032 0x0027 => 1, # '
1033 0x003D => 1, # =
1034 }->{$self->{nc}}) {
1035
1036 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bad attribute name');
1037 } else {
1038
1039 }
1040 $self->{ca}
1041 = {name => chr ($self->{nc}),
1042 value => '',
1043 line => $self->{line}, column => $self->{column}};
1044 $self->{state} = ATTRIBUTE_NAME_STATE;
1045
1046 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1047 $self->{line_prev} = $self->{line};
1048 $self->{column_prev} = $self->{column};
1049 $self->{column}++;
1050 $self->{nc}
1051 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1052 } else {
1053 $self->{set_nc}->($self);
1054 }
1055
1056 redo A;
1057 }
1058 } elsif ($self->{state} == ATTRIBUTE_NAME_STATE) {
1059 my $before_leave = sub {
1060 if (exists $self->{ct}->{attributes} # start tag or end tag
1061 ->{$self->{ca}->{name}}) { # MUST
1062
1063 $self->{parse_error}->(level => $self->{level}->{must}, type => 'duplicate attribute', text => $self->{ca}->{name}, line => $self->{ca}->{line}, column => $self->{ca}->{column});
1064 ## Discard $self->{ca} # MUST
1065 } else {
1066
1067 $self->{ct}->{attributes}->{$self->{ca}->{name}}
1068 = $self->{ca};
1069 }
1070 }; # $before_leave
1071
1072 if ($is_space->{$self->{nc}}) {
1073
1074 $before_leave->();
1075 $self->{state} = AFTER_ATTRIBUTE_NAME_STATE;
1076
1077 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1078 $self->{line_prev} = $self->{line};
1079 $self->{column_prev} = $self->{column};
1080 $self->{column}++;
1081 $self->{nc}
1082 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1083 } else {
1084 $self->{set_nc}->($self);
1085 }
1086
1087 redo A;
1088 } elsif ($self->{nc} == 0x003D) { # =
1089
1090 $before_leave->();
1091 $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
1092
1093 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1094 $self->{line_prev} = $self->{line};
1095 $self->{column_prev} = $self->{column};
1096 $self->{column}++;
1097 $self->{nc}
1098 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1099 } else {
1100 $self->{set_nc}->($self);
1101 }
1102
1103 redo A;
1104 } elsif ($self->{nc} == 0x003E) { # >
1105 $before_leave->();
1106 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1107
1108 $self->{last_stag_name} = $self->{ct}->{tag_name};
1109 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1110
1111 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1112 if ($self->{ct}->{attributes}) {
1113 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1114 }
1115 } else {
1116 die "$0: $self->{ct}->{type}: Unknown token type";
1117 }
1118 $self->{state} = DATA_STATE;
1119
1120 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1121 $self->{line_prev} = $self->{line};
1122 $self->{column_prev} = $self->{column};
1123 $self->{column}++;
1124 $self->{nc}
1125 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1126 } else {
1127 $self->{set_nc}->($self);
1128 }
1129
1130
1131 return ($self->{ct}); # start tag or end tag
1132
1133 redo A;
1134 } elsif (0x0041 <= $self->{nc} and
1135 $self->{nc} <= 0x005A) { # A..Z
1136
1137 $self->{ca}->{name} .= chr ($self->{nc} + 0x0020);
1138 ## Stay in the state
1139
1140 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1141 $self->{line_prev} = $self->{line};
1142 $self->{column_prev} = $self->{column};
1143 $self->{column}++;
1144 $self->{nc}
1145 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1146 } else {
1147 $self->{set_nc}->($self);
1148 }
1149
1150 redo A;
1151 } elsif ($self->{nc} == 0x002F) { # /
1152
1153 $before_leave->();
1154 $self->{state} = SELF_CLOSING_START_TAG_STATE;
1155
1156 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1157 $self->{line_prev} = $self->{line};
1158 $self->{column_prev} = $self->{column};
1159 $self->{column}++;
1160 $self->{nc}
1161 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1162 } else {
1163 $self->{set_nc}->($self);
1164 }
1165
1166 redo A;
1167 } elsif ($self->{nc} == -1) {
1168 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
1169 $before_leave->();
1170 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1171
1172 $self->{last_stag_name} = $self->{ct}->{tag_name};
1173 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1174 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1175 if ($self->{ct}->{attributes}) {
1176
1177 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1178 } else {
1179 ## NOTE: This state should never be reached.
1180
1181 }
1182 } else {
1183 die "$0: $self->{ct}->{type}: Unknown token type";
1184 }
1185 $self->{state} = DATA_STATE;
1186 # reconsume
1187
1188 return ($self->{ct}); # start tag or end tag
1189
1190 redo A;
1191 } else {
1192 if ($self->{nc} == 0x0022 or # "
1193 $self->{nc} == 0x0027) { # '
1194
1195 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bad attribute name');
1196 } else {
1197
1198 }
1199 $self->{ca}->{name} .= chr ($self->{nc});
1200 ## Stay in the state
1201
1202 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1203 $self->{line_prev} = $self->{line};
1204 $self->{column_prev} = $self->{column};
1205 $self->{column}++;
1206 $self->{nc}
1207 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1208 } else {
1209 $self->{set_nc}->($self);
1210 }
1211
1212 redo A;
1213 }
1214 } elsif ($self->{state} == AFTER_ATTRIBUTE_NAME_STATE) {
1215 if ($is_space->{$self->{nc}}) {
1216
1217 ## Stay in the state
1218
1219 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1220 $self->{line_prev} = $self->{line};
1221 $self->{column_prev} = $self->{column};
1222 $self->{column}++;
1223 $self->{nc}
1224 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1225 } else {
1226 $self->{set_nc}->($self);
1227 }
1228
1229 redo A;
1230 } elsif ($self->{nc} == 0x003D) { # =
1231
1232 $self->{state} = BEFORE_ATTRIBUTE_VALUE_STATE;
1233
1234 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1235 $self->{line_prev} = $self->{line};
1236 $self->{column_prev} = $self->{column};
1237 $self->{column}++;
1238 $self->{nc}
1239 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1240 } else {
1241 $self->{set_nc}->($self);
1242 }
1243
1244 redo A;
1245 } elsif ($self->{nc} == 0x003E) { # >
1246 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1247
1248 $self->{last_stag_name} = $self->{ct}->{tag_name};
1249 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1250 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1251 if ($self->{ct}->{attributes}) {
1252
1253 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1254 } else {
1255 ## NOTE: This state should never be reached.
1256
1257 }
1258 } else {
1259 die "$0: $self->{ct}->{type}: Unknown token type";
1260 }
1261 $self->{state} = DATA_STATE;
1262
1263 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1264 $self->{line_prev} = $self->{line};
1265 $self->{column_prev} = $self->{column};
1266 $self->{column}++;
1267 $self->{nc}
1268 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1269 } else {
1270 $self->{set_nc}->($self);
1271 }
1272
1273
1274 return ($self->{ct}); # start tag or end tag
1275
1276 redo A;
1277 } elsif (0x0041 <= $self->{nc} and
1278 $self->{nc} <= 0x005A) { # A..Z
1279
1280 $self->{ca}
1281 = {name => chr ($self->{nc} + 0x0020),
1282 value => '',
1283 line => $self->{line}, column => $self->{column}};
1284 $self->{state} = ATTRIBUTE_NAME_STATE;
1285
1286 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1287 $self->{line_prev} = $self->{line};
1288 $self->{column_prev} = $self->{column};
1289 $self->{column}++;
1290 $self->{nc}
1291 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1292 } else {
1293 $self->{set_nc}->($self);
1294 }
1295
1296 redo A;
1297 } elsif ($self->{nc} == 0x002F) { # /
1298
1299 $self->{state} = SELF_CLOSING_START_TAG_STATE;
1300
1301 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1302 $self->{line_prev} = $self->{line};
1303 $self->{column_prev} = $self->{column};
1304 $self->{column}++;
1305 $self->{nc}
1306 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1307 } else {
1308 $self->{set_nc}->($self);
1309 }
1310
1311 redo A;
1312 } elsif ($self->{nc} == -1) {
1313 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
1314 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1315
1316 $self->{last_stag_name} = $self->{ct}->{tag_name};
1317 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1318 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1319 if ($self->{ct}->{attributes}) {
1320
1321 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1322 } else {
1323 ## NOTE: This state should never be reached.
1324
1325 }
1326 } else {
1327 die "$0: $self->{ct}->{type}: Unknown token type";
1328 }
1329 $self->{state} = DATA_STATE;
1330 # reconsume
1331
1332 return ($self->{ct}); # start tag or end tag
1333
1334 redo A;
1335 } else {
1336 if ($self->{nc} == 0x0022 or # "
1337 $self->{nc} == 0x0027) { # '
1338
1339 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bad attribute name');
1340 } else {
1341
1342 }
1343 $self->{ca}
1344 = {name => chr ($self->{nc}),
1345 value => '',
1346 line => $self->{line}, column => $self->{column}};
1347 $self->{state} = ATTRIBUTE_NAME_STATE;
1348
1349 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1350 $self->{line_prev} = $self->{line};
1351 $self->{column_prev} = $self->{column};
1352 $self->{column}++;
1353 $self->{nc}
1354 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1355 } else {
1356 $self->{set_nc}->($self);
1357 }
1358
1359 redo A;
1360 }
1361 } elsif ($self->{state} == BEFORE_ATTRIBUTE_VALUE_STATE) {
1362 if ($is_space->{$self->{nc}}) {
1363
1364 ## Stay in the state
1365
1366 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1367 $self->{line_prev} = $self->{line};
1368 $self->{column_prev} = $self->{column};
1369 $self->{column}++;
1370 $self->{nc}
1371 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1372 } else {
1373 $self->{set_nc}->($self);
1374 }
1375
1376 redo A;
1377 } elsif ($self->{nc} == 0x0022) { # "
1378
1379 $self->{state} = ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE;
1380
1381 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1382 $self->{line_prev} = $self->{line};
1383 $self->{column_prev} = $self->{column};
1384 $self->{column}++;
1385 $self->{nc}
1386 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1387 } else {
1388 $self->{set_nc}->($self);
1389 }
1390
1391 redo A;
1392 } elsif ($self->{nc} == 0x0026) { # &
1393
1394 $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
1395 ## reconsume
1396 redo A;
1397 } elsif ($self->{nc} == 0x0027) { # '
1398
1399 $self->{state} = ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE;
1400
1401 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1402 $self->{line_prev} = $self->{line};
1403 $self->{column_prev} = $self->{column};
1404 $self->{column}++;
1405 $self->{nc}
1406 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1407 } else {
1408 $self->{set_nc}->($self);
1409 }
1410
1411 redo A;
1412 } elsif ($self->{nc} == 0x003E) { # >
1413 $self->{parse_error}->(level => $self->{level}->{must}, type => 'empty unquoted attribute value');
1414 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1415
1416 $self->{last_stag_name} = $self->{ct}->{tag_name};
1417 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1418 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1419 if ($self->{ct}->{attributes}) {
1420
1421 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1422 } else {
1423 ## NOTE: This state should never be reached.
1424
1425 }
1426 } else {
1427 die "$0: $self->{ct}->{type}: Unknown token type";
1428 }
1429 $self->{state} = DATA_STATE;
1430
1431 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1432 $self->{line_prev} = $self->{line};
1433 $self->{column_prev} = $self->{column};
1434 $self->{column}++;
1435 $self->{nc}
1436 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1437 } else {
1438 $self->{set_nc}->($self);
1439 }
1440
1441
1442 return ($self->{ct}); # start tag or end tag
1443
1444 redo A;
1445 } elsif ($self->{nc} == -1) {
1446 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
1447 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1448
1449 $self->{last_stag_name} = $self->{ct}->{tag_name};
1450 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1451 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1452 if ($self->{ct}->{attributes}) {
1453
1454 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1455 } else {
1456 ## NOTE: This state should never be reached.
1457
1458 }
1459 } else {
1460 die "$0: $self->{ct}->{type}: Unknown token type";
1461 }
1462 $self->{state} = DATA_STATE;
1463 ## reconsume
1464
1465 return ($self->{ct}); # start tag or end tag
1466
1467 redo A;
1468 } else {
1469 if ($self->{nc} == 0x003D) { # =
1470
1471 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bad attribute value');
1472 } else {
1473
1474 }
1475 $self->{ca}->{value} .= chr ($self->{nc});
1476 $self->{state} = ATTRIBUTE_VALUE_UNQUOTED_STATE;
1477
1478 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1479 $self->{line_prev} = $self->{line};
1480 $self->{column_prev} = $self->{column};
1481 $self->{column}++;
1482 $self->{nc}
1483 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1484 } else {
1485 $self->{set_nc}->($self);
1486 }
1487
1488 redo A;
1489 }
1490 } elsif ($self->{state} == ATTRIBUTE_VALUE_DOUBLE_QUOTED_STATE) {
1491 if ($self->{nc} == 0x0022) { # "
1492
1493 $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;
1494
1495 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1496 $self->{line_prev} = $self->{line};
1497 $self->{column_prev} = $self->{column};
1498 $self->{column}++;
1499 $self->{nc}
1500 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1501 } else {
1502 $self->{set_nc}->($self);
1503 }
1504
1505 redo A;
1506 } elsif ($self->{nc} == 0x0026) { # &
1507
1508 ## NOTE: In the spec, the tokenizer is switched to the
1509 ## "entity in attribute value state". In this implementation, the
1510 ## tokenizer is switched to the |ENTITY_STATE|, which is an
1511 ## implementation of the "consume a character reference" algorithm.
1512 $self->{prev_state} = $self->{state};
1513 $self->{entity_add} = 0x0022; # "
1514 $self->{state} = ENTITY_STATE;
1515
1516 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1517 $self->{line_prev} = $self->{line};
1518 $self->{column_prev} = $self->{column};
1519 $self->{column}++;
1520 $self->{nc}
1521 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1522 } else {
1523 $self->{set_nc}->($self);
1524 }
1525
1526 redo A;
1527 } elsif ($self->{nc} == -1) {
1528 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed attribute value');
1529 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1530
1531 $self->{last_stag_name} = $self->{ct}->{tag_name};
1532 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1533 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1534 if ($self->{ct}->{attributes}) {
1535
1536 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1537 } else {
1538 ## NOTE: This state should never be reached.
1539
1540 }
1541 } else {
1542 die "$0: $self->{ct}->{type}: Unknown token type";
1543 }
1544 $self->{state} = DATA_STATE;
1545 ## reconsume
1546
1547 return ($self->{ct}); # start tag or end tag
1548
1549 redo A;
1550 } else {
1551
1552 $self->{ca}->{value} .= chr ($self->{nc});
1553 $self->{read_until}->($self->{ca}->{value},
1554 q["&],
1555 length $self->{ca}->{value});
1556
1557 ## Stay in the state
1558
1559 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1560 $self->{line_prev} = $self->{line};
1561 $self->{column_prev} = $self->{column};
1562 $self->{column}++;
1563 $self->{nc}
1564 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1565 } else {
1566 $self->{set_nc}->($self);
1567 }
1568
1569 redo A;
1570 }
1571 } elsif ($self->{state} == ATTRIBUTE_VALUE_SINGLE_QUOTED_STATE) {
1572 if ($self->{nc} == 0x0027) { # '
1573
1574 $self->{state} = AFTER_ATTRIBUTE_VALUE_QUOTED_STATE;
1575
1576 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1577 $self->{line_prev} = $self->{line};
1578 $self->{column_prev} = $self->{column};
1579 $self->{column}++;
1580 $self->{nc}
1581 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1582 } else {
1583 $self->{set_nc}->($self);
1584 }
1585
1586 redo A;
1587 } elsif ($self->{nc} == 0x0026) { # &
1588
1589 ## NOTE: In the spec, the tokenizer is switched to the
1590 ## "entity in attribute value state". In this implementation, the
1591 ## tokenizer is switched to the |ENTITY_STATE|, which is an
1592 ## implementation of the "consume a character reference" algorithm.
1593 $self->{entity_add} = 0x0027; # '
1594 $self->{prev_state} = $self->{state};
1595 $self->{state} = ENTITY_STATE;
1596
1597 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1598 $self->{line_prev} = $self->{line};
1599 $self->{column_prev} = $self->{column};
1600 $self->{column}++;
1601 $self->{nc}
1602 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1603 } else {
1604 $self->{set_nc}->($self);
1605 }
1606
1607 redo A;
1608 } elsif ($self->{nc} == -1) {
1609 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed attribute value');
1610 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1611
1612 $self->{last_stag_name} = $self->{ct}->{tag_name};
1613 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1614 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1615 if ($self->{ct}->{attributes}) {
1616
1617 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1618 } else {
1619 ## NOTE: This state should never be reached.
1620
1621 }
1622 } else {
1623 die "$0: $self->{ct}->{type}: Unknown token type";
1624 }
1625 $self->{state} = DATA_STATE;
1626 ## reconsume
1627
1628 return ($self->{ct}); # start tag or end tag
1629
1630 redo A;
1631 } else {
1632
1633 $self->{ca}->{value} .= chr ($self->{nc});
1634 $self->{read_until}->($self->{ca}->{value},
1635 q['&],
1636 length $self->{ca}->{value});
1637
1638 ## Stay in the state
1639
1640 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1641 $self->{line_prev} = $self->{line};
1642 $self->{column_prev} = $self->{column};
1643 $self->{column}++;
1644 $self->{nc}
1645 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1646 } else {
1647 $self->{set_nc}->($self);
1648 }
1649
1650 redo A;
1651 }
1652 } elsif ($self->{state} == ATTRIBUTE_VALUE_UNQUOTED_STATE) {
1653 if ($is_space->{$self->{nc}}) {
1654
1655 $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1656
1657 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1658 $self->{line_prev} = $self->{line};
1659 $self->{column_prev} = $self->{column};
1660 $self->{column}++;
1661 $self->{nc}
1662 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1663 } else {
1664 $self->{set_nc}->($self);
1665 }
1666
1667 redo A;
1668 } elsif ($self->{nc} == 0x0026) { # &
1669
1670 ## NOTE: In the spec, the tokenizer is switched to the
1671 ## "entity in attribute value state". In this implementation, the
1672 ## tokenizer is switched to the |ENTITY_STATE|, which is an
1673 ## implementation of the "consume a character reference" algorithm.
1674 $self->{entity_add} = -1;
1675 $self->{prev_state} = $self->{state};
1676 $self->{state} = ENTITY_STATE;
1677
1678 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1679 $self->{line_prev} = $self->{line};
1680 $self->{column_prev} = $self->{column};
1681 $self->{column}++;
1682 $self->{nc}
1683 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1684 } else {
1685 $self->{set_nc}->($self);
1686 }
1687
1688 redo A;
1689 } elsif ($self->{nc} == 0x003E) { # >
1690 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1691
1692 $self->{last_stag_name} = $self->{ct}->{tag_name};
1693 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1694 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1695 if ($self->{ct}->{attributes}) {
1696
1697 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1698 } else {
1699 ## NOTE: This state should never be reached.
1700
1701 }
1702 } else {
1703 die "$0: $self->{ct}->{type}: Unknown token type";
1704 }
1705 $self->{state} = DATA_STATE;
1706
1707 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1708 $self->{line_prev} = $self->{line};
1709 $self->{column_prev} = $self->{column};
1710 $self->{column}++;
1711 $self->{nc}
1712 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1713 } else {
1714 $self->{set_nc}->($self);
1715 }
1716
1717
1718 return ($self->{ct}); # start tag or end tag
1719
1720 redo A;
1721 } elsif ($self->{nc} == -1) {
1722 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
1723 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1724
1725 $self->{last_stag_name} = $self->{ct}->{tag_name};
1726 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1727 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1728 if ($self->{ct}->{attributes}) {
1729
1730 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1731 } else {
1732 ## NOTE: This state should never be reached.
1733
1734 }
1735 } else {
1736 die "$0: $self->{ct}->{type}: Unknown token type";
1737 }
1738 $self->{state} = DATA_STATE;
1739 ## reconsume
1740
1741 return ($self->{ct}); # start tag or end tag
1742
1743 redo A;
1744 } else {
1745 if ({
1746 0x0022 => 1, # "
1747 0x0027 => 1, # '
1748 0x003D => 1, # =
1749 }->{$self->{nc}}) {
1750
1751 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bad attribute value');
1752 } else {
1753
1754 }
1755 $self->{ca}->{value} .= chr ($self->{nc});
1756 $self->{read_until}->($self->{ca}->{value},
1757 q["'=& >],
1758 length $self->{ca}->{value});
1759
1760 ## Stay in the state
1761
1762 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1763 $self->{line_prev} = $self->{line};
1764 $self->{column_prev} = $self->{column};
1765 $self->{column}++;
1766 $self->{nc}
1767 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1768 } else {
1769 $self->{set_nc}->($self);
1770 }
1771
1772 redo A;
1773 }
1774 } elsif ($self->{state} == AFTER_ATTRIBUTE_VALUE_QUOTED_STATE) {
1775 if ($is_space->{$self->{nc}}) {
1776
1777 $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1778
1779 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1780 $self->{line_prev} = $self->{line};
1781 $self->{column_prev} = $self->{column};
1782 $self->{column}++;
1783 $self->{nc}
1784 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1785 } else {
1786 $self->{set_nc}->($self);
1787 }
1788
1789 redo A;
1790 } elsif ($self->{nc} == 0x003E) { # >
1791 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1792
1793 $self->{last_stag_name} = $self->{ct}->{tag_name};
1794 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1795 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1796 if ($self->{ct}->{attributes}) {
1797
1798 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1799 } else {
1800 ## NOTE: This state should never be reached.
1801
1802 }
1803 } else {
1804 die "$0: $self->{ct}->{type}: Unknown token type";
1805 }
1806 $self->{state} = DATA_STATE;
1807
1808 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1809 $self->{line_prev} = $self->{line};
1810 $self->{column_prev} = $self->{column};
1811 $self->{column}++;
1812 $self->{nc}
1813 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1814 } else {
1815 $self->{set_nc}->($self);
1816 }
1817
1818
1819 return ($self->{ct}); # start tag or end tag
1820
1821 redo A;
1822 } elsif ($self->{nc} == 0x002F) { # /
1823
1824 $self->{state} = SELF_CLOSING_START_TAG_STATE;
1825
1826 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1827 $self->{line_prev} = $self->{line};
1828 $self->{column_prev} = $self->{column};
1829 $self->{column}++;
1830 $self->{nc}
1831 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1832 } else {
1833 $self->{set_nc}->($self);
1834 }
1835
1836 redo A;
1837 } elsif ($self->{nc} == -1) {
1838 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
1839 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1840
1841 $self->{last_stag_name} = $self->{ct}->{tag_name};
1842 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1843 if ($self->{ct}->{attributes}) {
1844
1845 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1846 } else {
1847 ## NOTE: This state should never be reached.
1848
1849 }
1850 } else {
1851 die "$0: $self->{ct}->{type}: Unknown token type";
1852 }
1853 $self->{state} = DATA_STATE;
1854 ## Reconsume.
1855 return ($self->{ct}); # start tag or end tag
1856 redo A;
1857 } else {
1858
1859 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no space between attributes');
1860 $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1861 ## reconsume
1862 redo A;
1863 }
1864 } elsif ($self->{state} == SELF_CLOSING_START_TAG_STATE) {
1865 if ($self->{nc} == 0x003E) { # >
1866 if ($self->{ct}->{type} == END_TAG_TOKEN) {
1867
1868 $self->{parse_error}->(level => $self->{level}->{must}, type => 'nestc', token => $self->{ct});
1869 ## TODO: Different type than slash in start tag
1870 $self->{content_model} = PCDATA_CONTENT_MODEL; # MUST
1871 if ($self->{ct}->{attributes}) {
1872
1873 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1874 } else {
1875
1876 }
1877 ## TODO: Test |<title></title/>|
1878 } else {
1879
1880 $self->{self_closing} = 1;
1881 }
1882
1883 $self->{state} = DATA_STATE;
1884
1885 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1886 $self->{line_prev} = $self->{line};
1887 $self->{column_prev} = $self->{column};
1888 $self->{column}++;
1889 $self->{nc}
1890 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1891 } else {
1892 $self->{set_nc}->($self);
1893 }
1894
1895
1896 return ($self->{ct}); # start tag or end tag
1897
1898 redo A;
1899 } elsif ($self->{nc} == -1) {
1900 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed tag');
1901 if ($self->{ct}->{type} == START_TAG_TOKEN) {
1902
1903 $self->{last_stag_name} = $self->{ct}->{tag_name};
1904 } elsif ($self->{ct}->{type} == END_TAG_TOKEN) {
1905 if ($self->{ct}->{attributes}) {
1906
1907 $self->{parse_error}->(level => $self->{level}->{must}, type => 'end tag attribute');
1908 } else {
1909 ## NOTE: This state should never be reached.
1910
1911 }
1912 } else {
1913 die "$0: $self->{ct}->{type}: Unknown token type";
1914 }
1915 $self->{state} = DATA_STATE;
1916 ## Reconsume.
1917 return ($self->{ct}); # start tag or end tag
1918 redo A;
1919 } else {
1920
1921 $self->{parse_error}->(level => $self->{level}->{must}, type => 'nestc');
1922 ## TODO: This error type is wrong.
1923 $self->{state} = BEFORE_ATTRIBUTE_NAME_STATE;
1924 ## Reconsume.
1925 redo A;
1926 }
1927 } elsif ($self->{state} == BOGUS_COMMENT_STATE) {
1928 ## (only happen if PCDATA state)
1929
1930 ## NOTE: Unlike spec's "bogus comment state", this implementation
1931 ## consumes characters one-by-one basis.
1932
1933 if ($self->{nc} == 0x003E) { # >
1934
1935 $self->{state} = DATA_STATE;
1936
1937 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1938 $self->{line_prev} = $self->{line};
1939 $self->{column_prev} = $self->{column};
1940 $self->{column}++;
1941 $self->{nc}
1942 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1943 } else {
1944 $self->{set_nc}->($self);
1945 }
1946
1947
1948 return ($self->{ct}); # comment
1949 redo A;
1950 } elsif ($self->{nc} == -1) {
1951
1952 $self->{state} = DATA_STATE;
1953 ## reconsume
1954
1955 return ($self->{ct}); # comment
1956 redo A;
1957 } else {
1958
1959 $self->{ct}->{data} .= chr ($self->{nc}); # comment
1960 $self->{read_until}->($self->{ct}->{data},
1961 q[>],
1962 length $self->{ct}->{data});
1963
1964 ## Stay in the state.
1965
1966 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1967 $self->{line_prev} = $self->{line};
1968 $self->{column_prev} = $self->{column};
1969 $self->{column}++;
1970 $self->{nc}
1971 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1972 } else {
1973 $self->{set_nc}->($self);
1974 }
1975
1976 redo A;
1977 }
1978 } elsif ($self->{state} == MARKUP_DECLARATION_OPEN_STATE) {
1979 ## (only happen if PCDATA state)
1980
1981 if ($self->{nc} == 0x002D) { # -
1982
1983 $self->{state} = MD_HYPHEN_STATE;
1984
1985 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
1986 $self->{line_prev} = $self->{line};
1987 $self->{column_prev} = $self->{column};
1988 $self->{column}++;
1989 $self->{nc}
1990 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
1991 } else {
1992 $self->{set_nc}->($self);
1993 }
1994
1995 redo A;
1996 } elsif ($self->{nc} == 0x0044 or # D
1997 $self->{nc} == 0x0064) { # d
1998 ## ASCII case-insensitive.
1999
2000 $self->{state} = MD_DOCTYPE_STATE;
2001 $self->{s_kwd} = chr $self->{nc};
2002
2003 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2004 $self->{line_prev} = $self->{line};
2005 $self->{column_prev} = $self->{column};
2006 $self->{column}++;
2007 $self->{nc}
2008 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2009 } else {
2010 $self->{set_nc}->($self);
2011 }
2012
2013 redo A;
2014 } elsif ((($self->{insertion_mode} & IN_FOREIGN_CONTENT_IM and
2015 $self->{open_elements}->[-1]->[1] & FOREIGN_EL) or
2016 $self->{is_xml}) and
2017 $self->{nc} == 0x005B) { # [
2018
2019 $self->{state} = MD_CDATA_STATE;
2020 $self->{s_kwd} = '[';
2021
2022 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2023 $self->{line_prev} = $self->{line};
2024 $self->{column_prev} = $self->{column};
2025 $self->{column}++;
2026 $self->{nc}
2027 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2028 } else {
2029 $self->{set_nc}->($self);
2030 }
2031
2032 redo A;
2033 } else {
2034
2035 }
2036
2037 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bogus comment',
2038 line => $self->{line_prev},
2039 column => $self->{column_prev} - 1);
2040 ## Reconsume.
2041 $self->{state} = BOGUS_COMMENT_STATE;
2042 $self->{ct} = {type => COMMENT_TOKEN, data => '',
2043 line => $self->{line_prev},
2044 column => $self->{column_prev} - 1,
2045 };
2046 redo A;
2047 } elsif ($self->{state} == MD_HYPHEN_STATE) {
2048 if ($self->{nc} == 0x002D) { # -
2049
2050 $self->{ct} = {type => COMMENT_TOKEN, data => '',
2051 line => $self->{line_prev},
2052 column => $self->{column_prev} - 2,
2053 };
2054 $self->{state} = COMMENT_START_STATE;
2055
2056 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2057 $self->{line_prev} = $self->{line};
2058 $self->{column_prev} = $self->{column};
2059 $self->{column}++;
2060 $self->{nc}
2061 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2062 } else {
2063 $self->{set_nc}->($self);
2064 }
2065
2066 redo A;
2067 } else {
2068
2069 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bogus comment',
2070 line => $self->{line_prev},
2071 column => $self->{column_prev} - 2);
2072 $self->{state} = BOGUS_COMMENT_STATE;
2073 ## Reconsume.
2074 $self->{ct} = {type => COMMENT_TOKEN,
2075 data => '-',
2076 line => $self->{line_prev},
2077 column => $self->{column_prev} - 2,
2078 };
2079 redo A;
2080 }
2081 } elsif ($self->{state} == MD_DOCTYPE_STATE) {
2082 ## ASCII case-insensitive.
2083 if ($self->{nc} == [
2084 undef,
2085 0x004F, # O
2086 0x0043, # C
2087 0x0054, # T
2088 0x0059, # Y
2089 0x0050, # P
2090 ]->[length $self->{s_kwd}] or
2091 $self->{nc} == [
2092 undef,
2093 0x006F, # o
2094 0x0063, # c
2095 0x0074, # t
2096 0x0079, # y
2097 0x0070, # p
2098 ]->[length $self->{s_kwd}]) {
2099
2100 ## Stay in the state.
2101 $self->{s_kwd} .= chr $self->{nc};
2102
2103 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2104 $self->{line_prev} = $self->{line};
2105 $self->{column_prev} = $self->{column};
2106 $self->{column}++;
2107 $self->{nc}
2108 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2109 } else {
2110 $self->{set_nc}->($self);
2111 }
2112
2113 redo A;
2114 } elsif ((length $self->{s_kwd}) == 6 and
2115 ($self->{nc} == 0x0045 or # E
2116 $self->{nc} == 0x0065)) { # e
2117
2118 $self->{state} = DOCTYPE_STATE;
2119 $self->{ct} = {type => DOCTYPE_TOKEN,
2120 quirks => 1,
2121 line => $self->{line_prev},
2122 column => $self->{column_prev} - 7,
2123 };
2124
2125 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2126 $self->{line_prev} = $self->{line};
2127 $self->{column_prev} = $self->{column};
2128 $self->{column}++;
2129 $self->{nc}
2130 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2131 } else {
2132 $self->{set_nc}->($self);
2133 }
2134
2135 redo A;
2136 } else {
2137
2138 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bogus comment',
2139 line => $self->{line_prev},
2140 column => $self->{column_prev} - 1 - length $self->{s_kwd});
2141 $self->{state} = BOGUS_COMMENT_STATE;
2142 ## Reconsume.
2143 $self->{ct} = {type => COMMENT_TOKEN,
2144 data => $self->{s_kwd},
2145 line => $self->{line_prev},
2146 column => $self->{column_prev} - 1 - length $self->{s_kwd},
2147 };
2148 redo A;
2149 }
2150 } elsif ($self->{state} == MD_CDATA_STATE) {
2151 if ($self->{nc} == {
2152 '[' => 0x0043, # C
2153 '[C' => 0x0044, # D
2154 '[CD' => 0x0041, # A
2155 '[CDA' => 0x0054, # T
2156 '[CDAT' => 0x0041, # A
2157 }->{$self->{s_kwd}}) {
2158
2159 ## Stay in the state.
2160 $self->{s_kwd} .= chr $self->{nc};
2161
2162 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2163 $self->{line_prev} = $self->{line};
2164 $self->{column_prev} = $self->{column};
2165 $self->{column}++;
2166 $self->{nc}
2167 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2168 } else {
2169 $self->{set_nc}->($self);
2170 }
2171
2172 redo A;
2173 } elsif ($self->{s_kwd} eq '[CDATA' and
2174 $self->{nc} == 0x005B) { # [
2175
2176 $self->{ct} = {type => CHARACTER_TOKEN,
2177 data => '',
2178 line => $self->{line_prev},
2179 column => $self->{column_prev} - 7};
2180 $self->{state} = CDATA_SECTION_STATE;
2181
2182 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2183 $self->{line_prev} = $self->{line};
2184 $self->{column_prev} = $self->{column};
2185 $self->{column}++;
2186 $self->{nc}
2187 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2188 } else {
2189 $self->{set_nc}->($self);
2190 }
2191
2192 redo A;
2193 } else {
2194
2195 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bogus comment',
2196 line => $self->{line_prev},
2197 column => $self->{column_prev} - 1 - length $self->{s_kwd});
2198 $self->{state} = BOGUS_COMMENT_STATE;
2199 ## Reconsume.
2200 $self->{ct} = {type => COMMENT_TOKEN,
2201 data => $self->{s_kwd},
2202 line => $self->{line_prev},
2203 column => $self->{column_prev} - 1 - length $self->{s_kwd},
2204 };
2205 redo A;
2206 }
2207 } elsif ($self->{state} == COMMENT_START_STATE) {
2208 if ($self->{nc} == 0x002D) { # -
2209
2210 $self->{state} = COMMENT_START_DASH_STATE;
2211
2212 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2213 $self->{line_prev} = $self->{line};
2214 $self->{column_prev} = $self->{column};
2215 $self->{column}++;
2216 $self->{nc}
2217 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2218 } else {
2219 $self->{set_nc}->($self);
2220 }
2221
2222 redo A;
2223 } elsif ($self->{nc} == 0x003E) { # >
2224
2225 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bogus comment');
2226 $self->{state} = DATA_STATE;
2227
2228 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2229 $self->{line_prev} = $self->{line};
2230 $self->{column_prev} = $self->{column};
2231 $self->{column}++;
2232 $self->{nc}
2233 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2234 } else {
2235 $self->{set_nc}->($self);
2236 }
2237
2238
2239 return ($self->{ct}); # comment
2240
2241 redo A;
2242 } elsif ($self->{nc} == -1) {
2243
2244 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed comment');
2245 $self->{state} = DATA_STATE;
2246 ## reconsume
2247
2248 return ($self->{ct}); # comment
2249
2250 redo A;
2251 } else {
2252
2253 $self->{ct}->{data} # comment
2254 .= chr ($self->{nc});
2255 $self->{state} = COMMENT_STATE;
2256
2257 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2258 $self->{line_prev} = $self->{line};
2259 $self->{column_prev} = $self->{column};
2260 $self->{column}++;
2261 $self->{nc}
2262 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2263 } else {
2264 $self->{set_nc}->($self);
2265 }
2266
2267 redo A;
2268 }
2269 } elsif ($self->{state} == COMMENT_START_DASH_STATE) {
2270 if ($self->{nc} == 0x002D) { # -
2271
2272 $self->{state} = COMMENT_END_STATE;
2273
2274 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2275 $self->{line_prev} = $self->{line};
2276 $self->{column_prev} = $self->{column};
2277 $self->{column}++;
2278 $self->{nc}
2279 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2280 } else {
2281 $self->{set_nc}->($self);
2282 }
2283
2284 redo A;
2285 } elsif ($self->{nc} == 0x003E) { # >
2286
2287 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bogus comment');
2288 $self->{state} = DATA_STATE;
2289
2290 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2291 $self->{line_prev} = $self->{line};
2292 $self->{column_prev} = $self->{column};
2293 $self->{column}++;
2294 $self->{nc}
2295 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2296 } else {
2297 $self->{set_nc}->($self);
2298 }
2299
2300
2301 return ($self->{ct}); # comment
2302
2303 redo A;
2304 } elsif ($self->{nc} == -1) {
2305
2306 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed comment');
2307 $self->{state} = DATA_STATE;
2308 ## reconsume
2309
2310 return ($self->{ct}); # comment
2311
2312 redo A;
2313 } else {
2314
2315 $self->{ct}->{data} # comment
2316 .= '-' . chr ($self->{nc});
2317 $self->{state} = COMMENT_STATE;
2318
2319 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2320 $self->{line_prev} = $self->{line};
2321 $self->{column_prev} = $self->{column};
2322 $self->{column}++;
2323 $self->{nc}
2324 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2325 } else {
2326 $self->{set_nc}->($self);
2327 }
2328
2329 redo A;
2330 }
2331 } elsif ($self->{state} == COMMENT_STATE) {
2332 if ($self->{nc} == 0x002D) { # -
2333
2334 $self->{state} = COMMENT_END_DASH_STATE;
2335
2336 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2337 $self->{line_prev} = $self->{line};
2338 $self->{column_prev} = $self->{column};
2339 $self->{column}++;
2340 $self->{nc}
2341 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2342 } else {
2343 $self->{set_nc}->($self);
2344 }
2345
2346 redo A;
2347 } elsif ($self->{nc} == -1) {
2348
2349 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed comment');
2350 $self->{state} = DATA_STATE;
2351 ## reconsume
2352
2353 return ($self->{ct}); # comment
2354
2355 redo A;
2356 } else {
2357
2358 $self->{ct}->{data} .= chr ($self->{nc}); # comment
2359 $self->{read_until}->($self->{ct}->{data},
2360 q[-],
2361 length $self->{ct}->{data});
2362
2363 ## Stay in the state
2364
2365 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2366 $self->{line_prev} = $self->{line};
2367 $self->{column_prev} = $self->{column};
2368 $self->{column}++;
2369 $self->{nc}
2370 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2371 } else {
2372 $self->{set_nc}->($self);
2373 }
2374
2375 redo A;
2376 }
2377 } elsif ($self->{state} == COMMENT_END_DASH_STATE) {
2378 if ($self->{nc} == 0x002D) { # -
2379
2380 $self->{state} = COMMENT_END_STATE;
2381
2382 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2383 $self->{line_prev} = $self->{line};
2384 $self->{column_prev} = $self->{column};
2385 $self->{column}++;
2386 $self->{nc}
2387 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2388 } else {
2389 $self->{set_nc}->($self);
2390 }
2391
2392 redo A;
2393 } elsif ($self->{nc} == -1) {
2394
2395 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed comment');
2396 $self->{state} = DATA_STATE;
2397 ## reconsume
2398
2399 return ($self->{ct}); # comment
2400
2401 redo A;
2402 } else {
2403
2404 $self->{ct}->{data} .= '-' . chr ($self->{nc}); # comment
2405 $self->{state} = COMMENT_STATE;
2406
2407 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2408 $self->{line_prev} = $self->{line};
2409 $self->{column_prev} = $self->{column};
2410 $self->{column}++;
2411 $self->{nc}
2412 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2413 } else {
2414 $self->{set_nc}->($self);
2415 }
2416
2417 redo A;
2418 }
2419 } elsif ($self->{state} == COMMENT_END_STATE) {
2420 if ($self->{nc} == 0x003E) { # >
2421
2422 $self->{state} = DATA_STATE;
2423
2424 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2425 $self->{line_prev} = $self->{line};
2426 $self->{column_prev} = $self->{column};
2427 $self->{column}++;
2428 $self->{nc}
2429 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2430 } else {
2431 $self->{set_nc}->($self);
2432 }
2433
2434
2435 return ($self->{ct}); # comment
2436
2437 redo A;
2438 } elsif ($self->{nc} == 0x002D) { # -
2439
2440 $self->{parse_error}->(level => $self->{level}->{must}, type => 'dash in comment',
2441 line => $self->{line_prev},
2442 column => $self->{column_prev});
2443 $self->{ct}->{data} .= '-'; # comment
2444 ## Stay in the state
2445
2446 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2447 $self->{line_prev} = $self->{line};
2448 $self->{column_prev} = $self->{column};
2449 $self->{column}++;
2450 $self->{nc}
2451 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2452 } else {
2453 $self->{set_nc}->($self);
2454 }
2455
2456 redo A;
2457 } elsif ($self->{nc} == -1) {
2458
2459 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed comment');
2460 $self->{state} = DATA_STATE;
2461 ## reconsume
2462
2463 return ($self->{ct}); # comment
2464
2465 redo A;
2466 } else {
2467
2468 $self->{parse_error}->(level => $self->{level}->{must}, type => 'dash in comment',
2469 line => $self->{line_prev},
2470 column => $self->{column_prev});
2471 $self->{ct}->{data} .= '--' . chr ($self->{nc}); # comment
2472 $self->{state} = COMMENT_STATE;
2473
2474 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2475 $self->{line_prev} = $self->{line};
2476 $self->{column_prev} = $self->{column};
2477 $self->{column}++;
2478 $self->{nc}
2479 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2480 } else {
2481 $self->{set_nc}->($self);
2482 }
2483
2484 redo A;
2485 }
2486 } elsif ($self->{state} == DOCTYPE_STATE) {
2487 if ($is_space->{$self->{nc}}) {
2488
2489 $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
2490
2491 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2492 $self->{line_prev} = $self->{line};
2493 $self->{column_prev} = $self->{column};
2494 $self->{column}++;
2495 $self->{nc}
2496 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2497 } else {
2498 $self->{set_nc}->($self);
2499 }
2500
2501 redo A;
2502 } else {
2503
2504 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no space before DOCTYPE name');
2505 $self->{state} = BEFORE_DOCTYPE_NAME_STATE;
2506 ## reconsume
2507 redo A;
2508 }
2509 } elsif ($self->{state} == BEFORE_DOCTYPE_NAME_STATE) {
2510 if ($is_space->{$self->{nc}}) {
2511
2512 ## Stay in the state
2513
2514 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2515 $self->{line_prev} = $self->{line};
2516 $self->{column_prev} = $self->{column};
2517 $self->{column}++;
2518 $self->{nc}
2519 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2520 } else {
2521 $self->{set_nc}->($self);
2522 }
2523
2524 redo A;
2525 } elsif ($self->{nc} == 0x003E) { # >
2526
2527 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no DOCTYPE name');
2528 $self->{state} = DATA_STATE;
2529
2530 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2531 $self->{line_prev} = $self->{line};
2532 $self->{column_prev} = $self->{column};
2533 $self->{column}++;
2534 $self->{nc}
2535 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2536 } else {
2537 $self->{set_nc}->($self);
2538 }
2539
2540
2541 return ($self->{ct}); # DOCTYPE (quirks)
2542
2543 redo A;
2544 } elsif ($self->{nc} == -1) {
2545
2546 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no DOCTYPE name');
2547 $self->{state} = DATA_STATE;
2548 ## reconsume
2549
2550 return ($self->{ct}); # DOCTYPE (quirks)
2551
2552 redo A;
2553 } else {
2554
2555 $self->{ct}->{name} = chr $self->{nc};
2556 delete $self->{ct}->{quirks};
2557 $self->{state} = DOCTYPE_NAME_STATE;
2558
2559 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2560 $self->{line_prev} = $self->{line};
2561 $self->{column_prev} = $self->{column};
2562 $self->{column}++;
2563 $self->{nc}
2564 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2565 } else {
2566 $self->{set_nc}->($self);
2567 }
2568
2569 redo A;
2570 }
2571 } elsif ($self->{state} == DOCTYPE_NAME_STATE) {
2572 ## ISSUE: Redundant "First," in the spec.
2573 if ($is_space->{$self->{nc}}) {
2574
2575 $self->{state} = AFTER_DOCTYPE_NAME_STATE;
2576
2577 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2578 $self->{line_prev} = $self->{line};
2579 $self->{column_prev} = $self->{column};
2580 $self->{column}++;
2581 $self->{nc}
2582 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2583 } else {
2584 $self->{set_nc}->($self);
2585 }
2586
2587 redo A;
2588 } elsif ($self->{nc} == 0x003E) { # >
2589
2590 $self->{state} = DATA_STATE;
2591
2592 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2593 $self->{line_prev} = $self->{line};
2594 $self->{column_prev} = $self->{column};
2595 $self->{column}++;
2596 $self->{nc}
2597 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2598 } else {
2599 $self->{set_nc}->($self);
2600 }
2601
2602
2603 return ($self->{ct}); # DOCTYPE
2604
2605 redo A;
2606 } elsif ($self->{nc} == -1) {
2607
2608 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed DOCTYPE');
2609 $self->{state} = DATA_STATE;
2610 ## reconsume
2611
2612 $self->{ct}->{quirks} = 1;
2613 return ($self->{ct}); # DOCTYPE
2614
2615 redo A;
2616 } else {
2617
2618 $self->{ct}->{name}
2619 .= chr ($self->{nc}); # DOCTYPE
2620 ## Stay in the state
2621
2622 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2623 $self->{line_prev} = $self->{line};
2624 $self->{column_prev} = $self->{column};
2625 $self->{column}++;
2626 $self->{nc}
2627 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2628 } else {
2629 $self->{set_nc}->($self);
2630 }
2631
2632 redo A;
2633 }
2634 } elsif ($self->{state} == AFTER_DOCTYPE_NAME_STATE) {
2635 if ($is_space->{$self->{nc}}) {
2636
2637 ## Stay in the state
2638
2639 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2640 $self->{line_prev} = $self->{line};
2641 $self->{column_prev} = $self->{column};
2642 $self->{column}++;
2643 $self->{nc}
2644 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2645 } else {
2646 $self->{set_nc}->($self);
2647 }
2648
2649 redo A;
2650 } elsif ($self->{nc} == 0x003E) { # >
2651
2652 $self->{state} = DATA_STATE;
2653
2654 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2655 $self->{line_prev} = $self->{line};
2656 $self->{column_prev} = $self->{column};
2657 $self->{column}++;
2658 $self->{nc}
2659 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2660 } else {
2661 $self->{set_nc}->($self);
2662 }
2663
2664
2665 return ($self->{ct}); # DOCTYPE
2666
2667 redo A;
2668 } elsif ($self->{nc} == -1) {
2669
2670 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed DOCTYPE');
2671 $self->{state} = DATA_STATE;
2672 ## reconsume
2673
2674 $self->{ct}->{quirks} = 1;
2675 return ($self->{ct}); # DOCTYPE
2676
2677 redo A;
2678 } elsif ($self->{nc} == 0x0050 or # P
2679 $self->{nc} == 0x0070) { # p
2680 $self->{state} = PUBLIC_STATE;
2681 $self->{s_kwd} = chr $self->{nc};
2682
2683 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2684 $self->{line_prev} = $self->{line};
2685 $self->{column_prev} = $self->{column};
2686 $self->{column}++;
2687 $self->{nc}
2688 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2689 } else {
2690 $self->{set_nc}->($self);
2691 }
2692
2693 redo A;
2694 } elsif ($self->{nc} == 0x0053 or # S
2695 $self->{nc} == 0x0073) { # s
2696 $self->{state} = SYSTEM_STATE;
2697 $self->{s_kwd} = chr $self->{nc};
2698
2699 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2700 $self->{line_prev} = $self->{line};
2701 $self->{column_prev} = $self->{column};
2702 $self->{column}++;
2703 $self->{nc}
2704 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2705 } else {
2706 $self->{set_nc}->($self);
2707 }
2708
2709 redo A;
2710 } else {
2711
2712 $self->{parse_error}->(level => $self->{level}->{must}, type => 'string after DOCTYPE name');
2713 $self->{ct}->{quirks} = 1;
2714
2715 $self->{state} = BOGUS_DOCTYPE_STATE;
2716
2717 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2718 $self->{line_prev} = $self->{line};
2719 $self->{column_prev} = $self->{column};
2720 $self->{column}++;
2721 $self->{nc}
2722 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2723 } else {
2724 $self->{set_nc}->($self);
2725 }
2726
2727 redo A;
2728 }
2729 } elsif ($self->{state} == PUBLIC_STATE) {
2730 ## ASCII case-insensitive
2731 if ($self->{nc} == [
2732 undef,
2733 0x0055, # U
2734 0x0042, # B
2735 0x004C, # L
2736 0x0049, # I
2737 ]->[length $self->{s_kwd}] or
2738 $self->{nc} == [
2739 undef,
2740 0x0075, # u
2741 0x0062, # b
2742 0x006C, # l
2743 0x0069, # i
2744 ]->[length $self->{s_kwd}]) {
2745
2746 ## Stay in the state.
2747 $self->{s_kwd} .= chr $self->{nc};
2748
2749 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2750 $self->{line_prev} = $self->{line};
2751 $self->{column_prev} = $self->{column};
2752 $self->{column}++;
2753 $self->{nc}
2754 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2755 } else {
2756 $self->{set_nc}->($self);
2757 }
2758
2759 redo A;
2760 } elsif ((length $self->{s_kwd}) == 5 and
2761 ($self->{nc} == 0x0043 or # C
2762 $self->{nc} == 0x0063)) { # c
2763
2764 $self->{state} = BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
2765
2766 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2767 $self->{line_prev} = $self->{line};
2768 $self->{column_prev} = $self->{column};
2769 $self->{column}++;
2770 $self->{nc}
2771 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2772 } else {
2773 $self->{set_nc}->($self);
2774 }
2775
2776 redo A;
2777 } else {
2778
2779 $self->{parse_error}->(level => $self->{level}->{must}, type => 'string after DOCTYPE name',
2780 line => $self->{line_prev},
2781 column => $self->{column_prev} + 1 - length $self->{s_kwd});
2782 $self->{ct}->{quirks} = 1;
2783
2784 $self->{state} = BOGUS_DOCTYPE_STATE;
2785 ## Reconsume.
2786 redo A;
2787 }
2788 } elsif ($self->{state} == SYSTEM_STATE) {
2789 ## ASCII case-insensitive
2790 if ($self->{nc} == [
2791 undef,
2792 0x0059, # Y
2793 0x0053, # S
2794 0x0054, # T
2795 0x0045, # E
2796 ]->[length $self->{s_kwd}] or
2797 $self->{nc} == [
2798 undef,
2799 0x0079, # y
2800 0x0073, # s
2801 0x0074, # t
2802 0x0065, # e
2803 ]->[length $self->{s_kwd}]) {
2804
2805 ## Stay in the state.
2806 $self->{s_kwd} .= chr $self->{nc};
2807
2808 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2809 $self->{line_prev} = $self->{line};
2810 $self->{column_prev} = $self->{column};
2811 $self->{column}++;
2812 $self->{nc}
2813 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2814 } else {
2815 $self->{set_nc}->($self);
2816 }
2817
2818 redo A;
2819 } elsif ((length $self->{s_kwd}) == 5 and
2820 ($self->{nc} == 0x004D or # M
2821 $self->{nc} == 0x006D)) { # m
2822
2823 $self->{state} = BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
2824
2825 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2826 $self->{line_prev} = $self->{line};
2827 $self->{column_prev} = $self->{column};
2828 $self->{column}++;
2829 $self->{nc}
2830 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2831 } else {
2832 $self->{set_nc}->($self);
2833 }
2834
2835 redo A;
2836 } else {
2837
2838 $self->{parse_error}->(level => $self->{level}->{must}, type => 'string after DOCTYPE name',
2839 line => $self->{line_prev},
2840 column => $self->{column_prev} + 1 - length $self->{s_kwd});
2841 $self->{ct}->{quirks} = 1;
2842
2843 $self->{state} = BOGUS_DOCTYPE_STATE;
2844 ## Reconsume.
2845 redo A;
2846 }
2847 } elsif ($self->{state} == BEFORE_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
2848 if ($is_space->{$self->{nc}}) {
2849
2850 ## Stay in the state
2851
2852 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2853 $self->{line_prev} = $self->{line};
2854 $self->{column_prev} = $self->{column};
2855 $self->{column}++;
2856 $self->{nc}
2857 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2858 } else {
2859 $self->{set_nc}->($self);
2860 }
2861
2862 redo A;
2863 } elsif ($self->{nc} eq 0x0022) { # "
2864
2865 $self->{ct}->{pubid} = ''; # DOCTYPE
2866 $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE;
2867
2868 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2869 $self->{line_prev} = $self->{line};
2870 $self->{column_prev} = $self->{column};
2871 $self->{column}++;
2872 $self->{nc}
2873 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2874 } else {
2875 $self->{set_nc}->($self);
2876 }
2877
2878 redo A;
2879 } elsif ($self->{nc} eq 0x0027) { # '
2880
2881 $self->{ct}->{pubid} = ''; # DOCTYPE
2882 $self->{state} = DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE;
2883
2884 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2885 $self->{line_prev} = $self->{line};
2886 $self->{column_prev} = $self->{column};
2887 $self->{column}++;
2888 $self->{nc}
2889 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2890 } else {
2891 $self->{set_nc}->($self);
2892 }
2893
2894 redo A;
2895 } elsif ($self->{nc} eq 0x003E) { # >
2896
2897 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no PUBLIC literal');
2898
2899 $self->{state} = DATA_STATE;
2900
2901 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2902 $self->{line_prev} = $self->{line};
2903 $self->{column_prev} = $self->{column};
2904 $self->{column}++;
2905 $self->{nc}
2906 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2907 } else {
2908 $self->{set_nc}->($self);
2909 }
2910
2911
2912 $self->{ct}->{quirks} = 1;
2913 return ($self->{ct}); # DOCTYPE
2914
2915 redo A;
2916 } elsif ($self->{nc} == -1) {
2917
2918 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed DOCTYPE');
2919
2920 $self->{state} = DATA_STATE;
2921 ## reconsume
2922
2923 $self->{ct}->{quirks} = 1;
2924 return ($self->{ct}); # DOCTYPE
2925
2926 redo A;
2927 } else {
2928
2929 $self->{parse_error}->(level => $self->{level}->{must}, type => 'string after PUBLIC');
2930 $self->{ct}->{quirks} = 1;
2931
2932 $self->{state} = BOGUS_DOCTYPE_STATE;
2933
2934 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2935 $self->{line_prev} = $self->{line};
2936 $self->{column_prev} = $self->{column};
2937 $self->{column}++;
2938 $self->{nc}
2939 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2940 } else {
2941 $self->{set_nc}->($self);
2942 }
2943
2944 redo A;
2945 }
2946 } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED_STATE) {
2947 if ($self->{nc} == 0x0022) { # "
2948
2949 $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
2950
2951 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2952 $self->{line_prev} = $self->{line};
2953 $self->{column_prev} = $self->{column};
2954 $self->{column}++;
2955 $self->{nc}
2956 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2957 } else {
2958 $self->{set_nc}->($self);
2959 }
2960
2961 redo A;
2962 } elsif ($self->{nc} == 0x003E) { # >
2963
2964 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed PUBLIC literal');
2965
2966 $self->{state} = DATA_STATE;
2967
2968 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
2969 $self->{line_prev} = $self->{line};
2970 $self->{column_prev} = $self->{column};
2971 $self->{column}++;
2972 $self->{nc}
2973 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
2974 } else {
2975 $self->{set_nc}->($self);
2976 }
2977
2978
2979 $self->{ct}->{quirks} = 1;
2980 return ($self->{ct}); # DOCTYPE
2981
2982 redo A;
2983 } elsif ($self->{nc} == -1) {
2984
2985 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed PUBLIC literal');
2986
2987 $self->{state} = DATA_STATE;
2988 ## reconsume
2989
2990 $self->{ct}->{quirks} = 1;
2991 return ($self->{ct}); # DOCTYPE
2992
2993 redo A;
2994 } else {
2995
2996 $self->{ct}->{pubid} # DOCTYPE
2997 .= chr $self->{nc};
2998 $self->{read_until}->($self->{ct}->{pubid}, q[">],
2999 length $self->{ct}->{pubid});
3000
3001 ## Stay in the state
3002
3003 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3004 $self->{line_prev} = $self->{line};
3005 $self->{column_prev} = $self->{column};
3006 $self->{column}++;
3007 $self->{nc}
3008 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3009 } else {
3010 $self->{set_nc}->($self);
3011 }
3012
3013 redo A;
3014 }
3015 } elsif ($self->{state} == DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED_STATE) {
3016 if ($self->{nc} == 0x0027) { # '
3017
3018 $self->{state} = AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE;
3019
3020 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3021 $self->{line_prev} = $self->{line};
3022 $self->{column_prev} = $self->{column};
3023 $self->{column}++;
3024 $self->{nc}
3025 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3026 } else {
3027 $self->{set_nc}->($self);
3028 }
3029
3030 redo A;
3031 } elsif ($self->{nc} == 0x003E) { # >
3032
3033 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed PUBLIC literal');
3034
3035 $self->{state} = DATA_STATE;
3036
3037 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3038 $self->{line_prev} = $self->{line};
3039 $self->{column_prev} = $self->{column};
3040 $self->{column}++;
3041 $self->{nc}
3042 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3043 } else {
3044 $self->{set_nc}->($self);
3045 }
3046
3047
3048 $self->{ct}->{quirks} = 1;
3049 return ($self->{ct}); # DOCTYPE
3050
3051 redo A;
3052 } elsif ($self->{nc} == -1) {
3053
3054 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed PUBLIC literal');
3055
3056 $self->{state} = DATA_STATE;
3057 ## reconsume
3058
3059 $self->{ct}->{quirks} = 1;
3060 return ($self->{ct}); # DOCTYPE
3061
3062 redo A;
3063 } else {
3064
3065 $self->{ct}->{pubid} # DOCTYPE
3066 .= chr $self->{nc};
3067 $self->{read_until}->($self->{ct}->{pubid}, q['>],
3068 length $self->{ct}->{pubid});
3069
3070 ## Stay in the state
3071
3072 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3073 $self->{line_prev} = $self->{line};
3074 $self->{column_prev} = $self->{column};
3075 $self->{column}++;
3076 $self->{nc}
3077 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3078 } else {
3079 $self->{set_nc}->($self);
3080 }
3081
3082 redo A;
3083 }
3084 } elsif ($self->{state} == AFTER_DOCTYPE_PUBLIC_IDENTIFIER_STATE) {
3085 if ($is_space->{$self->{nc}}) {
3086
3087 ## Stay in the state
3088
3089 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3090 $self->{line_prev} = $self->{line};
3091 $self->{column_prev} = $self->{column};
3092 $self->{column}++;
3093 $self->{nc}
3094 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3095 } else {
3096 $self->{set_nc}->($self);
3097 }
3098
3099 redo A;
3100 } elsif ($self->{nc} == 0x0022) { # "
3101
3102 $self->{ct}->{sysid} = ''; # DOCTYPE
3103 $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
3104
3105 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3106 $self->{line_prev} = $self->{line};
3107 $self->{column_prev} = $self->{column};
3108 $self->{column}++;
3109 $self->{nc}
3110 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3111 } else {
3112 $self->{set_nc}->($self);
3113 }
3114
3115 redo A;
3116 } elsif ($self->{nc} == 0x0027) { # '
3117
3118 $self->{ct}->{sysid} = ''; # DOCTYPE
3119 $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
3120
3121 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3122 $self->{line_prev} = $self->{line};
3123 $self->{column_prev} = $self->{column};
3124 $self->{column}++;
3125 $self->{nc}
3126 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3127 } else {
3128 $self->{set_nc}->($self);
3129 }
3130
3131 redo A;
3132 } elsif ($self->{nc} == 0x003E) { # >
3133
3134 $self->{state} = DATA_STATE;
3135
3136 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3137 $self->{line_prev} = $self->{line};
3138 $self->{column_prev} = $self->{column};
3139 $self->{column}++;
3140 $self->{nc}
3141 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3142 } else {
3143 $self->{set_nc}->($self);
3144 }
3145
3146
3147 return ($self->{ct}); # DOCTYPE
3148
3149 redo A;
3150 } elsif ($self->{nc} == -1) {
3151
3152 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed DOCTYPE');
3153
3154 $self->{state} = DATA_STATE;
3155 ## reconsume
3156
3157 $self->{ct}->{quirks} = 1;
3158 return ($self->{ct}); # DOCTYPE
3159
3160 redo A;
3161 } else {
3162
3163 $self->{parse_error}->(level => $self->{level}->{must}, type => 'string after PUBLIC literal');
3164 $self->{ct}->{quirks} = 1;
3165
3166 $self->{state} = BOGUS_DOCTYPE_STATE;
3167
3168 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3169 $self->{line_prev} = $self->{line};
3170 $self->{column_prev} = $self->{column};
3171 $self->{column}++;
3172 $self->{nc}
3173 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3174 } else {
3175 $self->{set_nc}->($self);
3176 }
3177
3178 redo A;
3179 }
3180 } elsif ($self->{state} == BEFORE_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
3181 if ($is_space->{$self->{nc}}) {
3182
3183 ## Stay in the state
3184
3185 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3186 $self->{line_prev} = $self->{line};
3187 $self->{column_prev} = $self->{column};
3188 $self->{column}++;
3189 $self->{nc}
3190 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3191 } else {
3192 $self->{set_nc}->($self);
3193 }
3194
3195 redo A;
3196 } elsif ($self->{nc} == 0x0022) { # "
3197
3198 $self->{ct}->{sysid} = ''; # DOCTYPE
3199 $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE;
3200
3201 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3202 $self->{line_prev} = $self->{line};
3203 $self->{column_prev} = $self->{column};
3204 $self->{column}++;
3205 $self->{nc}
3206 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3207 } else {
3208 $self->{set_nc}->($self);
3209 }
3210
3211 redo A;
3212 } elsif ($self->{nc} == 0x0027) { # '
3213
3214 $self->{ct}->{sysid} = ''; # DOCTYPE
3215 $self->{state} = DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE;
3216
3217 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3218 $self->{line_prev} = $self->{line};
3219 $self->{column_prev} = $self->{column};
3220 $self->{column}++;
3221 $self->{nc}
3222 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3223 } else {
3224 $self->{set_nc}->($self);
3225 }
3226
3227 redo A;
3228 } elsif ($self->{nc} == 0x003E) { # >
3229
3230 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no SYSTEM literal');
3231 $self->{state} = DATA_STATE;
3232
3233 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3234 $self->{line_prev} = $self->{line};
3235 $self->{column_prev} = $self->{column};
3236 $self->{column}++;
3237 $self->{nc}
3238 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3239 } else {
3240 $self->{set_nc}->($self);
3241 }
3242
3243
3244 $self->{ct}->{quirks} = 1;
3245 return ($self->{ct}); # DOCTYPE
3246
3247 redo A;
3248 } elsif ($self->{nc} == -1) {
3249
3250 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed DOCTYPE');
3251
3252 $self->{state} = DATA_STATE;
3253 ## reconsume
3254
3255 $self->{ct}->{quirks} = 1;
3256 return ($self->{ct}); # DOCTYPE
3257
3258 redo A;
3259 } else {
3260
3261 $self->{parse_error}->(level => $self->{level}->{must}, type => 'string after SYSTEM');
3262 $self->{ct}->{quirks} = 1;
3263
3264 $self->{state} = BOGUS_DOCTYPE_STATE;
3265
3266 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3267 $self->{line_prev} = $self->{line};
3268 $self->{column_prev} = $self->{column};
3269 $self->{column}++;
3270 $self->{nc}
3271 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3272 } else {
3273 $self->{set_nc}->($self);
3274 }
3275
3276 redo A;
3277 }
3278 } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED_STATE) {
3279 if ($self->{nc} == 0x0022) { # "
3280
3281 $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
3282
3283 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3284 $self->{line_prev} = $self->{line};
3285 $self->{column_prev} = $self->{column};
3286 $self->{column}++;
3287 $self->{nc}
3288 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3289 } else {
3290 $self->{set_nc}->($self);
3291 }
3292
3293 redo A;
3294 } elsif ($self->{nc} == 0x003E) { # >
3295
3296 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed SYSTEM literal');
3297
3298 $self->{state} = DATA_STATE;
3299
3300 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3301 $self->{line_prev} = $self->{line};
3302 $self->{column_prev} = $self->{column};
3303 $self->{column}++;
3304 $self->{nc}
3305 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3306 } else {
3307 $self->{set_nc}->($self);
3308 }
3309
3310
3311 $self->{ct}->{quirks} = 1;
3312 return ($self->{ct}); # DOCTYPE
3313
3314 redo A;
3315 } elsif ($self->{nc} == -1) {
3316
3317 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed SYSTEM literal');
3318
3319 $self->{state} = DATA_STATE;
3320 ## reconsume
3321
3322 $self->{ct}->{quirks} = 1;
3323 return ($self->{ct}); # DOCTYPE
3324
3325 redo A;
3326 } else {
3327
3328 $self->{ct}->{sysid} # DOCTYPE
3329 .= chr $self->{nc};
3330 $self->{read_until}->($self->{ct}->{sysid}, q[">],
3331 length $self->{ct}->{sysid});
3332
3333 ## Stay in the state
3334
3335 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3336 $self->{line_prev} = $self->{line};
3337 $self->{column_prev} = $self->{column};
3338 $self->{column}++;
3339 $self->{nc}
3340 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3341 } else {
3342 $self->{set_nc}->($self);
3343 }
3344
3345 redo A;
3346 }
3347 } elsif ($self->{state} == DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED_STATE) {
3348 if ($self->{nc} == 0x0027) { # '
3349
3350 $self->{state} = AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE;
3351
3352 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3353 $self->{line_prev} = $self->{line};
3354 $self->{column_prev} = $self->{column};
3355 $self->{column}++;
3356 $self->{nc}
3357 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3358 } else {
3359 $self->{set_nc}->($self);
3360 }
3361
3362 redo A;
3363 } elsif ($self->{nc} == 0x003E) { # >
3364
3365 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed SYSTEM literal');
3366
3367 $self->{state} = DATA_STATE;
3368
3369 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3370 $self->{line_prev} = $self->{line};
3371 $self->{column_prev} = $self->{column};
3372 $self->{column}++;
3373 $self->{nc}
3374 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3375 } else {
3376 $self->{set_nc}->($self);
3377 }
3378
3379
3380 $self->{ct}->{quirks} = 1;
3381 return ($self->{ct}); # DOCTYPE
3382
3383 redo A;
3384 } elsif ($self->{nc} == -1) {
3385
3386 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed SYSTEM literal');
3387
3388 $self->{state} = DATA_STATE;
3389 ## reconsume
3390
3391 $self->{ct}->{quirks} = 1;
3392 return ($self->{ct}); # DOCTYPE
3393
3394 redo A;
3395 } else {
3396
3397 $self->{ct}->{sysid} # DOCTYPE
3398 .= chr $self->{nc};
3399 $self->{read_until}->($self->{ct}->{sysid}, q['>],
3400 length $self->{ct}->{sysid});
3401
3402 ## Stay in the state
3403
3404 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3405 $self->{line_prev} = $self->{line};
3406 $self->{column_prev} = $self->{column};
3407 $self->{column}++;
3408 $self->{nc}
3409 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3410 } else {
3411 $self->{set_nc}->($self);
3412 }
3413
3414 redo A;
3415 }
3416 } elsif ($self->{state} == AFTER_DOCTYPE_SYSTEM_IDENTIFIER_STATE) {
3417 if ($is_space->{$self->{nc}}) {
3418
3419 ## Stay in the state
3420
3421 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3422 $self->{line_prev} = $self->{line};
3423 $self->{column_prev} = $self->{column};
3424 $self->{column}++;
3425 $self->{nc}
3426 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3427 } else {
3428 $self->{set_nc}->($self);
3429 }
3430
3431 redo A;
3432 } elsif ($self->{nc} == 0x003E) { # >
3433
3434 $self->{state} = DATA_STATE;
3435
3436 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3437 $self->{line_prev} = $self->{line};
3438 $self->{column_prev} = $self->{column};
3439 $self->{column}++;
3440 $self->{nc}
3441 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3442 } else {
3443 $self->{set_nc}->($self);
3444 }
3445
3446
3447 return ($self->{ct}); # DOCTYPE
3448
3449 redo A;
3450 } elsif ($self->{nc} == -1) {
3451
3452 $self->{parse_error}->(level => $self->{level}->{must}, type => 'unclosed DOCTYPE');
3453 $self->{state} = DATA_STATE;
3454 ## reconsume
3455
3456 $self->{ct}->{quirks} = 1;
3457 return ($self->{ct}); # DOCTYPE
3458
3459 redo A;
3460 } else {
3461
3462 $self->{parse_error}->(level => $self->{level}->{must}, type => 'string after SYSTEM literal');
3463 #$self->{ct}->{quirks} = 1;
3464
3465 $self->{state} = BOGUS_DOCTYPE_STATE;
3466
3467 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3468 $self->{line_prev} = $self->{line};
3469 $self->{column_prev} = $self->{column};
3470 $self->{column}++;
3471 $self->{nc}
3472 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3473 } else {
3474 $self->{set_nc}->($self);
3475 }
3476
3477 redo A;
3478 }
3479 } elsif ($self->{state} == BOGUS_DOCTYPE_STATE) {
3480 if ($self->{nc} == 0x003E) { # >
3481
3482 $self->{state} = DATA_STATE;
3483
3484 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3485 $self->{line_prev} = $self->{line};
3486 $self->{column_prev} = $self->{column};
3487 $self->{column}++;
3488 $self->{nc}
3489 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3490 } else {
3491 $self->{set_nc}->($self);
3492 }
3493
3494
3495 return ($self->{ct}); # DOCTYPE
3496
3497 redo A;
3498 } elsif ($self->{nc} == -1) {
3499
3500 $self->{state} = DATA_STATE;
3501 ## reconsume
3502
3503 return ($self->{ct}); # DOCTYPE
3504
3505 redo A;
3506 } else {
3507
3508 my $s = '';
3509 $self->{read_until}->($s, q[>], 0);
3510
3511 ## Stay in the state
3512
3513 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3514 $self->{line_prev} = $self->{line};
3515 $self->{column_prev} = $self->{column};
3516 $self->{column}++;
3517 $self->{nc}
3518 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3519 } else {
3520 $self->{set_nc}->($self);
3521 }
3522
3523 redo A;
3524 }
3525 } elsif ($self->{state} == CDATA_SECTION_STATE) {
3526 ## NOTE: "CDATA section state" in the state is jointly implemented
3527 ## by three states, |CDATA_SECTION_STATE|, |CDATA_SECTION_MSE1_STATE|,
3528 ## and |CDATA_SECTION_MSE2_STATE|.
3529
3530 if ($self->{nc} == 0x005D) { # ]
3531
3532 $self->{state} = CDATA_SECTION_MSE1_STATE;
3533
3534 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3535 $self->{line_prev} = $self->{line};
3536 $self->{column_prev} = $self->{column};
3537 $self->{column}++;
3538 $self->{nc}
3539 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3540 } else {
3541 $self->{set_nc}->($self);
3542 }
3543
3544 redo A;
3545 } elsif ($self->{nc} == -1) {
3546 $self->{state} = DATA_STATE;
3547
3548 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3549 $self->{line_prev} = $self->{line};
3550 $self->{column_prev} = $self->{column};
3551 $self->{column}++;
3552 $self->{nc}
3553 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3554 } else {
3555 $self->{set_nc}->($self);
3556 }
3557
3558 if (length $self->{ct}->{data}) { # character
3559
3560 return ($self->{ct}); # character
3561 } else {
3562
3563 ## No token to emit. $self->{ct} is discarded.
3564 }
3565 redo A;
3566 } else {
3567
3568 $self->{ct}->{data} .= chr $self->{nc};
3569 $self->{read_until}->($self->{ct}->{data},
3570 q<]>,
3571 length $self->{ct}->{data});
3572
3573 ## Stay in the state.
3574
3575 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3576 $self->{line_prev} = $self->{line};
3577 $self->{column_prev} = $self->{column};
3578 $self->{column}++;
3579 $self->{nc}
3580 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3581 } else {
3582 $self->{set_nc}->($self);
3583 }
3584
3585 redo A;
3586 }
3587
3588 ## ISSUE: "text tokens" in spec.
3589 } elsif ($self->{state} == CDATA_SECTION_MSE1_STATE) {
3590 if ($self->{nc} == 0x005D) { # ]
3591
3592 $self->{state} = CDATA_SECTION_MSE2_STATE;
3593
3594 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3595 $self->{line_prev} = $self->{line};
3596 $self->{column_prev} = $self->{column};
3597 $self->{column}++;
3598 $self->{nc}
3599 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3600 } else {
3601 $self->{set_nc}->($self);
3602 }
3603
3604 redo A;
3605 } else {
3606
3607 $self->{ct}->{data} .= ']';
3608 $self->{state} = CDATA_SECTION_STATE;
3609 ## Reconsume.
3610 redo A;
3611 }
3612 } elsif ($self->{state} == CDATA_SECTION_MSE2_STATE) {
3613 if ($self->{nc} == 0x003E) { # >
3614 $self->{state} = DATA_STATE;
3615
3616 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3617 $self->{line_prev} = $self->{line};
3618 $self->{column_prev} = $self->{column};
3619 $self->{column}++;
3620 $self->{nc}
3621 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3622 } else {
3623 $self->{set_nc}->($self);
3624 }
3625
3626 if (length $self->{ct}->{data}) { # character
3627
3628 return ($self->{ct}); # character
3629 } else {
3630
3631 ## No token to emit. $self->{ct} is discarded.
3632 }
3633 redo A;
3634 } elsif ($self->{nc} == 0x005D) { # ]
3635 # character
3636 $self->{ct}->{data} .= ']'; ## Add first "]" of "]]]".
3637 ## Stay in the state.
3638
3639 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3640 $self->{line_prev} = $self->{line};
3641 $self->{column_prev} = $self->{column};
3642 $self->{column}++;
3643 $self->{nc}
3644 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3645 } else {
3646 $self->{set_nc}->($self);
3647 }
3648
3649 redo A;
3650 } else {
3651
3652 $self->{ct}->{data} .= ']]'; # character
3653 $self->{state} = CDATA_SECTION_STATE;
3654 ## Reconsume.
3655 redo A;
3656 }
3657 } elsif ($self->{state} == ENTITY_STATE) {
3658 if ($is_space->{$self->{nc}} or
3659 {
3660 0x003C => 1, 0x0026 => 1, -1 => 1, # <, &
3661 $self->{entity_add} => 1,
3662 }->{$self->{nc}}) {
3663
3664 ## Don't consume
3665 ## No error
3666 ## Return nothing.
3667 #
3668 } elsif ($self->{nc} == 0x0023) { # #
3669
3670 $self->{state} = ENTITY_HASH_STATE;
3671 $self->{s_kwd} = '#';
3672
3673 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3674 $self->{line_prev} = $self->{line};
3675 $self->{column_prev} = $self->{column};
3676 $self->{column}++;
3677 $self->{nc}
3678 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3679 } else {
3680 $self->{set_nc}->($self);
3681 }
3682
3683 redo A;
3684 } elsif ((0x0041 <= $self->{nc} and
3685 $self->{nc} <= 0x005A) or # A..Z
3686 (0x0061 <= $self->{nc} and
3687 $self->{nc} <= 0x007A)) { # a..z
3688
3689 require Whatpm::_NamedEntityList;
3690 $self->{state} = ENTITY_NAME_STATE;
3691 $self->{s_kwd} = chr $self->{nc};
3692 $self->{entity__value} = $self->{s_kwd};
3693 $self->{entity__match} = 0;
3694
3695 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3696 $self->{line_prev} = $self->{line};
3697 $self->{column_prev} = $self->{column};
3698 $self->{column}++;
3699 $self->{nc}
3700 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3701 } else {
3702 $self->{set_nc}->($self);
3703 }
3704
3705 redo A;
3706 } else {
3707
3708 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bare ero');
3709 ## Return nothing.
3710 #
3711 }
3712
3713 ## NOTE: No character is consumed by the "consume a character
3714 ## reference" algorithm. In other word, there is an "&" character
3715 ## that does not introduce a character reference, which would be
3716 ## appended to the parent element or the attribute value in later
3717 ## process of the tokenizer.
3718
3719 if ($self->{prev_state} == DATA_STATE) {
3720
3721 $self->{state} = $self->{prev_state};
3722 ## Reconsume.
3723 return ({type => CHARACTER_TOKEN, data => '&',
3724 line => $self->{line_prev},
3725 column => $self->{column_prev},
3726 });
3727 redo A;
3728 } else {
3729
3730 $self->{ca}->{value} .= '&';
3731 $self->{state} = $self->{prev_state};
3732 ## Reconsume.
3733 redo A;
3734 }
3735 } elsif ($self->{state} == ENTITY_HASH_STATE) {
3736 if ($self->{nc} == 0x0078 or # x
3737 $self->{nc} == 0x0058) { # X
3738
3739 $self->{state} = HEXREF_X_STATE;
3740 $self->{s_kwd} .= chr $self->{nc};
3741
3742 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3743 $self->{line_prev} = $self->{line};
3744 $self->{column_prev} = $self->{column};
3745 $self->{column}++;
3746 $self->{nc}
3747 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3748 } else {
3749 $self->{set_nc}->($self);
3750 }
3751
3752 redo A;
3753 } elsif (0x0030 <= $self->{nc} and
3754 $self->{nc} <= 0x0039) { # 0..9
3755
3756 $self->{state} = NCR_NUM_STATE;
3757 $self->{s_kwd} = $self->{nc} - 0x0030;
3758
3759 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3760 $self->{line_prev} = $self->{line};
3761 $self->{column_prev} = $self->{column};
3762 $self->{column}++;
3763 $self->{nc}
3764 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3765 } else {
3766 $self->{set_nc}->($self);
3767 }
3768
3769 redo A;
3770 } else {
3771 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bare nero',
3772 line => $self->{line_prev},
3773 column => $self->{column_prev} - 1);
3774
3775 ## NOTE: According to the spec algorithm, nothing is returned,
3776 ## and then "&#" is appended to the parent element or the attribute
3777 ## value in the later processing.
3778
3779 if ($self->{prev_state} == DATA_STATE) {
3780
3781 $self->{state} = $self->{prev_state};
3782 ## Reconsume.
3783 return ({type => CHARACTER_TOKEN,
3784 data => '&#',
3785 line => $self->{line_prev},
3786 column => $self->{column_prev} - 1,
3787 });
3788 redo A;
3789 } else {
3790
3791 $self->{ca}->{value} .= '&#';
3792 $self->{state} = $self->{prev_state};
3793 ## Reconsume.
3794 redo A;
3795 }
3796 }
3797 } elsif ($self->{state} == NCR_NUM_STATE) {
3798 if (0x0030 <= $self->{nc} and
3799 $self->{nc} <= 0x0039) { # 0..9
3800
3801 $self->{s_kwd} *= 10;
3802 $self->{s_kwd} += $self->{nc} - 0x0030;
3803
3804 ## Stay in the state.
3805
3806 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3807 $self->{line_prev} = $self->{line};
3808 $self->{column_prev} = $self->{column};
3809 $self->{column}++;
3810 $self->{nc}
3811 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3812 } else {
3813 $self->{set_nc}->($self);
3814 }
3815
3816 redo A;
3817 } elsif ($self->{nc} == 0x003B) { # ;
3818
3819
3820 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3821 $self->{line_prev} = $self->{line};
3822 $self->{column_prev} = $self->{column};
3823 $self->{column}++;
3824 $self->{nc}
3825 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3826 } else {
3827 $self->{set_nc}->($self);
3828 }
3829
3830 #
3831 } else {
3832
3833 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no refc');
3834 ## Reconsume.
3835 #
3836 }
3837
3838 my $code = $self->{s_kwd};
3839 my $l = $self->{line_prev};
3840 my $c = $self->{column_prev};
3841 if ($charref_map->{$code}) {
3842
3843 $self->{parse_error}->(level => $self->{level}->{must}, type => 'invalid character reference',
3844 text => (sprintf 'U+%04X', $code),
3845 line => $l, column => $c);
3846 $code = $charref_map->{$code};
3847 } elsif ($code > 0x10FFFF) {
3848
3849 $self->{parse_error}->(level => $self->{level}->{must}, type => 'invalid character reference',
3850 text => (sprintf 'U-%08X', $code),
3851 line => $l, column => $c);
3852 $code = 0xFFFD;
3853 }
3854
3855 if ($self->{prev_state} == DATA_STATE) {
3856
3857 $self->{state} = $self->{prev_state};
3858 ## Reconsume.
3859 return ({type => CHARACTER_TOKEN, data => chr $code,
3860 line => $l, column => $c,
3861 });
3862 redo A;
3863 } else {
3864
3865 $self->{ca}->{value} .= chr $code;
3866 $self->{ca}->{has_reference} = 1;
3867 $self->{state} = $self->{prev_state};
3868 ## Reconsume.
3869 redo A;
3870 }
3871 } elsif ($self->{state} == HEXREF_X_STATE) {
3872 if ((0x0030 <= $self->{nc} and $self->{nc} <= 0x0039) or
3873 (0x0041 <= $self->{nc} and $self->{nc} <= 0x0046) or
3874 (0x0061 <= $self->{nc} and $self->{nc} <= 0x0066)) {
3875 # 0..9, A..F, a..f
3876
3877 $self->{state} = HEXREF_HEX_STATE;
3878 $self->{s_kwd} = 0;
3879 ## Reconsume.
3880 redo A;
3881 } else {
3882 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bare hcro',
3883 line => $self->{line_prev},
3884 column => $self->{column_prev} - 2);
3885
3886 ## NOTE: According to the spec algorithm, nothing is returned,
3887 ## and then "&#" followed by "X" or "x" is appended to the parent
3888 ## element or the attribute value in the later processing.
3889
3890 if ($self->{prev_state} == DATA_STATE) {
3891
3892 $self->{state} = $self->{prev_state};
3893 ## Reconsume.
3894 return ({type => CHARACTER_TOKEN,
3895 data => '&' . $self->{s_kwd},
3896 line => $self->{line_prev},
3897 column => $self->{column_prev} - length $self->{s_kwd},
3898 });
3899 redo A;
3900 } else {
3901
3902 $self->{ca}->{value} .= '&' . $self->{s_kwd};
3903 $self->{state} = $self->{prev_state};
3904 ## Reconsume.
3905 redo A;
3906 }
3907 }
3908 } elsif ($self->{state} == HEXREF_HEX_STATE) {
3909 if (0x0030 <= $self->{nc} and $self->{nc} <= 0x0039) {
3910 # 0..9
3911
3912 $self->{s_kwd} *= 0x10;
3913 $self->{s_kwd} += $self->{nc} - 0x0030;
3914 ## Stay in the state.
3915
3916 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3917 $self->{line_prev} = $self->{line};
3918 $self->{column_prev} = $self->{column};
3919 $self->{column}++;
3920 $self->{nc}
3921 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3922 } else {
3923 $self->{set_nc}->($self);
3924 }
3925
3926 redo A;
3927 } elsif (0x0061 <= $self->{nc} and
3928 $self->{nc} <= 0x0066) { # a..f
3929
3930 $self->{s_kwd} *= 0x10;
3931 $self->{s_kwd} += $self->{nc} - 0x0060 + 9;
3932 ## Stay in the state.
3933
3934 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3935 $self->{line_prev} = $self->{line};
3936 $self->{column_prev} = $self->{column};
3937 $self->{column}++;
3938 $self->{nc}
3939 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3940 } else {
3941 $self->{set_nc}->($self);
3942 }
3943
3944 redo A;
3945 } elsif (0x0041 <= $self->{nc} and
3946 $self->{nc} <= 0x0046) { # A..F
3947
3948 $self->{s_kwd} *= 0x10;
3949 $self->{s_kwd} += $self->{nc} - 0x0040 + 9;
3950 ## Stay in the state.
3951
3952 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3953 $self->{line_prev} = $self->{line};
3954 $self->{column_prev} = $self->{column};
3955 $self->{column}++;
3956 $self->{nc}
3957 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3958 } else {
3959 $self->{set_nc}->($self);
3960 }
3961
3962 redo A;
3963 } elsif ($self->{nc} == 0x003B) { # ;
3964
3965
3966 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
3967 $self->{line_prev} = $self->{line};
3968 $self->{column_prev} = $self->{column};
3969 $self->{column}++;
3970 $self->{nc}
3971 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
3972 } else {
3973 $self->{set_nc}->($self);
3974 }
3975
3976 #
3977 } else {
3978
3979 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no refc',
3980 line => $self->{line},
3981 column => $self->{column});
3982 ## Reconsume.
3983 #
3984 }
3985
3986 my $code = $self->{s_kwd};
3987 my $l = $self->{line_prev};
3988 my $c = $self->{column_prev};
3989 if ($charref_map->{$code}) {
3990
3991 $self->{parse_error}->(level => $self->{level}->{must}, type => 'invalid character reference',
3992 text => (sprintf 'U+%04X', $code),
3993 line => $l, column => $c);
3994 $code = $charref_map->{$code};
3995 } elsif ($code > 0x10FFFF) {
3996
3997 $self->{parse_error}->(level => $self->{level}->{must}, type => 'invalid character reference',
3998 text => (sprintf 'U-%08X', $code),
3999 line => $l, column => $c);
4000 $code = 0xFFFD;
4001 }
4002
4003 if ($self->{prev_state} == DATA_STATE) {
4004
4005 $self->{state} = $self->{prev_state};
4006 ## Reconsume.
4007 return ({type => CHARACTER_TOKEN, data => chr $code,
4008 line => $l, column => $c,
4009 });
4010 redo A;
4011 } else {
4012
4013 $self->{ca}->{value} .= chr $code;
4014 $self->{ca}->{has_reference} = 1;
4015 $self->{state} = $self->{prev_state};
4016 ## Reconsume.
4017 redo A;
4018 }
4019 } elsif ($self->{state} == ENTITY_NAME_STATE) {
4020 if (length $self->{s_kwd} < 30 and
4021 ## NOTE: Some number greater than the maximum length of entity name
4022 ((0x0041 <= $self->{nc} and # a
4023 $self->{nc} <= 0x005A) or # x
4024 (0x0061 <= $self->{nc} and # a
4025 $self->{nc} <= 0x007A) or # z
4026 (0x0030 <= $self->{nc} and # 0
4027 $self->{nc} <= 0x0039) or # 9
4028 $self->{nc} == 0x003B)) { # ;
4029 our $EntityChar;
4030 $self->{s_kwd} .= chr $self->{nc};
4031 if (defined $EntityChar->{$self->{s_kwd}}) {
4032 if ($self->{nc} == 0x003B) { # ;
4033
4034 $self->{entity__value} = $EntityChar->{$self->{s_kwd}};
4035 $self->{entity__match} = 1;
4036
4037 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
4038 $self->{line_prev} = $self->{line};
4039 $self->{column_prev} = $self->{column};
4040 $self->{column}++;
4041 $self->{nc}
4042 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
4043 } else {
4044 $self->{set_nc}->($self);
4045 }
4046
4047 #
4048 } else {
4049
4050 $self->{entity__value} = $EntityChar->{$self->{s_kwd}};
4051 $self->{entity__match} = -1;
4052 ## Stay in the state.
4053
4054 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
4055 $self->{line_prev} = $self->{line};
4056 $self->{column_prev} = $self->{column};
4057 $self->{column}++;
4058 $self->{nc}
4059 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
4060 } else {
4061 $self->{set_nc}->($self);
4062 }
4063
4064 redo A;
4065 }
4066 } else {
4067
4068 $self->{entity__value} .= chr $self->{nc};
4069 $self->{entity__match} *= 2;
4070 ## Stay in the state.
4071
4072 if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
4073 $self->{line_prev} = $self->{line};
4074 $self->{column_prev} = $self->{column};
4075 $self->{column}++;
4076 $self->{nc}
4077 = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
4078 } else {
4079 $self->{set_nc}->($self);
4080 }
4081
4082 redo A;
4083 }
4084 }
4085
4086 my $data;
4087 my $has_ref;
4088 if ($self->{entity__match} > 0) {
4089
4090 $data = $self->{entity__value};
4091 $has_ref = 1;
4092 #
4093 } elsif ($self->{entity__match} < 0) {
4094 $self->{parse_error}->(level => $self->{level}->{must}, type => 'no refc');
4095 if ($self->{prev_state} != DATA_STATE and # in attribute
4096 $self->{entity__match} < -1) {
4097
4098 $data = '&' . $self->{s_kwd};
4099 #
4100 } else {
4101
4102 $data = $self->{entity__value};
4103 $has_ref = 1;
4104 #
4105 }
4106 } else {
4107
4108 $self->{parse_error}->(level => $self->{level}->{must}, type => 'bare ero',
4109 line => $self->{line_prev},
4110 column => $self->{column_prev} - length $self->{s_kwd});
4111 $data = '&' . $self->{s_kwd};
4112 #
4113 }
4114
4115 ## NOTE: In these cases, when a character reference is found,
4116 ## it is consumed and a character token is returned, or, otherwise,
4117 ## nothing is consumed and returned, according to the spec algorithm.
4118 ## In this implementation, anything that has been examined by the
4119 ## tokenizer is appended to the parent element or the attribute value
4120 ## as string, either literal string when no character reference or
4121 ## entity-replaced string otherwise, in this stage, since any characters
4122 ## that would not be consumed are appended in the data state or in an
4123 ## appropriate attribute value state anyway.
4124
4125 if ($self->{prev_state} == DATA_STATE) {
4126
4127 $self->{state} = $self->{prev_state};
4128 ## Reconsume.
4129 return ({type => CHARACTER_TOKEN,
4130 data => $data,
4131 line => $self->{line_prev},
4132 column => $self->{column_prev} + 1 - length $self->{s_kwd},
4133 });
4134 redo A;
4135 } else {
4136
4137 $self->{ca}->{value} .= $data;
4138 $self->{ca}->{has_reference} = 1 if $has_ref;
4139 $self->{state} = $self->{prev_state};
4140 ## Reconsume.
4141 redo A;
4142 }
4143 } else {
4144 die "$0: $self->{state}: Unknown state";
4145 }
4146 } # A
4147
4148 die "$0: _get_next_token: unexpected case";
4149 } # _get_next_token
4150
4151 1;
4152 ## $Date: 2008/10/14 04:32:49 $
admin@suikawiki.org
 ViewVC Help
Powered by ViewVC 1.1.24