/[suikacvs]/markup/html/whatpm/Whatpm/ContentChecker.pm
Suika

Contents of /markup/html/whatpm/Whatpm/ContentChecker.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.51 - (show annotations) (download)
Sun Nov 18 11:06:14 2007 UTC (16 years, 11 months ago) by wakaba
Branch: MAIN
Changes since 1.50: +51 -5 lines
++ whatpm/Whatpm/ChangeLog	18 Nov 2007 11:06:04 -0000
	* ContentChecker.pm (check_document): Check the existence
	of character encoding declaration and selection of encoding
	for HTML document.

2007-11-18  Wakaba  <wakaba@suika.fam.cx>

1 package Whatpm::ContentChecker;
2 use strict;
3 our $VERSION=do{my @r=(q$Revision: 1.50 $=~/\d+/g);sprintf "%d."."%02d" x $#r,@r};
4
5 require Whatpm::URIChecker;
6
7 ## ISSUE: How XML and XML Namespaces conformance can (or cannot)
8 ## be applied to an in-memory representation (i.e. DOM)?
9
10 ## TODO: Conformance of an HTML document with non-html root element.
11
12 my $HTML_NS = q<http://www.w3.org/1999/xhtml>;
13 my $XML_NS = q<http://www.w3.org/XML/1998/namespace>;
14 my $XMLNS_NS = q<http://www.w3.org/2000/xmlns/>;
15
16 my $Namespace = {
17 q<http://www.w3.org/2005/Atom> => {module => 'Whatpm::ContentChecker::Atom'},
18 $HTML_NS => {module => 'Whatpm::ContentChecker::HTML'},
19 $XML_NS => {loaded => 1},
20 $XMLNS_NS => {loaded => 1},
21 };
22
23 our $AttrChecker = {
24 $XML_NS => {
25 space => sub {
26 my ($self, $attr) = @_;
27 my $value = $attr->value;
28 if ($value eq 'default' or $value eq 'preserve') {
29 #
30 } else {
31 ## NOTE: An XML "error"
32 $self->{onerror}->(node => $attr, level => 'error',
33 type => 'invalid attribute value');
34 }
35 },
36 lang => sub {
37 my ($self, $attr) = @_;
38 my $value = $attr->value;
39 if ($value eq '') {
40 #
41 } else {
42 require Whatpm::LangTag;
43 Whatpm::LangTag->check_rfc3066_language_tag ($value, sub {
44 my %opt = @_;
45 my $type = 'LangTag:'.$opt{type};
46 $type .= ':' . $opt{subtag} if defined $opt{subtag};
47 $self->{onerror}->(node => $attr, type => $type,
48 value => $opt{value}, level => $opt{level});
49 });
50 }
51
52 ## NOTE: "The values of the attribute are language identifiers
53 ## as defined by [IETF RFC 3066], Tags for the Identification
54 ## of Languages, or its successor; in addition, the empty string
55 ## may be specified." ("may" in lower case)
56 ## NOTE: Is an RFC 3066-valid (but RFC 4647-invalid) language tag
57 ## allowed today?
58
59 ## TODO: test data
60
61 if ($attr->owner_document->manakai_is_html) { # MUST NOT
62 $self->{onerror}->(node => $attr, type => 'in HTML:xml:lang');
63 ## TODO: Test data...
64 }
65 },
66 base => sub {
67 my ($self, $attr) = @_;
68 my $value = $attr->value;
69 if ($value =~ /[^\x{0000}-\x{10FFFF}]/) { ## ISSUE: Should we disallow noncharacters?
70 $self->{onerror}->(node => $attr,
71 type => 'invalid attribute value');
72 }
73 ## NOTE: Conformance to URI standard is not checked since there is
74 ## no author requirement on conformance in the XML Base specification.
75 },
76 id => sub {
77 my ($self, $attr) = @_;
78 my $value = $attr->value;
79 $value =~ s/[\x09\x0A\x0D\x20]+/ /g;
80 $value =~ s/^\x20//;
81 $value =~ s/\x20$//;
82 ## TODO: NCName in XML 1.0 or 1.1
83 ## TODO: declared type is ID?
84 if ($self->{id}->{$value}) { ## NOTE: An xml:id error
85 $self->{onerror}->(node => $attr, level => 'error',
86 type => 'duplicate ID');
87 push @{$self->{id}->{$value}}, $attr;
88 } else {
89 $self->{id}->{$value} = [$attr];
90 }
91 },
92 },
93 $XMLNS_NS => {
94 '' => sub {
95 my ($self, $attr) = @_;
96 my $ln = $attr->manakai_local_name;
97 my $value = $attr->value;
98 if ($value eq $XML_NS and $ln ne 'xml') {
99 $self->{onerror}
100 ->(node => $attr, level => 'NC',
101 type => 'Reserved Prefixes and Namespace Names:=xml');
102 } elsif ($value eq $XMLNS_NS) {
103 $self->{onerror}
104 ->(node => $attr, level => 'NC',
105 type => 'Reserved Prefixes and Namespace Names:=xmlns');
106 }
107 if ($ln eq 'xml' and $value ne $XML_NS) {
108 $self->{onerror}
109 ->(node => $attr, level => 'NC',
110 type => 'Reserved Prefixes and Namespace Names:xmlns:xml=');
111 } elsif ($ln eq 'xmlns') {
112 $self->{onerror}
113 ->(node => $attr, level => 'NC',
114 type => 'Reserved Prefixes and Namespace Names:xmlns:xmlns=');
115 }
116 ## TODO: If XML 1.0 and empty
117 },
118 xmlns => sub {
119 my ($self, $attr) = @_;
120 ## TODO: In XML 1.0, URI reference [RFC 3986] or an empty string
121 ## TODO: In XML 1.1, IRI reference [RFC 3987] or an empty string
122 ## TODO: relative references are deprecated
123 my $value = $attr->value;
124 if ($value eq $XML_NS) {
125 $self->{onerror}
126 ->(node => $attr, level => 'NC',
127 type => 'Reserved Prefixes and Namespace Names:=xml');
128 } elsif ($value eq $XMLNS_NS) {
129 $self->{onerror}
130 ->(node => $attr, level => 'NC',
131 type => 'Reserved Prefixes and Namespace Names:=xmlns');
132 }
133 },
134 },
135 };
136
137 ## ISSUE: Should we really allow these attributes?
138 $AttrChecker->{''}->{'xml:space'} = $AttrChecker->{$XML_NS}->{space};
139 $AttrChecker->{''}->{'xml:lang'} = $AttrChecker->{$XML_NS}->{lang};
140 $AttrChecker->{''}->{'xml:base'} = $AttrChecker->{$XML_NS}->{base};
141 $AttrChecker->{''}->{'xml:id'} = $AttrChecker->{$XML_NS}->{id};
142
143 ## ANY
144 our $AnyChecker = sub {
145 my ($self, $todo) = @_;
146 my $el = $todo->{node};
147 my $new_todos = [];
148 my @nodes = (@{$el->child_nodes});
149 while (@nodes) {
150 my $node = shift @nodes;
151 $self->_remove_minuses ($node) and next if ref $node eq 'HASH';
152
153 my $nt = $node->node_type;
154 if ($nt == 1) {
155 my $node_ns = $node->namespace_uri;
156 $node_ns = '' unless defined $node_ns;
157 my $node_ln = $node->manakai_local_name;
158 if ($self->{minuses}->{$node_ns}->{$node_ln}) {
159 $self->{onerror}->(node => $node, type => 'element not allowed');
160 }
161 push @$new_todos, {type => 'element', node => $node};
162 } elsif ($nt == 5) {
163 unshift @nodes, @{$node->child_nodes};
164 }
165 }
166 return ($new_todos);
167 }; # $AnyChecker
168
169 our $ElementDefault = {
170 checker => sub {
171 my ($self, $todo) = @_;
172 $self->{onerror}->(node => $todo->{node}, level => 'unsupported',
173 type => 'element');
174 return $AnyChecker->($self, $todo);
175 },
176 attrs_checker => sub {
177 my ($self, $todo) = @_;
178 for my $attr (@{$todo->{node}->attributes}) {
179 my $attr_ns = $attr->namespace_uri;
180 $attr_ns = '' unless defined $attr_ns;
181 my $attr_ln = $attr->manakai_local_name;
182 my $checker = $AttrChecker->{$attr_ns}->{$attr_ln}
183 || $AttrChecker->{$attr_ns}->{''};
184 if ($checker) {
185 $checker->($self, $attr);
186 } else {
187 $self->{onerror}->(node => $attr, level => 'unsupported',
188 type => 'attribute');
189 }
190 }
191 },
192 };
193
194 my $HTMLTransparentElements = {
195 $HTML_NS => {qw/ins 1 font 1 noscript 1/},
196 ## NOTE: |html:noscript| is transparent if scripting is disabled
197 ## and not in |head|.
198 };
199
200 our $Element = {};
201
202 sub check_document ($$$) {
203 my ($self, $doc, $onerror) = @_;
204 $self = bless {}, $self unless ref $self;
205 $self->{onerror} = $onerror;
206
207 $self->{must_level} = 'm';
208 $self->{fact_level} = 'f';
209 $self->{should_level} = 's';
210 $self->{good_level} = 'w';
211
212 my $docel = $doc->document_element;
213 unless (defined $docel) {
214 ## ISSUE: Should we check content of Document node?
215 $onerror->(node => $doc, type => 'no document element');
216 ## ISSUE: Is this non-conforming (to what spec)? Or just a warning?
217 return {
218 class => {},
219 id => {}, table => [], term => {},
220 };
221 }
222
223 ## ISSUE: Unexpanded entity references and HTML5 conformance
224
225 my $docel_nsuri = $docel->namespace_uri;
226 $docel_nsuri = '' unless defined $docel_nsuri;
227 unless ($Namespace->{$docel_nsuri}->{loaded}) {
228 if ($Namespace->{$docel_nsuri}->{module}) {
229 eval qq{ require $Namespace->{$docel_nsuri}->{module} } or die $@;
230 } else {
231 $Namespace->{$docel_nsuri}->{loaded} = 1;
232 }
233 }
234 my $docel_def = $Element->{$docel_nsuri}->{$docel->manakai_local_name} ||
235 $Element->{$docel_nsuri}->{''} ||
236 $ElementDefault;
237 if ($docel_def->{is_root}) {
238 #
239 } elsif ($docel_def->{is_xml_root}) {
240 unless ($doc->manakai_is_html) {
241 #
242 } else {
243 $onerror->(node => $docel, type => 'element not allowed:root:xml');
244 }
245 } else {
246 $onerror->(node => $docel, type => 'element not allowed:root');
247 }
248
249 ## TODO: Check for other items other than document element
250 ## (second (errorous) element, text nodes, PI nodes, doctype nodes)
251
252 my $return = $self->check_element ($docel, $onerror);
253
254 my $charset_name = $doc->input_encoding;
255 if (defined $charset_name) {
256 require Message::Charset::Info;
257 my $charset = $Message::Charset::Info::IANACharset->{$charset_name};
258
259 if ($doc->manakai_is_html and
260 not $doc->manakai_has_bom and
261 not defined $doc->manakai_charset) {
262 unless ($charset->{is_html_ascii_superset}) {
263 $onerror->(node => $doc, level => $self->{must_level},
264 type => 'non ascii superset:'.$charset_name);
265 }
266
267 if (not $self->{has_charset} and
268 not $charset->{iana_names}->{'us-ascii'}) {
269 $onerror->(node => $doc, level => $self->{must_level},
270 type => 'no character encoding declaration:'.$charset_name);
271 }
272 }
273
274 if ($charset->{iana_names}->{'utf-8'}) {
275 #
276 } elsif ($charset->{iana_names}->{'jis_x0212-1990'} or
277 $charset->{iana_names}->{'x-jis0208'} or
278 $charset->{iana_names}->{'utf-32'} or ## ISSUE: UTF-32BE? UTF-32LE?
279 $charset->{is_ebcdic_based}) {
280 $onerror->(node => $doc,
281 type => 'character encoding:'.$charset_name,
282 level => $self->{should_level});
283 } elsif ($charset->{iana_names}->{'cesu-8'} or
284 $charset->{iana_names}->{'utf-8'} or ## ISSUE: UNICODE-1-1-UTF-7?
285 $charset->{iana_names}->{'bocu-1'} or
286 $charset->{iana_names}->{'scsu'}) {
287 $onerror->(node => $doc,
288 type => 'character encoding:'.$charset_name,
289 level => $self->{must_level});
290 } else {
291 $onerror->(node => $doc,
292 type => 'character encoding:'.$charset_name,
293 level => $self->{good_level});
294 }
295 }
296
297 return $return;
298 } # check_document
299
300 sub check_element ($$$) {
301 my ($self, $el, $onerror) = @_;
302 $self = bless {}, $self unless ref $self;
303 $self->{onerror} = $onerror;
304
305 $self->{must_level} = 'm';
306 $self->{fact_level} = 'f';
307 $self->{should_level} = 's';
308 $self->{good_level} = 'w';
309
310 $self->{pluses} = {};
311 $self->{minuses} = {};
312 $self->{id} = {};
313 $self->{term} = {};
314 $self->{usemap} = [];
315 $self->{contextmenu} = [];
316 $self->{map} = {};
317 $self->{menu} = {};
318 $self->{has_link_type} = {};
319 #$self->{has_uri_attr};
320 #$self->{has_hyperlink_element};
321 #$self->{has_charset};
322 $self->{return} = {
323 class => {},
324 id => $self->{id}, table => [], term => $self->{term},
325 };
326
327 my @todo = ({type => 'element', node => $el});
328 while (@todo) {
329 my $todo = shift @todo;
330 if ($todo->{type} eq 'element') {
331 my $prefix = $todo->{node}->prefix;
332 if (defined $prefix and $prefix eq 'xmlns') {
333 $self->{onerror}
334 ->(node => $todo->{node}, level => 'NC',
335 type => 'Reserved Prefixes and Namespace Names:<xmlns:>');
336 }
337 my $nsuri = $todo->{node}->namespace_uri;
338 $nsuri = '' unless defined $nsuri;
339 unless ($Namespace->{$nsuri}->{loaded}) {
340 if ($Namespace->{$nsuri}->{module}) {
341 eval qq{ require $Namespace->{$nsuri}->{module} } or die $@;
342 } else {
343 $Namespace->{$nsuri}->{loaded} = 1;
344 }
345 }
346 my $ln = $todo->{node}->manakai_local_name;
347 my $eldef = $Element->{$nsuri}->{$ln} ||
348 $Element->{$nsuri}->{''} ||
349 $ElementDefault;
350 $eldef->{attrs_checker}->($self, $todo);
351 my ($new_todos) = $eldef->{checker}->($self, $todo);
352 unshift @todo, @$new_todos;
353 } elsif ($todo->{type} eq 'element-attributes') {
354 my $prefix = $todo->{node}->prefix;
355 if (defined $prefix and $prefix eq 'xmlns') {
356 $self->{onerror}
357 ->(node => $todo->{node}, level => 'NC',
358 type => 'Reserved Prefixes and Namespace Names:<xmlns:>');
359 }
360 my $nsuri = $todo->{node}->namespace_uri;
361 $nsuri = '' unless defined $nsuri;
362 unless ($Namespace->{$nsuri}->{loaded}) {
363 if ($Namespace->{$nsuri}->{module}) {
364 eval qq{ require $Namespace->{$nsuri}->{module} } or die $@;
365 } else {
366 $Namespace->{$nsuri}->{loaded} = 1;
367 }
368 }
369 my $ln = $todo->{node}->manakai_local_name;
370 my $eldef = $Element->{$nsuri}->{$ln} ||
371 $Element->{$nsuri}->{''} ||
372 $ElementDefault;
373 $eldef->{attrs_checker}->($self, $todo);
374 } elsif ($todo->{type} eq 'plus' or $todo->{type} eq 'minus') {
375 $self->_remove_minuses ($todo);
376 } elsif ($todo->{type} eq 'code') {
377 $todo->{code}->();
378 } else {
379 die "$0: Internal error: Unsupported checking action type |$todo->{type}|";
380 }
381 }
382
383 for (@{$self->{usemap}}) {
384 unless ($self->{map}->{$_->[0]}) {
385 $self->{onerror}->(node => $_->[1], type => 'no referenced map');
386 }
387 }
388
389 for (@{$self->{contextmenu}}) {
390 unless ($self->{menu}->{$_->[0]}) {
391 $self->{onerror}->(node => $_->[1], type => 'no referenced menu');
392 }
393 }
394
395 delete $self->{pluses};
396 delete $self->{minuses};
397 delete $self->{onerror};
398 delete $self->{id};
399 delete $self->{usemap};
400 delete $self->{map};
401 return $self->{return};
402 } # check_element
403
404 sub _add_minuses ($@) {
405 my $self = shift;
406 my $r = {};
407 for my $list (@_) {
408 for my $ns (keys %$list) {
409 for my $ln (keys %{$list->{$ns}}) {
410 unless ($self->{minuses}->{$ns}->{$ln}) {
411 $self->{minuses}->{$ns}->{$ln} = 1;
412 $r->{$ns}->{$ln} = 1;
413 }
414 }
415 }
416 }
417 return {type => 'plus', list => $r};
418 } # _add_minuses
419
420 sub _add_pluses ($@) {
421 my $self = shift;
422 my $r = {};
423 for my $list (@_) {
424 for my $ns (keys %$list) {
425 for my $ln (keys %{$list->{$ns}}) {
426 unless ($self->{pluses}->{$ns}->{$ln}) {
427 $self->{pluses}->{$ns}->{$ln} = 1;
428 $r->{$ns}->{$ln} = 1;
429 }
430 }
431 }
432 }
433 return {type => 'minus', list => $r};
434 } # _add_pluses
435
436 sub _remove_minuses ($$) {
437 my ($self, $todo) = @_;
438 if ($todo->{type} eq 'minus') {
439 for my $ns (keys %{$todo->{list}}) {
440 for my $ln (keys %{$todo->{list}->{$ns}}) {
441 delete $self->{pluses}->{$ns}->{$ln} if $todo->{list}->{$ns}->{$ln};
442 }
443 }
444 } elsif ($todo->{type} eq 'plus') {
445 for my $ns (keys %{$todo->{list}}) {
446 for my $ln (keys %{$todo->{list}->{$ns}}) {
447 delete $self->{minuses}->{$ns}->{$ln} if $todo->{list}->{$ns}->{$ln};
448 }
449 }
450 } else {
451 die "$0: Unknown +- type: $todo->{type}";
452 }
453 1;
454 } # _remove_minuses
455
456 ## NOTE: Priority for "minuses" and "pluses" are currently left
457 ## undefined and implemented inconsistently; it is not a problem for
458 ## now, since no element belongs to both lists.
459
460 sub _check_get_children ($$$) {
461 my ($self, $node, $parent_todo) = @_;
462 my $new_todos = [];
463 my $sib = [];
464 TP: {
465 my $node_ns = $node->namespace_uri;
466 $node_ns = '' unless defined $node_ns;
467 my $node_ln = $node->manakai_local_name;
468 if ($HTMLTransparentElements->{$node_ns}->{$node_ln}) {
469 if ($node_ns eq $HTML_NS and $node_ln eq 'noscript') {
470 if ($parent_todo->{flag}->{in_head}) {
471 #
472 } else {
473 my $end = $self->_add_minuses ({$HTML_NS, {noscript => 1}});
474 push @$sib, $end;
475
476 unshift @$sib, @{$node->child_nodes};
477 push @$new_todos, {type => 'element-attributes', node => $node};
478 last TP;
479 }
480 } else {
481 unshift @$sib, @{$node->child_nodes};
482 push @$new_todos, {type => 'element-attributes', node => $node};
483 last TP;
484 }
485 }
486 if ($node_ns eq $HTML_NS and ($node_ln eq 'video' or $node_ln eq 'audio')) {
487 if ($node->has_attribute_ns (undef, 'src')) {
488 unshift @$sib, @{$node->child_nodes};
489 push @$new_todos, {type => 'element-attributes', node => $node};
490 last TP;
491 } else {
492 my @cn = @{$node->child_nodes};
493 CN: while (@cn) {
494 my $cn = shift @cn;
495 my $cnt = $cn->node_type;
496 if ($cnt == 1) {
497 my $cn_nsuri = $cn->namespace_uri;
498 $cn_nsuri = '' unless defined $cn_nsuri;
499 if ($cn_nsuri eq $HTML_NS and $cn->manakai_local_name eq 'source') {
500 #
501 } else {
502 last CN;
503 }
504 } elsif ($cnt == 3 or $cnt == 4) {
505 if ($cn->data =~ /[^\x09-\x0D\x20]/) {
506 last CN;
507 }
508 }
509 } # CN
510 unshift @$sib, @cn;
511 }
512 }
513 push @$new_todos, {type => 'element', node => $node};
514 } # TP
515
516 for my $new_todo (@$new_todos) {
517 $new_todo->{flag} = {%{$parent_todo->{flag} or {}}};
518 }
519
520 return ($sib, $new_todos);
521 } # _check_get_children
522
523 =head1 LICENSE
524
525 Copyright 2007 Wakaba <w@suika.fam.cx>
526
527 This library is free software; you can redistribute it
528 and/or modify it under the same terms as Perl itself.
529
530 =cut
531
532 1;
533 # $Date: 2007/10/14 09:21:46 $

admin@suikawiki.org
ViewVC Help
Powered by ViewVC 1.1.24