#!/usr/bin/perl use strict; use utf8; use lib qw[/home/httpd/html/www/markup/html/whatpm /home/wakaba/work/manakai2/lib]; use CGI::Carp qw[fatalsToBrowser]; use Scalar::Util qw[refaddr]; use Time::HiRes qw/time/; sub htescape ($) { my $s = $_[0]; $s =~ s/&/&/g; $s =~ s//>/g; $s =~ s/"/"/g; $s =~ s{([\x00-\x09\x0B-\x1F\x7F-\xA0\x{FEFF}\x{FFFC}-\x{FFFF}])}{ sprintf 'U+%04X', ord $1; }ge; return $s; } # htescape use Message::CGI::HTTP; my $http = Message::CGI::HTTP->new; if ($http->get_meta_variable ('PATH_INFO') ne '/') { print STDOUT "Status: 404 Not Found\nContent-Type: text/plain; charset=us-ascii\n\n400"; exit; } binmode STDOUT, ':utf8'; $| = 1; require Message::DOM::DOMImplementation; my $dom = Message::DOM::DOMImplementation->new; load_text_catalog ('en'); ## TODO: conneg my @nav; print STDOUT qq[Content-Type: text/html; charset=utf-8 Web Document Conformance Checker (BETA)

Web Document Conformance Checker (beta)

]; $| = 0; my $input = get_input_document ($http, $dom); my $inner_html_element = $http->get_parameter ('e'); my $char_length = 0; my %time; print qq[

Request URI: <@{[htescape $input->{request_uri}]}>
Document URI: <@{[htescape $input->{uri}]}>

yet push @nav, ['#document-info' => 'Information']; if (defined $input->{s}) { $char_length = length $input->{s}; print STDOUT qq[

Base URI

<@{[htescape $input->{base_uri}]}>

Internet Media Type

@{[htescape $input->{media_type}]} @{[$input->{media_type_overridden} ? '(overridden)' : '']}

Character Encoding

@{[defined $input->{charset} ? ''.htescape ($input->{charset}).'' : '(none)']} @{[$input->{charset_overridden} ? '(overridden)' : '']}

Length

$char_length byte@{[$char_length == 1 ? '' : 's']}

]; my $result = {conforming_min => 1, conforming_max => 1}; print_http_header_section ($input, $result); my $doc; my $el; my $manifest; if ($input->{media_type} eq 'text/html') { ($doc, $el) = print_syntax_error_html_section ($input, $result); print_source_string_section (\($input->{s}), $input->{charset}); } elsif ({ 'text/xml' => 1, 'application/atom+xml' => 1, 'application/rss+xml' => 1, 'application/svg+xml' => 1, 'application/xhtml+xml' => 1, 'application/xml' => 1, }->{$input->{media_type}}) { ($doc, $el) = print_syntax_error_xml_section ($input, $result); print_source_string_section (\($input->{s}), $doc->input_encoding); } elsif ($input->{media_type} eq 'text/cache-manifest') { ## TODO: MUST be text/cache-manifest $manifest = print_syntax_error_manifest_section ($input, $result); print_source_string_section (\($input->{s}), 'utf-8'); } else { ## TODO: Change HTTP status code?? print_result_unknown_type_section ($input, $result); } if (defined $doc or defined $el) { print_structure_dump_dom_section ($doc, $el); my $elements = print_structure_error_dom_section ($doc, $el, $result); print_table_section ($elements->{table}) if @{$elements->{table}}; print_id_section ($elements->{id}) if keys %{$elements->{id}}; print_term_section ($elements->{term}) if keys %{$elements->{term}}; print_class_section ($elements->{class}) if keys %{$elements->{class}}; } elsif (defined $manifest) { print_structure_dump_manifest_section ($manifest); print_structure_error_manifest_section ($manifest, $result); } print_result_section ($result); } else { print STDOUT qq[]; print_result_input_error_section ($input); } print STDOUT qq[ ]; for (qw/decode parse parse_html parse_xml parse_manifest check check_manifest/) { next unless defined $time{$_}; open my $file, '>>', ".cc-$_.txt" or die ".cc-$_.txt: $!"; print $file $char_length, "\t", $time{$_}, "\n"; } exit; sub add_error ($$$) { my ($layer, $err, $result) = @_; if (defined $err->{level}) { if ($err->{level} eq 's') { $result->{$layer}->{should}++; $result->{$layer}->{score_min} -= 2; $result->{conforming_min} = 0; } elsif ($err->{level} eq 'w' or $err->{level} eq 'g') { $result->{$layer}->{warning}++; } elsif ($err->{level} eq 'unsupported') { $result->{$layer}->{unsupported}++; $result->{unsupported} = 1; } else { $result->{$layer}->{must}++; $result->{$layer}->{score_max} -= 2; $result->{$layer}->{score_min} -= 2; $result->{conforming_min} = 0; $result->{conforming_max} = 0; } } else { $result->{$layer}->{must}++; $result->{$layer}->{score_max} -= 2; $result->{$layer}->{score_min} -= 2; $result->{conforming_min} = 0; $result->{conforming_max} = 0; } } # add_error sub print_http_header_section ($$) { my ($input, $result) = @_; return unless defined $input->{header_status_code} or defined $input->{header_status_text} or @{$input->{header_field}}; push @nav, ['#source-header' => 'HTTP Header']; print STDOUT qq[

HTTP Header

Note: Due to the limitation of the network library in use, the content of this section might not be the real header.

]; if (defined $input->{header_status_code}) { print STDOUT qq[]; print STDOUT qq[]; } if (defined $input->{header_status_text}) { print STDOUT qq[]; print STDOUT qq[]; } for (@{$input->{header_field}}) { print STDOUT qq[]; print STDOUT qq[]; } print STDOUT qq[

Status code	`@{[htescape ($input->{header_status_code})]}`
Status text	`@{[htescape ($input->{header_status_text})]}`
`@{[htescape ($_->[0])]}`	`@{[htescape ($_->[1])]}`

]; } # print_http_header_section sub print_syntax_error_html_section ($$) { my ($input, $result) = @_; require Encode; require Whatpm::HTML; print STDOUT qq[

Parse Errors

Line $opt{line} column $opt{column}
Line $opt{line}: ], get_error_level_label (\%opt); print STDOUT qq[$msg

]; return ($doc, $el); } # print_syntax_error_html_section sub print_syntax_error_xml_section ($$) { my ($input, $result) = @_; require Message::DOM::XMLParserTemp; print STDOUT qq[

Parse Errors

Line $line column ]; print STDOUT $err->location->column_number, ": "; print STDOUT htescape $err->text, "

]; return ($doc, undef); } # print_syntax_error_xml_section sub print_syntax_error_manifest_section ($$) { my ($input, $result) = @_; require Whatpm::CacheManifest; print STDOUT qq[

Parse Errors

], get_error_label (\%opt), qq[: ], get_error_level_label (\%opt); print STDOUT qq[$msg

]; return $manifest; } # print_syntax_error_manifest_section sub print_source_string_section ($$) { require Encode; my $enc = Encode::find_encoding ($_[1]); ## TODO: charset name -> Perl name return unless $enc; my $s = \($enc->decode (${$_[0]})); my $i = 1; push @nav, ['#source-string' => 'Source']; print STDOUT qq[

Document Source

], htescape $1, "
], htescape $1, "

"; } # print_input_string_section sub print_document_tree ($) { my $node = shift; my $r = '

] . htescape ($child->tag_name) . ''; ## ISSUE: case if ($child->has_attributes) { $r .= '
- ] . htescape ($attr->[0]) . ' = '; ## ISSUE: case? $r .= '' . htescape ($attr->[1]) . '
'; } if ($child->has_child_nodes) { $r .= '
' . htescape ($child->data) . '
<[CDATA[' . htescape ($child->data) . ']]>

Document'; $r .= qq[
- @{[scalar get_text ('manakaiIsHTML:'.($child->manakai_is_html?1:0))]}
- @{[scalar get_text ('manakaiCompatMode:'.$child->manakai_compat_mode)]}
- XML version = @{[htescape ($child->xml_version)]}
- XML encoding = @{[htescape ($child->xml_encoding)]}
- XML encoding = (null)
- XML standalone = @{[$child->xml_standalone ? 'true' : 'false']}
]; if ($child->has_child_nodes) { $r .= '
<!DOCTYPE>
- Name = @{[htescape ($child->name)]}
- Public identifier = @{[htescape ($child->public_id)]}
- System identifier = @{[htescape ($child->system_id)]}
<?@{[htescape ($child->target)]} @{[htescape ($child->data)]}?>
@{[$child->node_type]} @{[htescape ($child->node_name)]}

'; print STDOUT $r; } # print_document_tree sub print_structure_dump_dom_section ($$) { my ($doc, $el) = @_; print STDOUT qq[

Document Tree

]; push @nav, ['#document-tree' => 'Tree']; print_document_tree ($el || $doc); print STDOUT qq[

]; } # print_structure_dump_dom_section sub print_structure_dump_manifest_section ($) { my $manifest = shift; print STDOUT qq[

Cache Manifest

]; push @nav, ['#dump-manifest' => 'Caceh Manifest']; print STDOUT qq[

Explicit entries

<$euri>

Fallback entries

]; for my $uri (sort {$a cmp $b} keys %{$manifest->[1]}) { my $euri = htescape ($uri); my $euri2 = htescape ($manifest->[1]->{$uri}); print STDOUT qq[]; } print STDOUT qq[

Oppotunistic Caching Namespace	Fallback Entry
`<$euri>`	`<$euri2>`

Online whitelist

<$euri>

]; } # print_structure_dump_manifest_section sub print_structure_error_dom_section ($$$) { my ($doc, $el, $result) = @_; print STDOUT qq[

Document Errors

] . get_error_label (\%opt) . qq[: ], get_error_level_label (\%opt); print STDOUT $msg, "

]; return $elements; } # print_structure_error_dom_section sub print_structure_error_manifest_section ($$$) { my ($manifest, $result) = @_; print STDOUT qq[

Document Errors

] . get_error_label (\%opt) . qq[: ], $msg, "

]; } # print_structure_error_manifest_section sub print_table_section ($) { my $tables = shift; push @nav, ['#tables' => 'Tables']; print STDOUT qq[

Tables

]; require JSON; my $i = 0; for my $table_el (@$tables) { $i++; print STDOUT qq[

] . get_node_link ($table_el) . q[

]; ## TODO: Make |ContentChecker| return |form_table| result ## so that this script don't have to run the algorithm twice. my $table = Whatpm::HTMLTable->form_table ($table_el); for (@{$table->{column_group}}, @{$table->{column}}, $table->{caption}) { next unless $_; delete $_->{element}; } for (@{$table->{row_group}}) { next unless $_; next unless $_->{element}; $_->{type} = $_->{element}->manakai_local_name; delete $_->{element}; } for (@{$table->{cell}}) { next unless $_; for (@{$_}) { next unless $_; for (@$_) { $_->{id} = refaddr $_->{element} if defined $_->{element}; delete $_->{element}; $_->{is_header} = $_->{is_header} ? 1 : 0; } } } print STDOUT '

]; } print STDOUT qq[

]; } # print_table_section sub print_id_section ($) { my $ids = shift; push @nav, ['#identifiers' => 'IDs']; print STDOUT qq[

Identifiers

@{[htescape $id]}: ].get_node_link ($_).qq[

]; } # print_id_section sub print_term_section ($) { my $terms = shift; push @nav, ['#terms' => 'Terms']; print STDOUT qq[

Terms

@{[htescape $term]}: ].get_node_link ($_).qq[

]; } # print_term_section sub print_class_section ($) { my $classes = shift; push @nav, ['#classes' => 'Classes']; print STDOUT qq[

Classes

@{[htescape $class]}: ].get_node_link ($_).qq[

]; } # print_class_section sub print_result_section ($) { my $result = shift; print STDOUT qq[

Result

]; if ($result->{unsupported} and $result->{conforming_max}) { print STDOUT qq[

The conformance checker cannot decide whether the document is conforming or not, since the document contains one or more unsupported features. The document might or might not be conforming.

]; } elsif ($result->{conforming_min}) { print STDOUT qq[

No conformance-error is found in this document.

]; } elsif ($result->{conforming_max}) { print STDOUT qq[

This document is likely non-conforming, but in rare case it might be conforming.

]; } else { print STDOUT qq[

This document is non-conforming.

]; } print STDOUT qq[]; my $must_error = 0; my $should_error = 0; my $warning = 0; my $score_min = 0; my $score_max = 0; my $score_base = 20; my $score_unit = $score_base / 100; for ( [Transfer => 'transfer', ''], [Character => 'char', ''], [Syntax => 'syntax', '#parse-errors'], [Structure => 'structure', '#document-errors'], ) { $must_error += ($result->{$_->[1]}->{must} += 0); $should_error += ($result->{$_->[1]}->{should} += 0); $warning += ($result->{$_->[1]}->{warning} += 0); $score_min += (($result->{$_->[1]}->{score_min} *= $score_unit) += $score_base); $score_max += (($result->{$_->[1]}->{score_max} *= $score_unit) += $score_base); my $uncertain = $result->{$_->[1]}->{unsupported} ? '?' : ''; my $label = $_->[0]; if ($result->{$_->[1]}->{must} or $result->{$_->[1]}->{should} or $result->{$_->[1]}->{warning} or $result->{$_->[1]}->{unsupported}) { $label = qq[$label]; } print STDOUT qq[]; if ($uncertain) { print qq[]; } elsif ($result->{$_->[1]}->{score_min} != $result->{$_->[1]}->{score_max}) { print qq[]; } else { print qq[]; } } $score_max += $score_base; print STDOUT qq[

	MUST‐level Errors	SHOULD‐level Errors	Warnings	Score
$label	$result->{$_->[1]}->{must}$uncertain	$result->{$_->[1]}->{should}$uncertain	$result->{$_->[1]}->{warning}$uncertain	−∞..$result->{$_->[1]}->{score_max}	$result->{$_->[1]}->{score_min}..$result->{$_->[1]}->{score_max}
$result->{$_->[1]}->{score_min}
Semantics	0?	0?	0?	−∞..$score_base
Total	$must_error?	$should_error?	$warning?	−∞..$score_max

Important: This conformance checking service is under development. The result above might be wrong.

]; push @nav, ['#result-summary' => 'Result']; } # print_result_section sub print_result_unknown_type_section ($$) { my ($input, $result) = @_; my $euri = htescape ($input->{uri}); print STDOUT qq[

Errors

<$euri>: Not supported: Media type @{[htescape $input->{media_type}]} is not supported.

]; push @nav, ['#parse-errors' => 'Errors']; add_error (char => {level => 'unsupported'} => $result); add_error (syntax => {level => 'unsupported'} => $result); add_error (structure => {level => 'unsupported'} => $result); } # print_result_unknown_type_section sub print_result_input_error_section ($) { my $input = shift; print STDOUT qq[

Input Error: @{[htescape ($input->{error_status_text})]}

]; push @nav, ['#result-summary' => 'Result']; } # print_Result_input_error_section sub get_error_label ($) { my $err = shift; my $r = ''; if (defined $err->{line}) { if ($err->{column} > 0) { $r = qq[Line $err->{line} column $err->{column}]; } else { $err->{line} = $err->{line} - 1 || 1; $r = qq[Line $err->{line}]; } } if (defined $err->{node}) { $r .= ' ' if length $r; $r = get_node_link ($err->{node}); } if (defined $err->{index}) { $r .= ' ' if length $r; $r .= 'Index ' . (0+$err->{index}); } if (defined $err->{value}) { $r .= ' ' if length $r; $r .= '' . htescape ($err->{value}) . ''; } return $r; } # get_error_label sub get_error_level_label ($) { my $err = shift; my $r = ''; if (not defined $err->{level} or $err->{level} eq 'm') { $r = qq[MUST‐level error: ]; } elsif ($err->{level} eq 's') { $r = qq[SHOULD‐level error: ]; } elsif ($err->{level} eq 'w') { $r = qq[Warning: ]; } elsif ($err->{level} eq 'unsupported') { $r = qq[Not supported: ]; } else { my $elevel = htescape ($err->{level}); $r = qq[$elevel: ]; } return $r; } # get_error_level_label sub get_node_path ($) { my $node = shift; my @r; while (defined $node) { my $rs; if ($node->node_type == 1) { $rs = $node->manakai_local_name; $node = $node->parent_node; } elsif ($node->node_type == 2) { $rs = '@' . $node->manakai_local_name; $node = $node->owner_element; } elsif ($node->node_type == 3) { $rs = '"' . $node->data . '"'; $node = $node->parent_node; } elsif ($node->node_type == 9) { @r = ('') unless @r; $rs = ''; $node = $node->parent_node; } else { $rs = '#' . $node->node_type; $node = $node->parent_node; } unshift @r, $rs; } return join '/', @r; } # get_node_path sub get_node_link ($) { return qq[] . htescape (get_node_path ($_[0])) . qq[]; } # get_node_link { my $Msg = {}; sub load_text_catalog ($) { my $lang = shift; # MUST be a canonical lang name open my $file, '<', "cc-msg.$lang.txt" or die "$0: cc-msg.$lang.txt: $!"; while (<$file>) { if (s/^([^;]+);([^;]*);//) { my ($type, $cls, $msg) = ($1, $2, $_); $msg =~ tr/\x0D\x0A//d; $Msg->{$type} = [$cls, $msg]; } } } # load_text_catalog sub get_text ($) { my ($type, $level, $node) = @_; $type = $level . ':' . $type if defined $level; my @arg; { if (defined $Msg->{$type}) { my $msg = $Msg->{$type}->[1]; $msg =~ s{\$([0-9]+)}{ defined $arg[$1] ? htescape ($arg[$1]) : '(undef)'; }ge; $msg =~ s{{\@([A-Za-z0-9:_.-]+)}}{ UNIVERSAL::can ($node, 'get_attribute_ns') ? htescape ($node->get_attribute_ns (undef, $1)) : '' }ge; $msg =~ s{{\@}}{ UNIVERSAL::can ($node, 'value') ? htescape ($node->value) : '' }ge; $msg =~ s{{local-name}}{ UNIVERSAL::can ($node, 'manakai_local_name') ? htescape ($node->manakai_local_name) : '' }ge; $msg =~ s{{element-local-name}}{ (UNIVERSAL::can ($node, 'owner_element') and $node->owner_element) ? htescape ($node->owner_element->manakai_local_name) : '' }ge; return ($type, $Msg->{$type}->[0], $msg); } elsif ($type =~ s/:([^:]*)$//) { unshift @arg, $1; redo; } } return ($type, '', htescape ($_[0])); } # get_text } sub get_input_document ($$) { my ($http, $dom) = @_; my $request_uri = $http->get_parameter ('uri'); my $r = {}; if (defined $request_uri and length $request_uri) { my $uri = $dom->create_uri_reference ($request_uri); unless ({ http => 1, }->{lc $uri->uri_scheme}) { return {uri => $request_uri, request_uri => $request_uri, error_status_text => 'URI scheme not allowed'}; } require Message::Util::HostPermit; my $host_permit = new Message::Util::HostPermit; $host_permit->add_rule (<check ($uri->uri_host, $uri->uri_port || 80)) { return {uri => $request_uri, request_uri => $request_uri, error_status_text => 'Connection to the host is forbidden'}; } require LWP::UserAgent; my $ua = WDCC::LWPUA->new; $ua->{wdcc_dom} = $dom; $ua->{wdcc_host_permit} = $host_permit; $ua->agent ('Mozilla'); ## TODO: for now. $ua->parse_head (0); $ua->protocols_allowed ([qw/http/]); $ua->max_size (1000_000); my $req = HTTP::Request->new (GET => $request_uri); my $res = $ua->request ($req); ## TODO: 401 sets |is_success| true. if ($res->is_success or $http->get_parameter ('error-page')) { $r->{base_uri} = $res->base; ## NOTE: It does check |Content-Base|, |Content-Location|, and . ## TODO: Use our own code! $r->{uri} = $res->request->uri; $r->{request_uri} = $request_uri; ## TODO: More strict parsing... my $ct = $res->header ('Content-Type'); if (defined $ct and $ct =~ m#^([0-9A-Za-z._+-]+/[0-9A-Za-z._+-]+)#) { $r->{media_type} = lc $1; } if (defined $ct and $ct =~ /;\s*charset\s*=\s*"?([^\s;"]+)"?/i) { $r->{charset} = lc $1; $r->{charset} =~ tr/\\//d; } my $input_charset = $http->get_parameter ('charset'); if (defined $input_charset and length $input_charset) { $r->{charset_overridden} = (not defined $r->{charset} or $r->{charset} ne $input_charset); $r->{charset} = $input_charset; } $r->{s} = ''.$res->content; } else { $r->{uri} = $res->request->uri; $r->{request_uri} = $request_uri; $r->{error_status_text} = $res->status_line; } $r->{header_field} = []; $res->scan (sub { push @{$r->{header_field}}, [$_[0], $_[1]]; }); $r->{header_status_code} = $res->code; $r->{header_status_text} = $res->message; } else { $r->{s} = ''.$http->get_parameter ('s'); $r->{uri} = q; $r->{request_uri} = q; $r->{base_uri} = q; $r->{charset} = ''.$http->get_parameter ('_charset_'); $r->{charset} =~ s/\s+//g; $r->{charset} = 'utf-8' if $r->{charset} eq ''; $r->{header_field} = []; } my $input_format = $http->get_parameter ('i'); if (defined $input_format and length $input_format) { $r->{media_type_overridden} = (not defined $r->{media_type} or $input_format ne $r->{media_type}); $r->{media_type} = $input_format; } if (defined $r->{s} and not defined $r->{media_type}) { $r->{media_type} = 'text/html'; $r->{media_type_overridden} = 1; } if ($r->{media_type} eq 'text/xml') { unless (defined $r->{charset}) { $r->{charset} = 'us-ascii'; } elsif ($r->{charset_overridden} and $r->{charset} eq 'us-ascii') { $r->{charset_overridden} = 0; } } if (length $r->{s} > 1000_000) { $r->{error_status_text} = 'Entity-body too large'; delete $r->{s}; return $r; } return $r; } # get_input_document package WDCC::LWPUA; BEGIN { push our @ISA, 'LWP::UserAgent'; } sub redirect_ok { my $ua = shift; unless ($ua->SUPER::redirect_ok (@_)) { return 0; } my $uris = $_[1]->header ('Location'); return 0 unless $uris; my $uri = $ua->{wdcc_dom}->create_uri_reference ($uris); unless ({ http => 1, }->{lc $uri->uri_scheme}) { return 0; } unless ($ua->{wdcc_host_permit}->check ($uri->uri_host, $uri->uri_port || 80)) { return 0; } return 1; } # redirect_ok =head1 AUTHOR Wakaba . =head1 LICENSE Copyright 2007 Wakaba This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut ## $Date: 2007/11/11 06:57:16 $