whatpm/t/HTML-tokenizer.t

#!/usr/bin/perl
use strict;

my $dir_name;
my $test_dir_name;
BEGIN {
  $test_dir_name = 't/';
  $dir_name = 't/tokenizer/';
  my $skip = "You don't have JSON module";
  eval q{
         use JSON 1.00;
         $skip = "You don't have make command";
         system ("cd $test_dir_name; make tokenizer-files") == 0 or die
           unless -f $dir_name.'test1.test';
         $skip = '';
        };
  if ($skip) {
    print "1..1\n";
    print "ok 1 # $skip\n";
    exit;
  }
  $JSON::UnMapping = 1;
  $JSON::UTF8 = 1;
}

use Test;
BEGIN { plan tests => 180 }

use Data::Dumper;
$Data::Dumper::Useqq = 1;
sub Data::Dumper::qquote {
  my $s = shift;
  $s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
  return q<qq'> . $s . q<'>;
} # Data::Dumper::qquote

use Whatpm::HTML;

for my $file_name (grep {$_} split /\s+/, qq[
                      ${dir_name}test1.test
                      ${dir_name}test2.test
                      ${dir_name}contentModelFlags.test
                      ${dir_name}escapeFlag.test
                      ${test_dir_name}tokenizer-test-1.test
                     ]) {
  open my $file, '<', $file_name
    or die "$0: $file_name: $!";
  local $/ = undef;
  my $js = <$file>;
  close $file;

  print "# $file_name\n";
  my $tests = jsonToObj ($js)->{tests};
  TEST: for my $test (@$tests) {
    my $s = $test->{input};
    
    my $j = 1;
    while ($j < @{$test->{output}}) {
      if (ref $test->{output}->[$j - 1] and
          $test->{output}->[$j - 1]->[0] eq 'Character' and
          ref $test->{output}->[$j] and 
          $test->{output}->[$j]->[0] eq 'Character') {
        $test->{output}->[$j - 1]->[1]
          .= $test->{output}->[$j]->[1];
        splice @{$test->{output}}, $j, 1;
      }
      $j++;
    }

    my @cm = @{$test->{contentModelFlags} || ['PCDATA']};
    my $last_start_tag = $test->{lastStartTag};
    for my $cm (@cm) {
      my $p = Whatpm::HTML->new;
      my $i = 0;
      my @token;
      $p->{set_next_input_character} = sub {
        my $self = shift;

        pop @{$self->{prev_input_character}};
        unshift @{$self->{prev_input_character}}, $self->{next_input_character};

        $self->{next_input_character} = -1 and return if $i >= length $s;
        $self->{next_input_character} = ord substr $s, $i++, 1;

        if ($self->{next_input_character} == 0x000D) { # CR
          if ($i >= length $s) {
            #
          } else {
            my $next_char = ord substr $s, $i++, 1;
            if ($next_char == 0x000A) { # LF
              #
            } else {
              push @{$self->{char}}, $next_char;
            }
          }
          $self->{next_input_character} = 0x000A; # LF # MUST
        } elsif ($self->{next_input_character} > 0x10FFFF) {
          $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
          push @token, 'ParseError';
        } elsif ($self->{next_input_character} == 0x0000) { # NULL
          $self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
          push @token, 'ParseError';
        }
      };
      $p->{prev_input_character} = [-1, -1, -1];
      $p->{next_input_character} = -1;
      
      $p->{parse_error} = sub {
        push @token, 'ParseError';
      };
      
      $p->_initialize_tokenizer;
      $p->{content_model_flag} = $cm;
      $p->{last_emitted_start_tag_name} = $last_start_tag;

      while (1) {
        my $token = $p->_get_next_token;
        last if $token->{type} eq 'end-of-file';
        
        my $test_token = [
         {
          DOCTYPE => 'DOCTYPE',
          'start tag' => 'StartTag',
          'end tag' => 'EndTag',
          comment => 'Comment',
          character => 'Character',
         }->{$token->{type}} || $token->{type},
        ];
        $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
        $test_token->[1] = $token->{data} if defined $token->{data};
        if ($token->{type} eq 'start tag') {
          $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
        } elsif ($token->{type} eq 'DOCTYPE') {
          $test_token->[1] = $token->{name};
          $test_token->[2] = $token->{public_identifier};
          $test_token->[3] = $token->{system_identifier};
          $test_token->[4] = $token->{correct} ? 1 : 0;
        }

        if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
            $test_token->[0] eq 'Character') {
          $token[-1]->[1] .= $test_token->[1];
        } else {
          push @token, $test_token;
        }
      }
      
      my $expected_dump = Dumper ($test->{output});
      my $parser_dump = Dumper (\@token);
      ok $parser_dump, $expected_dump,
        $test->{description} . ': ' . $test->{input};
    }
  }
}

## $Date: 2007/06/30 13:12:33 $
1	wakaba	1.1	#!/usr/bin/perl
2			use strict;
3
4	wakaba	1.2	my $dir_name;
5	wakaba	1.4	my $test_dir_name;
6	wakaba	1.1	BEGIN {
7	wakaba	1.4	$test_dir_name = 't/';
8	wakaba	1.2	$dir_name = 't/tokenizer/';
9	wakaba	1.1	my $skip = "You don't have JSON module";
10			eval q{
11			use JSON 1.00;
12			$skip = "You don't have make command";
13	wakaba	1.3	system ("cd $test_dir_name; make tokenizer-files") == 0 or die
14	wakaba	1.2	unless -f $dir_name.'test1.test';
15	wakaba	1.1	$skip = '';
16			};
17			if ($skip) {
18			print "1..1\n";
19			print "ok 1 # $skip\n";
20			exit;
21			}
22			$JSON::UnMapping = 1;
23	wakaba	1.2	$JSON::UTF8 = 1;
24	wakaba	1.1	}
25
26			use Test;
27	wakaba	1.13	BEGIN { plan tests => 180 }
28	wakaba	1.2
29	wakaba	1.1	use Data::Dumper;
30	wakaba	1.2	$Data::Dumper::Useqq = 1;
31			sub Data::Dumper::qquote {
32			my $s = shift;
33			$s =~ s/([^\x20\x21-\x26\x28-\x5B\x5D-\x7E])/sprintf '\x{%02X}', ord $1/ge;
34			return q<qq'> . $s . q<'>;
35			} # Data::Dumper::qquote
36	wakaba	1.1
37	wakaba	1.5	use Whatpm::HTML;
38	wakaba	1.1
39	wakaba	1.4	for my $file_name (grep {$_} split /\s+/, qq[
40			${dir_name}test1.test
41			${dir_name}test2.test
42			${dir_name}contentModelFlags.test
43	wakaba	1.8	${dir_name}escapeFlag.test
44	wakaba	1.4	${test_dir_name}tokenizer-test-1.test
45	wakaba	1.1	]) {
46	wakaba	1.4	open my $file, '<', $file_name
47			or die "$0: $file_name: $!";
48	wakaba	1.1	local $/ = undef;
49			my $js = <$file>;
50			close $file;
51	wakaba	1.9
52			print "# $file_name\n";
53	wakaba	1.1	my $tests = jsonToObj ($js)->{tests};
54			TEST: for my $test (@$tests) {
55			my $s = $test->{input};
56
57			my $j = 1;
58			while ($j < @{$test->{output}}) {
59			if (ref $test->{output}->[$j - 1] and
60			$test->{output}->[$j - 1]->[0] eq 'Character' and
61			ref $test->{output}->[$j] and
62			$test->{output}->[$j]->[0] eq 'Character') {
63			$test->{output}->[$j - 1]->[1]
64			.= $test->{output}->[$j]->[1];
65			splice @{$test->{output}}, $j, 1;
66			}
67			$j++;
68			}
69
70	wakaba	1.2	my @cm = @{$test->{contentModelFlags} \|\| ['PCDATA']};
71			my $last_start_tag = $test->{lastStartTag};
72	wakaba	1.1	for my $cm (@cm) {
73	wakaba	1.5	my $p = Whatpm::HTML->new;
74	wakaba	1.1	my $i = 0;
75	wakaba	1.9	my @token;
76	wakaba	1.1	$p->{set_next_input_character} = sub {
77			my $self = shift;
78	wakaba	1.10
79			pop @{$self->{prev_input_character}};
80			unshift @{$self->{prev_input_character}}, $self->{next_input_character};
81
82	wakaba	1.1	$self->{next_input_character} = -1 and return if $i >= length $s;
83			$self->{next_input_character} = ord substr $s, $i++, 1;
84	wakaba	1.2
85			if ($self->{next_input_character} == 0x000D) { # CR
86			if ($i >= length $s) {
87			#
88			} else {
89			my $next_char = ord substr $s, $i++, 1;
90			if ($next_char == 0x000A) { # LF
91			#
92			} else {
93			push @{$self->{char}}, $next_char;
94			}
95			}
96			$self->{next_input_character} = 0x000A; # LF # MUST
97			} elsif ($self->{next_input_character} > 0x10FFFF) {
98			$self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
99	wakaba	1.9	push @token, 'ParseError';
100	wakaba	1.2	} elsif ($self->{next_input_character} == 0x0000) { # NULL
101			$self->{next_input_character} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
102	wakaba	1.9	push @token, 'ParseError';
103	wakaba	1.2	}
104	wakaba	1.1	};
105	wakaba	1.10	$p->{prev_input_character} = [-1, -1, -1];
106			$p->{next_input_character} = -1;
107	wakaba	1.1
108			$p->{parse_error} = sub {
109			push @token, 'ParseError';
110			};
111
112			$p->_initialize_tokenizer;
113			$p->{content_model_flag} = $cm;
114	wakaba	1.2	$p->{last_emitted_start_tag_name} = $last_start_tag;
115	wakaba	1.1
116			while (1) {
117			my $token = $p->_get_next_token;
118			last if $token->{type} eq 'end-of-file';
119
120			my $test_token = [
121			{
122			DOCTYPE => 'DOCTYPE',
123			'start tag' => 'StartTag',
124			'end tag' => 'EndTag',
125			comment => 'Comment',
126			character => 'Character',
127			}->{$token->{type}} \|\| $token->{type},
128			];
129			$test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
130			$test_token->[1] = $token->{data} if defined $token->{data};
131	wakaba	1.11	if ($token->{type} eq 'start tag') {
132			$test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}};
133			} elsif ($token->{type} eq 'DOCTYPE') {
134			$test_token->[1] = $token->{name};
135			$test_token->[2] = $token->{public_identifier};
136			$test_token->[3] = $token->{system_identifier};
137			$test_token->[4] = $token->{correct} ? 1 : 0;
138			}
139
140	wakaba	1.1	if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
141			$test_token->[0] eq 'Character') {
142			$token[-1]->[1] .= $test_token->[1];
143			} else {
144			push @token, $test_token;
145			}
146			}
147
148			my $expected_dump = Dumper ($test->{output});
149			my $parser_dump = Dumper (\@token);
150	wakaba	1.2	ok $parser_dump, $expected_dump,
151			$test->{description} . ': ' . $test->{input};
152	wakaba	1.1	}
153			}
154			}
155
156	wakaba	1.13	## $Date: 2007/06/30 13:12:33 $