whatpm/t/HTML-tokenizer.t

#!/usr/bin/perl
use strict;

BEGIN {
  my $skip = "You don't have JSON module";
  eval q{
         use JSON 1.00;
         $skip = "You don't have make command";
         system ('make', 'tokenizer-files') == 0 or die
           unless -f 'tokenizer/test1.test';
         $skip = '';
        };
  if ($skip) {
    print "1..1\n";
    print "ok 1 # $skip\n";
    exit;
  }
  $JSON::UnMapping = 1;
}

use Test;
use Data::Dumper;
BEGIN { plan tests => 38 }

use What::HTML;

for my $file_name (qw[
                      tokenizer/test1.test
                      tokenizer/test2.test
                      tokenizer/contentModelFlags.test
                     ]) {
  open my $file, '<:utf8', $file_name or die "$0: $file_name: $!";
  local $/ = undef;
  my $js = <$file>;
  close $file;
  
  my $tests = jsonToObj ($js)->{tests};
  TEST: for my $test (@$tests) {
    my $s = $test->{input};
    
    my $j = 1;
    while ($j < @{$test->{output}}) {
      if (ref $test->{output}->[$j - 1] and
          $test->{output}->[$j - 1]->[0] eq 'Character' and
          ref $test->{output}->[$j] and 
          $test->{output}->[$j]->[0] eq 'Character') {
        $test->{output}->[$j - 1]->[1]
          .= $test->{output}->[$j]->[1];
        splice @{$test->{output}}, $j, 1;
      }
      $j++;
    }

    my @cm = @{$test->{content_model_flags} || ['PCDATA']};
    for my $cm (@cm) {
      my $p = What::HTML->new;
      my $i = 0;
      $p->{set_next_input_character} = sub {
        my $self = shift;
        $self->{next_input_character} = -1 and return if $i >= length $s;
        $self->{next_input_character} = ord substr $s, $i++, 1;
      };
      
      my @token;
      $p->{parse_error} = sub {
        push @token, 'ParseError';
      };
      
      $p->_initialize_tokenizer;
      $p->{content_model_flag} = $cm;

      while (1) {
        my $token = $p->_get_next_token;
        last if $token->{type} eq 'end-of-file';
        
        my $test_token = [
         {
          DOCTYPE => 'DOCTYPE',
          'start tag' => 'StartTag',
          'end tag' => 'EndTag',
          comment => 'Comment',
          character => 'Character',
         }->{$token->{type}} || $token->{type},
        ];
        $test_token->[1] = $token->{name} if defined $token->{name};
        $test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
        $test_token->[1] = $token->{data} if defined $token->{data};
        $test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';
        $test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}
          if $token->{type} eq 'start tag';
        
        if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
            $test_token->[0] eq 'Character') {
          $token[-1]->[1] .= $test_token->[1];
        } else {
          push @token, $test_token;
        }
      }
      
      my $expected_dump = Dumper ($test->{output});
      my $parser_dump = Dumper (\@token);
      ok $parser_dump, $expected_dump, $test->{description};
    }
  }
}

1	wakaba	1.1	#!/usr/bin/perl
2			use strict;
3
4			BEGIN {
5			my $skip = "You don't have JSON module";
6			eval q{
7			use JSON 1.00;
8			$skip = "You don't have make command";
9			system ('make', 'tokenizer-files') == 0 or die
10			unless -f 'tokenizer/test1.test';
11			$skip = '';
12			};
13			if ($skip) {
14			print "1..1\n";
15			print "ok 1 # $skip\n";
16			exit;
17			}
18			$JSON::UnMapping = 1;
19			}
20
21			use Test;
22			use Data::Dumper;
23			BEGIN { plan tests => 38 }
24
25			use What::HTML;
26
27			for my $file_name (qw[
28			tokenizer/test1.test
29			tokenizer/test2.test
30			tokenizer/contentModelFlags.test
31			]) {
32			open my $file, '<:utf8', $file_name or die "$0: $file_name: $!";
33			local $/ = undef;
34			my $js = <$file>;
35			close $file;
36
37			my $tests = jsonToObj ($js)->{tests};
38			TEST: for my $test (@$tests) {
39			my $s = $test->{input};
40
41			my $j = 1;
42			while ($j < @{$test->{output}}) {
43			if (ref $test->{output}->[$j - 1] and
44			$test->{output}->[$j - 1]->[0] eq 'Character' and
45			ref $test->{output}->[$j] and
46			$test->{output}->[$j]->[0] eq 'Character') {
47			$test->{output}->[$j - 1]->[1]
48			.= $test->{output}->[$j]->[1];
49			splice @{$test->{output}}, $j, 1;
50			}
51			$j++;
52			}
53
54			my @cm = @{$test->{content_model_flags} \|\| ['PCDATA']};
55			for my $cm (@cm) {
56			my $p = What::HTML->new;
57			my $i = 0;
58			$p->{set_next_input_character} = sub {
59			my $self = shift;
60			$self->{next_input_character} = -1 and return if $i >= length $s;
61			$self->{next_input_character} = ord substr $s, $i++, 1;
62			};
63
64			my @token;
65			$p->{parse_error} = sub {
66			push @token, 'ParseError';
67			};
68
69			$p->_initialize_tokenizer;
70			$p->{content_model_flag} = $cm;
71
72			while (1) {
73			my $token = $p->_get_next_token;
74			last if $token->{type} eq 'end-of-file';
75
76			my $test_token = [
77			{
78			DOCTYPE => 'DOCTYPE',
79			'start tag' => 'StartTag',
80			'end tag' => 'EndTag',
81			comment => 'Comment',
82			character => 'Character',
83			}->{$token->{type}} \|\| $token->{type},
84			];
85			$test_token->[1] = $token->{name} if defined $token->{name};
86			$test_token->[1] = $token->{tag_name} if defined $token->{tag_name};
87			$test_token->[1] = $token->{data} if defined $token->{data};
88			$test_token->[2] = $token->{error} ? 1 : 0 if $token->{type} eq 'DOCTYPE';
89			$test_token->[2] = {map {$_->{name} => $_->{value}} values %{$token->{attributes}}}
90			if $token->{type} eq 'start tag';
91
92			if (@token and ref $token[-1] and $token[-1]->[0] eq 'Character' and
93			$test_token->[0] eq 'Character') {
94			$token[-1]->[1] .= $test_token->[1];
95			} else {
96			push @token, $test_token;
97			}
98			}
99
100			my $expected_dump = Dumper ($test->{output});
101			my $parser_dump = Dumper (\@token);
102			ok $parser_dump, $expected_dump, $test->{description};
103			}
104			}
105			}
106