Whatpm/XML/Parser.pm

package Whatpm::XML::Parser;
use strict;

push our @ISA, 'Whatpm::HTML';
use Whatpm::HTML::Tokenizer qw/:token/;

sub parse_char_string ($$$;$$) {
  #my ($self, $s, $doc, $onerror, $get_wrapper) = @_;
  my $self = shift;
  my $s = ref $_[0] ? $_[0] : \($_[0]);
  require Whatpm::Charset::DecodeHandle;
  my $input = Whatpm::Charset::DecodeHandle::CharString->new ($s);
  return $self->parse_char_stream ($input, @_[1..$#_]);
} # parse_char_string

sub parse_char_stream ($$$;$$) {
  my $self = ref $_[0] ? shift : shift->new;
  my $input = $_[0];
  $self->{document} = $_[1];
  @{$self->{document}->child_nodes} = ();

  ## NOTE: |set_inner_html| copies most of this method's code

  $self->{confident} = 1 unless exists $self->{confident};
  $self->{document}->input_encoding ($self->{input_encoding})
      if defined $self->{input_encoding};
## TODO: |{input_encoding}| is needless?

  $self->{line_prev} = $self->{line} = 1;
  $self->{column_prev} = -1;
  $self->{column} = 0;
  $self->{set_nc} = sub {
    my $self = shift;

    my $char = '';
    if (defined $self->{next_nc}) {
      $char = $self->{next_nc};
      delete $self->{next_nc};
      $self->{nc} = ord $char;
    } else {
      $self->{char_buffer} = '';
      $self->{char_buffer_pos} = 0;

      my $count = $input->manakai_read_until
         ($self->{char_buffer}, qr/[^\x00\x0A\x0D]/, $self->{char_buffer_pos});
      if ($count) {
        $self->{line_prev} = $self->{line};
        $self->{column_prev} = $self->{column};
        $self->{column}++;
        $self->{nc}
            = ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
        return;
      }

      if ($input->read ($char, 1)) {
        $self->{nc} = ord $char;
      } else {
        $self->{nc} = -1;
        return;
      }
    }

    ($self->{line_prev}, $self->{column_prev})
        = ($self->{line}, $self->{column});
    $self->{column}++;
    
    if ($self->{nc} == 0x000A) { # LF
      
      $self->{line}++;
      $self->{column} = 0;
    } elsif ($self->{nc} == 0x000D) { # CR
      
## TODO: support for abort/streaming
      my $next = '';
      if ($input->read ($next, 1) and $next ne "\x0A") {
        $self->{next_nc} = $next;
      }
      $self->{nc} = 0x000A; # LF # MUST
      $self->{line}++;
      $self->{column} = 0;
    } elsif ($self->{nc} == 0x0000) { # NULL
      
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'NULL');
      $self->{nc} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
    }
  };

  $self->{read_until} = sub {
    #my ($scalar, $specials_range, $offset) = @_;
    return 0 if defined $self->{next_nc};

    my $pattern = qr/[^$_[1]\x00\x0A\x0D]/;
    my $offset = $_[2] || 0;

    if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
      pos ($self->{char_buffer}) = $self->{char_buffer_pos};
      if ($self->{char_buffer} =~ /\G(?>$pattern)+/) {
        substr ($_[0], $offset)
            = substr ($self->{char_buffer}, $-[0], $+[0] - $-[0]);
        my $count = $+[0] - $-[0];
        if ($count) {
          $self->{column} += $count;
          $self->{char_buffer_pos} += $count;
          $self->{line_prev} = $self->{line};
          $self->{column_prev} = $self->{column} - 1;
          $self->{nc} = -1;
        }
        return $count;
      } else {
        return 0;
      }
    } else {
      my $count = $input->manakai_read_until ($_[0], $pattern, $_[2]);
      if ($count) {
        $self->{column} += $count;
        $self->{line_prev} = $self->{line};
        $self->{column_prev} = $self->{column} - 1;
        $self->{nc} = -1;
      }
      return $count;
    }
  }; # $self->{read_until}

  my $onerror = $_[2] || sub {
    my (%opt) = @_;
    my $line = $opt{token} ? $opt{token}->{line} : $opt{line};
    my $column = $opt{token} ? $opt{token}->{column} : $opt{column};
    warn "Parse error ($opt{type}) at line $line column $column\n";
  };
  $self->{parse_error} = sub {
    $onerror->(line => $self->{line}, column => $self->{column}, @_);
  };

  my $char_onerror = sub {
    my (undef, $type, %opt) = @_;
    $self->{parse_error}->(level => $self->{level}->{must}, layer => 'encode',
                    line => $self->{line}, column => $self->{column} + 1,
                    %opt, type => $type);
  }; # $char_onerror

  if ($_[3]) {
    $input = $_[3]->($input);
    $input->onerror ($char_onerror);
  } else {
    $input->onerror ($char_onerror) unless defined $input->onerror;
  }

  $self->_initialize_tokenizer;
  $self->_initialize_tree_constructor;
  $self->_construct_tree;
  $self->_terminate_tree_constructor;

  delete $self->{parse_error}; # remove loop

  return $self->{document};
} # parse_char_stream

sub new ($) {
  my $class = shift;
  my $self = bless {
    level => {must => 'm',
              should => 's',
              warn => 'w',
              info => 'i',
              uncertain => 'u'},
  }, $class;
  $self->{set_nc} = sub {
    $self->{nc} = -1;
  };
  $self->{parse_error} = sub {
    # 
  };
  $self->{change_encoding} = sub {
    # if ($_[0] is a supported encoding) {
    #   run "change the encoding" algorithm;
    #   throw Whatpm::HTML::RestartParser (charset => $new_encoding);
    # }
  };
  $self->{application_cache_selection} = sub {
    #
  };

  $self->{is_xml} = 1;

  return $self;
} # new

sub _initialize_tree_constructor ($) {
  my $self = shift;
  ## NOTE: $self->{document} MUST be specified before this method is called
  $self->{document}->strict_error_checking (0);
  ## TODO: Turn mutation events off # MUST
  $self->{document}->dom_config
      ->{'http://suika.fam.cx/www/2006/dom-config/strict-document-children'}
      = 0;
  $self->{document}->manakai_is_html (0);
  $self->{document}->set_user_data (manakai_source_line => 1);
  $self->{document}->set_user_data (manakai_source_column => 1);
} # _initialize_tree_constructor

sub _terminate_tree_constructor ($) {
  my $self = shift;
  $self->{document}->strict_error_checking (1);
  $self->{document}->dom_config
      ->{'http://suika.fam.cx/www/2006/dom-config/strict-document-children'}
      = 1;
  ## TODO: Turn mutation events on
} # _terminate_tree_constructor

## Tree construction stage


## NOTE: Differences from the XML5 draft are marked as "XML5:".

## XML5: No namespace support.

## XML5: XML5 has "empty tag token".  In this implementation, it is
## represented as a start tag token with $self->{self_closing} flag
## set to true.

## XML5: XML5 has "short end tag token".  In this implementation, it
## is represented as an end tag token with $token->{tag_name} flag set
## to an empty string.

## XML5: Start, main, end phases.  In this implementation, they are
## represented by insertion modes.

## Insertion modes
sub INITIAL_IM () { 0 }
sub BEFORE_ROOT_ELEMENT_IM () { 1 }
sub IN_ELEMENT_IM () { 2 }
sub AFTER_ROOT_ELEMENT_IM () { 3 }

{
my $token; ## TODO: change to $self->{t}

sub _construct_tree ($) {
  my ($self) = @_;

  $token = $self->_get_next_token;

  delete $self->{tainted};
  $self->{open_elements} = [];
  $self->{insertion_mode} = INITIAL_IM;
  
  while (1) {
    if ($self->{insertion_mode} == IN_ELEMENT_IM) {
      $self->_tree_in_element;
    } elsif ($self->{insertion_mode} == AFTER_ROOT_ELEMENT_IM) {
      $self->_tree_after_root_element;
    } elsif ($self->{insertion_mode} == BEFORE_ROOT_ELEMENT_IM) {
      $self->_tree_before_root_element;
    } elsif ($self->{insertion_mode} == INITIAL_IM) {
      $self->_tree_initial;
    } else {
      die "$0: Unknown XML insertion mode: $self->{insertion_mode}";
    }

    last if $token->{type} == ABORT_TOKEN;
  }
} # _construct_tree

sub _tree_initial ($) {
  my $self = shift;

  B: while (1) {
    if ($token->{type} == DOCTYPE_TOKEN) {
      ## XML5: No "DOCTYPE" token.
      
      my $doctype = $self->{document}->create_document_type_definition
          (defined $token->{name} ? $token->{name} : '');
      
      ## NOTE: Default value for both |public_id| and |system_id| attributes
      ## are empty strings, so that we don't set any value in missing cases.
      $doctype->public_id ($token->{public_identifier})
          if defined $token->{public_identifier};
      $doctype->system_id ($token->{system_identifier})
          if defined $token->{system_identifier};
      
      ## TODO: internal_subset
      
      $self->{document}->append_child ($doctype);
      
      $self->{insertion_mode} = BEFORE_ROOT_ELEMENT_IM;
      $token = $self->_get_next_token;
      return;
    } elsif ($token->{type} == START_TAG_TOKEN or
             $token->{type} == END_OF_FILE_TOKEN) {
      $self->{insertion_mode} = BEFORE_ROOT_ELEMENT_IM;
      ## Reprocess.
      return;
    } elsif ($token->{type} == COMMENT_TOKEN) {
      my $comment = $self->{document}->create_comment ($token->{data});
      $self->{document}->append_child ($comment);
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == PI_TOKEN) {
      my $pi = $self->{document}->create_processing_instruction
          ($token->{target}, $token->{data});
      $self->{document}->append_child ($pi);

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == CHARACTER_TOKEN) {
      if (not $self->{tainted} and
          $token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
        #
      }
      
      if (length $token->{data}) {
        ## XML5: Ignore the token.

        unless ($self->{tainted}) {
          $self->{parse_error}->(level => $self->{level}->{must}, type => 'text outside of root element',
                          token => $token);
          $self->{tainted} = 1;
        }

        $self->{document}->manakai_append_text ($token->{data});
      }

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == END_TAG_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
                      text => $token->{tag_name},
                      token => $token);
      ## Ignore the token.
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == ABORT_TOKEN) {
      return;
    } else {
      die "$0: XML parser initial: Unknown token type $token->{type}";
    }
  } # B
} # _tree_initial

sub _tree_before_root_element ($) {
  my $self = shift;

  B: while (1) {
    if ($token->{type} == START_TAG_TOKEN) {
      my ($prefix, $ln) = split /:/, $token->{tag_name}, 2;
      ($prefix, $ln) = (undef, $prefix) unless defined $ln;
      my $ns; ## TODO:
      my $el = $self->{document}->create_element_ns ($ns, [$prefix, $ln]);
      $self->{document}->append_child ($el);

      if ($self->{self_closing}) {
        delete $self->{self_closing};
        $self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
      } else {
        push @{$self->{open_elements}}, [$el, $token->{tag_name}];
        $self->{insertion_mode} = IN_ELEMENT_IM;
      }

      #delete $self->{tainted};

      $token = $self->_get_next_token;
      return;
    } elsif ($token->{type} == COMMENT_TOKEN) {
      my $comment = $self->{document}->create_comment ($token->{data});
      $self->{document}->append_child ($comment);
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == PI_TOKEN) {
      my $pi = $self->{document}->create_processing_instruction
          ($token->{target}, $token->{data});
      $self->{document}->append_child ($pi);

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == CHARACTER_TOKEN) {
      if (not $self->{tainted} and
          $token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
        #
      }
      
      if (length $token->{data}) {
        ## XML5: Ignore the token.

        unless ($self->{tainted}) {
          $self->{parse_error}->(level => $self->{level}->{must}, type => 'text outside of root element',
                          token => $token);
          $self->{tainted} = 1;
        }

        $self->{document}->manakai_append_text ($token->{data});
      }

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == END_OF_FILE_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'no root element',
                      token => $token);
      
      $self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
      ## Reprocess.
      return;
    } elsif ($token->{type} == END_TAG_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
                      text => $token->{tag_name},
                      token => $token);
      ## Ignore the token.

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == DOCTYPE_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'in html:#doctype',
                      token => $token);
      ## Ignore the token.
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == ABORT_TOKEN) {
      return;
    } else {
      die "$0: XML parser initial: Unknown token type $token->{type}";
    }
  } # B
} # _tree_before_root_element

sub _tree_in_element ($) {
  my $self = shift;
  
  B: while (1) {
    if ($token->{type} == CHARACTER_TOKEN) {
      $self->{open_elements}->[-1]->[0]->manakai_append_text ($token->{data});
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == START_TAG_TOKEN) {
      my ($prefix, $ln) = split /:/, $token->{tag_name}, 2;
      ($prefix, $ln) = (undef, $prefix) unless defined $ln;
      my $ns; ## TODO:
      my $el = $self->{document}->create_element_ns ($ns, [$prefix, $ln]);
      $self->{open_elements}->[-1]->[0]->append_child ($el);

      if ($self->{self_closing}) {
        delete $self->{self_closing};
      } else {
        push @{$self->{open_elements}}, [$el, $token->{tag_name}];
      }
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == END_TAG_TOKEN) {
      if ($token->{tag_name} eq '') {
        ## Short end tag token.
        pop @{$self->{open_elements}};
      } elsif ($self->{open_elements}->[-1]->[1] eq $token->{tag_name}) {
        pop @{$self->{open_elements}};
      } else {
        $self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
                        text => $token->{tag_name},
                        token => $token);
        
        ## Has an element in scope
        INSCOPE: for my $i (reverse 0..$#{$self->{open_elements}}) {
          if ($self->{open_elements}->[$i]->[1] eq $token->{tag_name}) {
            splice @{$self->{open_elements}}, $i;
            last INSCOPE;
          }
        } # INSCOPE
      }
      
      unless (@{$self->{open_elements}}) {
        $self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
        $token = $self->_get_next_token;
        return;
      } else {
        ## Stay in the state.
        $token = $self->_get_next_token;
        redo B;
      }
    } elsif ($token->{type} == COMMENT_TOKEN) {
      my $comment = $self->{document}->create_comment ($token->{data});
      $self->{open_elements}->[-1]->[0]->append_child ($comment);
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == PI_TOKEN) {
      my $pi = $self->{document}->create_processing_instruction
          ($token->{target}, $token->{data});
      $self->{open_elements}->[-1]->[0]->append_child ($pi);

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == END_OF_FILE_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'in body:#eof',
                      token => $token);
      
      $self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
      $token = $self->_get_next_token;
      return;
    } elsif ($token->{type} == DOCTYPE_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'in html:#doctype',
                      token => $token);
      ## Ignore the token.
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == ABORT_TOKEN) {
      return;
    } else {
      die "$0: XML parser initial: Unknown token type $token->{type}";
    }
  } # B
} # _tree_in_element

sub _tree_after_root_element ($) {
  my $self = shift;

  B: while (1) {
    if ($token->{type} == START_TAG_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'second root element',
                      token => $token);

      ## XML5: Ignore the token.
      
      my ($prefix, $ln) = split /:/, $token->{tag_name}, 2;
      ($prefix, $ln) = (undef, $prefix) unless defined $ln;
      my $ns; ## TODO:
      my $el = $self->{document}->create_element_ns ($ns, [$prefix, $ln]);
      $self->{document}->append_child ($el);
      
      if ($self->{self_closing}) {
        delete $self->{self_closing};
        ## Stay in the mode.
      } else {
        push @{$self->{open_elements}}, [$el, $token->{tag_name}];
        $self->{insertion_mode} = IN_ELEMENT_IM;
      }
      
      #delete $self->{tainted};

      $token = $self->_get_next_token;
      return;
    } elsif ($token->{type} == COMMENT_TOKEN) {
      my $comment = $self->{document}->create_comment ($token->{data});
      $self->{document}->append_child ($comment);
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == PI_TOKEN) {
      my $pi = $self->{document}->create_processing_instruction
          ($token->{target}, $token->{data});
      $self->{document}->append_child ($pi);

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == CHARACTER_TOKEN) {
      if (not $self->{tainted} and
          $token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
        #
      }
      
      if (length $token->{data}) {
        ## XML5: Ignore the token.

        unless ($self->{tainted}) {
          $self->{parse_error}->(level => $self->{level}->{must}, type => 'text outside of root element',
                          token => $token);
          $self->{tainted} = 1;
        }

        $self->{document}->manakai_append_text ($token->{data});
      }

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == END_OF_FILE_TOKEN) {
      ## Stop parsing.

      ## TODO: implement "stop parsing".

      $token = {type => ABORT_TOKEN};
      return;
    } elsif ($token->{type} == END_TAG_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
                      text => $token->{tag_name},
                      token => $token);
      ## Ignore the token.

      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == DOCTYPE_TOKEN) {
      $self->{parse_error}->(level => $self->{level}->{must}, type => 'in html:#doctype',
                      token => $token);
      ## Ignore the token.
      
      ## Stay in the mode.
      $token = $self->_get_next_token;
      next B;
    } elsif ($token->{type} == ABORT_TOKEN) {
      return;
    } else {
      die "$0: XML parser initial: Unknown token type $token->{type}";
    }
  } # B
} # _tree_after_root_element

}

1;
1	package Whatpm::XML::Parser;
2	use strict;
3
4	push our @ISA, 'Whatpm::HTML';
5	use Whatpm::HTML::Tokenizer qw/:token/;
6
7	sub parse_char_string ($$$;$$) {
8	#my ($self, $s, $doc, $onerror, $get_wrapper) = @_;
9	my $self = shift;
10	my $s = ref $_[0] ? $_[0] : \($_[0]);
11	require Whatpm::Charset::DecodeHandle;
12	my $input = Whatpm::Charset::DecodeHandle::CharString->new ($s);
13	return $self->parse_char_stream ($input, @_[1..$#_]);
14	} # parse_char_string
15
16	sub parse_char_stream ($$$;$$) {
17	my $self = ref $_[0] ? shift : shift->new;
18	my $input = $_[0];
19	$self->{document} = $_[1];
20	@{$self->{document}->child_nodes} = ();
21
22	## NOTE: \|set_inner_html\| copies most of this method's code
23
24	$self->{confident} = 1 unless exists $self->{confident};
25	$self->{document}->input_encoding ($self->{input_encoding})
26	if defined $self->{input_encoding};
27	## TODO: \|{input_encoding}\| is needless?
28
29	$self->{line_prev} = $self->{line} = 1;
30	$self->{column_prev} = -1;
31	$self->{column} = 0;
32	$self->{set_nc} = sub {
33	my $self = shift;
34
35	my $char = '';
36	if (defined $self->{next_nc}) {
37	$char = $self->{next_nc};
38	delete $self->{next_nc};
39	$self->{nc} = ord $char;
40	} else {
41	$self->{char_buffer} = '';
42	$self->{char_buffer_pos} = 0;
43
44	my $count = $input->manakai_read_until
45	($self->{char_buffer}, qr/[^\x00\x0A\x0D]/, $self->{char_buffer_pos});
46	if ($count) {
47	$self->{line_prev} = $self->{line};
48	$self->{column_prev} = $self->{column};
49	$self->{column}++;
50	$self->{nc}
51	= ord substr ($self->{char_buffer}, $self->{char_buffer_pos}++, 1);
52	return;
53	}
54
55	if ($input->read ($char, 1)) {
56	$self->{nc} = ord $char;
57	} else {
58	$self->{nc} = -1;
59	return;
60	}
61	}
62
63	($self->{line_prev}, $self->{column_prev})
64	= ($self->{line}, $self->{column});
65	$self->{column}++;
66
67	if ($self->{nc} == 0x000A) { # LF
68
69	$self->{line}++;
70	$self->{column} = 0;
71	} elsif ($self->{nc} == 0x000D) { # CR
72
73	## TODO: support for abort/streaming
74	my $next = '';
75	if ($input->read ($next, 1) and $next ne "\x0A") {
76	$self->{next_nc} = $next;
77	}
78	$self->{nc} = 0x000A; # LF # MUST
79	$self->{line}++;
80	$self->{column} = 0;
81	} elsif ($self->{nc} == 0x0000) { # NULL
82
83	$self->{parse_error}->(level => $self->{level}->{must}, type => 'NULL');
84	$self->{nc} = 0xFFFD; # REPLACEMENT CHARACTER # MUST
85	}
86	};
87
88	$self->{read_until} = sub {
89	#my ($scalar, $specials_range, $offset) = @_;
90	return 0 if defined $self->{next_nc};
91
92	my $pattern = qr/[^$_[1]\x00\x0A\x0D]/;
93	my $offset = $_[2] \|\| 0;
94
95	if ($self->{char_buffer_pos} < length $self->{char_buffer}) {
96	pos ($self->{char_buffer}) = $self->{char_buffer_pos};
97	if ($self->{char_buffer} =~ /\G(?>$pattern)+/) {
98	substr ($_[0], $offset)
99	= substr ($self->{char_buffer}, $-[0], $+[0] - $-[0]);
100	my $count = $+[0] - $-[0];
101	if ($count) {
102	$self->{column} += $count;
103	$self->{char_buffer_pos} += $count;
104	$self->{line_prev} = $self->{line};
105	$self->{column_prev} = $self->{column} - 1;
106	$self->{nc} = -1;
107	}
108	return $count;
109	} else {
110	return 0;
111	}
112	} else {
113	my $count = $input->manakai_read_until ($_[0], $pattern, $_[2]);
114	if ($count) {
115	$self->{column} += $count;
116	$self->{line_prev} = $self->{line};
117	$self->{column_prev} = $self->{column} - 1;
118	$self->{nc} = -1;
119	}
120	return $count;
121	}
122	}; # $self->{read_until}
123
124	my $onerror = $_[2] \|\| sub {
125	my (%opt) = @_;
126	my $line = $opt{token} ? $opt{token}->{line} : $opt{line};
127	my $column = $opt{token} ? $opt{token}->{column} : $opt{column};
128	warn "Parse error ($opt{type}) at line $line column $column\n";
129	};
130	$self->{parse_error} = sub {
131	$onerror->(line => $self->{line}, column => $self->{column}, @_);
132	};
133
134	my $char_onerror = sub {
135	my (undef, $type, %opt) = @_;
136	$self->{parse_error}->(level => $self->{level}->{must}, layer => 'encode',
137	line => $self->{line}, column => $self->{column} + 1,
138	%opt, type => $type);
139	}; # $char_onerror
140
141	if ($_[3]) {
142	$input = $_[3]->($input);
143	$input->onerror ($char_onerror);
144	} else {
145	$input->onerror ($char_onerror) unless defined $input->onerror;
146	}
147
148	$self->_initialize_tokenizer;
149	$self->_initialize_tree_constructor;
150	$self->_construct_tree;
151	$self->_terminate_tree_constructor;
152
153	delete $self->{parse_error}; # remove loop
154
155	return $self->{document};
156	} # parse_char_stream
157
158	sub new ($) {
159	my $class = shift;
160	my $self = bless {
161	level => {must => 'm',
162	should => 's',
163	warn => 'w',
164	info => 'i',
165	uncertain => 'u'},
166	}, $class;
167	$self->{set_nc} = sub {
168	$self->{nc} = -1;
169	};
170	$self->{parse_error} = sub {
171	#
172	};
173	$self->{change_encoding} = sub {
174	# if ($_[0] is a supported encoding) {
175	# run "change the encoding" algorithm;
176	# throw Whatpm::HTML::RestartParser (charset => $new_encoding);
177	# }
178	};
179	$self->{application_cache_selection} = sub {
180	#
181	};
182
183	$self->{is_xml} = 1;
184
185	return $self;
186	} # new
187
188	sub _initialize_tree_constructor ($) {
189	my $self = shift;
190	## NOTE: $self->{document} MUST be specified before this method is called
191	$self->{document}->strict_error_checking (0);
192	## TODO: Turn mutation events off # MUST
193	$self->{document}->dom_config
194	->{'http://suika.fam.cx/www/2006/dom-config/strict-document-children'}
195	= 0;
196	$self->{document}->manakai_is_html (0);
197	$self->{document}->set_user_data (manakai_source_line => 1);
198	$self->{document}->set_user_data (manakai_source_column => 1);
199	} # _initialize_tree_constructor
200
201	sub _terminate_tree_constructor ($) {
202	my $self = shift;
203	$self->{document}->strict_error_checking (1);
204	$self->{document}->dom_config
205	->{'http://suika.fam.cx/www/2006/dom-config/strict-document-children'}
206	= 1;
207	## TODO: Turn mutation events on
208	} # _terminate_tree_constructor
209
210	## Tree construction stage
211
212
213	## NOTE: Differences from the XML5 draft are marked as "XML5:".
214
215	## XML5: No namespace support.
216
217	## XML5: XML5 has "empty tag token". In this implementation, it is
218	## represented as a start tag token with $self->{self_closing} flag
219	## set to true.
220
221	## XML5: XML5 has "short end tag token". In this implementation, it
222	## is represented as an end tag token with $token->{tag_name} flag set
223	## to an empty string.
224
225	## XML5: Start, main, end phases. In this implementation, they are
226	## represented by insertion modes.
227
228	## Insertion modes
229	sub INITIAL_IM () { 0 }
230	sub BEFORE_ROOT_ELEMENT_IM () { 1 }
231	sub IN_ELEMENT_IM () { 2 }
232	sub AFTER_ROOT_ELEMENT_IM () { 3 }
233
234	{
235	my $token; ## TODO: change to $self->{t}
236
237	sub _construct_tree ($) {
238	my ($self) = @_;
239
240	$token = $self->_get_next_token;
241
242	delete $self->{tainted};
243	$self->{open_elements} = [];
244	$self->{insertion_mode} = INITIAL_IM;
245
246	while (1) {
247	if ($self->{insertion_mode} == IN_ELEMENT_IM) {
248	$self->_tree_in_element;
249	} elsif ($self->{insertion_mode} == AFTER_ROOT_ELEMENT_IM) {
250	$self->_tree_after_root_element;
251	} elsif ($self->{insertion_mode} == BEFORE_ROOT_ELEMENT_IM) {
252	$self->_tree_before_root_element;
253	} elsif ($self->{insertion_mode} == INITIAL_IM) {
254	$self->_tree_initial;
255	} else {
256	die "$0: Unknown XML insertion mode: $self->{insertion_mode}";
257	}
258
259	last if $token->{type} == ABORT_TOKEN;
260	}
261	} # _construct_tree
262
263	sub _tree_initial ($) {
264	my $self = shift;
265
266	B: while (1) {
267	if ($token->{type} == DOCTYPE_TOKEN) {
268	## XML5: No "DOCTYPE" token.
269
270	my $doctype = $self->{document}->create_document_type_definition
271	(defined $token->{name} ? $token->{name} : '');
272
273	## NOTE: Default value for both \|public_id\| and \|system_id\| attributes
274	## are empty strings, so that we don't set any value in missing cases.
275	$doctype->public_id ($token->{public_identifier})
276	if defined $token->{public_identifier};
277	$doctype->system_id ($token->{system_identifier})
278	if defined $token->{system_identifier};
279
280	## TODO: internal_subset
281
282	$self->{document}->append_child ($doctype);
283
284	$self->{insertion_mode} = BEFORE_ROOT_ELEMENT_IM;
285	$token = $self->_get_next_token;
286	return;
287	} elsif ($token->{type} == START_TAG_TOKEN or
288	$token->{type} == END_OF_FILE_TOKEN) {
289	$self->{insertion_mode} = BEFORE_ROOT_ELEMENT_IM;
290	## Reprocess.
291	return;
292	} elsif ($token->{type} == COMMENT_TOKEN) {
293	my $comment = $self->{document}->create_comment ($token->{data});
294	$self->{document}->append_child ($comment);
295
296	## Stay in the mode.
297	$token = $self->_get_next_token;
298	next B;
299	} elsif ($token->{type} == PI_TOKEN) {
300	my $pi = $self->{document}->create_processing_instruction
301	($token->{target}, $token->{data});
302	$self->{document}->append_child ($pi);
303
304	## Stay in the mode.
305	$token = $self->_get_next_token;
306	next B;
307	} elsif ($token->{type} == CHARACTER_TOKEN) {
308	if (not $self->{tainted} and
309	$token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
310	#
311	}
312
313	if (length $token->{data}) {
314	## XML5: Ignore the token.
315
316	unless ($self->{tainted}) {
317	$self->{parse_error}->(level => $self->{level}->{must}, type => 'text outside of root element',
318	token => $token);
319	$self->{tainted} = 1;
320	}
321
322	$self->{document}->manakai_append_text ($token->{data});
323	}
324
325	## Stay in the mode.
326	$token = $self->_get_next_token;
327	next B;
328	} elsif ($token->{type} == END_TAG_TOKEN) {
329	$self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
330	text => $token->{tag_name},
331	token => $token);
332	## Ignore the token.
333
334	## Stay in the mode.
335	$token = $self->_get_next_token;
336	next B;
337	} elsif ($token->{type} == ABORT_TOKEN) {
338	return;
339	} else {
340	die "$0: XML parser initial: Unknown token type $token->{type}";
341	}
342	} # B
343	} # _tree_initial
344
345	sub _tree_before_root_element ($) {
346	my $self = shift;
347
348	B: while (1) {
349	if ($token->{type} == START_TAG_TOKEN) {
350	my ($prefix, $ln) = split /:/, $token->{tag_name}, 2;
351	($prefix, $ln) = (undef, $prefix) unless defined $ln;
352	my $ns; ## TODO:
353	my $el = $self->{document}->create_element_ns ($ns, [$prefix, $ln]);
354	$self->{document}->append_child ($el);
355
356	if ($self->{self_closing}) {
357	delete $self->{self_closing};
358	$self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
359	} else {
360	push @{$self->{open_elements}}, [$el, $token->{tag_name}];
361	$self->{insertion_mode} = IN_ELEMENT_IM;
362	}
363
364	#delete $self->{tainted};
365
366	$token = $self->_get_next_token;
367	return;
368	} elsif ($token->{type} == COMMENT_TOKEN) {
369	my $comment = $self->{document}->create_comment ($token->{data});
370	$self->{document}->append_child ($comment);
371
372	## Stay in the mode.
373	$token = $self->_get_next_token;
374	next B;
375	} elsif ($token->{type} == PI_TOKEN) {
376	my $pi = $self->{document}->create_processing_instruction
377	($token->{target}, $token->{data});
378	$self->{document}->append_child ($pi);
379
380	## Stay in the mode.
381	$token = $self->_get_next_token;
382	next B;
383	} elsif ($token->{type} == CHARACTER_TOKEN) {
384	if (not $self->{tainted} and
385	$token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
386	#
387	}
388
389	if (length $token->{data}) {
390	## XML5: Ignore the token.
391
392	unless ($self->{tainted}) {
393	$self->{parse_error}->(level => $self->{level}->{must}, type => 'text outside of root element',
394	token => $token);
395	$self->{tainted} = 1;
396	}
397
398	$self->{document}->manakai_append_text ($token->{data});
399	}
400
401	## Stay in the mode.
402	$token = $self->_get_next_token;
403	next B;
404	} elsif ($token->{type} == END_OF_FILE_TOKEN) {
405	$self->{parse_error}->(level => $self->{level}->{must}, type => 'no root element',
406	token => $token);
407
408	$self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
409	## Reprocess.
410	return;
411	} elsif ($token->{type} == END_TAG_TOKEN) {
412	$self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
413	text => $token->{tag_name},
414	token => $token);
415	## Ignore the token.
416
417	## Stay in the mode.
418	$token = $self->_get_next_token;
419	next B;
420	} elsif ($token->{type} == DOCTYPE_TOKEN) {
421	$self->{parse_error}->(level => $self->{level}->{must}, type => 'in html:#doctype',
422	token => $token);
423	## Ignore the token.
424
425	## Stay in the mode.
426	$token = $self->_get_next_token;
427	next B;
428	} elsif ($token->{type} == ABORT_TOKEN) {
429	return;
430	} else {
431	die "$0: XML parser initial: Unknown token type $token->{type}";
432	}
433	} # B
434	} # _tree_before_root_element
435
436	sub _tree_in_element ($) {
437	my $self = shift;
438
439	B: while (1) {
440	if ($token->{type} == CHARACTER_TOKEN) {
441	$self->{open_elements}->[-1]->[0]->manakai_append_text ($token->{data});
442
443	## Stay in the mode.
444	$token = $self->_get_next_token;
445	next B;
446	} elsif ($token->{type} == START_TAG_TOKEN) {
447	my ($prefix, $ln) = split /:/, $token->{tag_name}, 2;
448	($prefix, $ln) = (undef, $prefix) unless defined $ln;
449	my $ns; ## TODO:
450	my $el = $self->{document}->create_element_ns ($ns, [$prefix, $ln]);
451	$self->{open_elements}->[-1]->[0]->append_child ($el);
452
453	if ($self->{self_closing}) {
454	delete $self->{self_closing};
455	} else {
456	push @{$self->{open_elements}}, [$el, $token->{tag_name}];
457	}
458
459	## Stay in the mode.
460	$token = $self->_get_next_token;
461	next B;
462	} elsif ($token->{type} == END_TAG_TOKEN) {
463	if ($token->{tag_name} eq '') {
464	## Short end tag token.
465	pop @{$self->{open_elements}};
466	} elsif ($self->{open_elements}->[-1]->[1] eq $token->{tag_name}) {
467	pop @{$self->{open_elements}};
468	} else {
469	$self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
470	text => $token->{tag_name},
471	token => $token);
472
473	## Has an element in scope
474	INSCOPE: for my $i (reverse 0..$#{$self->{open_elements}}) {
475	if ($self->{open_elements}->[$i]->[1] eq $token->{tag_name}) {
476	splice @{$self->{open_elements}}, $i;
477	last INSCOPE;
478	}
479	} # INSCOPE
480	}
481
482	unless (@{$self->{open_elements}}) {
483	$self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
484	$token = $self->_get_next_token;
485	return;
486	} else {
487	## Stay in the state.
488	$token = $self->_get_next_token;
489	redo B;
490	}
491	} elsif ($token->{type} == COMMENT_TOKEN) {
492	my $comment = $self->{document}->create_comment ($token->{data});
493	$self->{open_elements}->[-1]->[0]->append_child ($comment);
494
495	## Stay in the mode.
496	$token = $self->_get_next_token;
497	next B;
498	} elsif ($token->{type} == PI_TOKEN) {
499	my $pi = $self->{document}->create_processing_instruction
500	($token->{target}, $token->{data});
501	$self->{open_elements}->[-1]->[0]->append_child ($pi);
502
503	## Stay in the mode.
504	$token = $self->_get_next_token;
505	next B;
506	} elsif ($token->{type} == END_OF_FILE_TOKEN) {
507	$self->{parse_error}->(level => $self->{level}->{must}, type => 'in body:#eof',
508	token => $token);
509
510	$self->{insertion_mode} = AFTER_ROOT_ELEMENT_IM;
511	$token = $self->_get_next_token;
512	return;
513	} elsif ($token->{type} == DOCTYPE_TOKEN) {
514	$self->{parse_error}->(level => $self->{level}->{must}, type => 'in html:#doctype',
515	token => $token);
516	## Ignore the token.
517
518	## Stay in the mode.
519	$token = $self->_get_next_token;
520	next B;
521	} elsif ($token->{type} == ABORT_TOKEN) {
522	return;
523	} else {
524	die "$0: XML parser initial: Unknown token type $token->{type}";
525	}
526	} # B
527	} # _tree_in_element
528
529	sub _tree_after_root_element ($) {
530	my $self = shift;
531
532	B: while (1) {
533	if ($token->{type} == START_TAG_TOKEN) {
534	$self->{parse_error}->(level => $self->{level}->{must}, type => 'second root element',
535	token => $token);
536
537	## XML5: Ignore the token.
538
539	my ($prefix, $ln) = split /:/, $token->{tag_name}, 2;
540	($prefix, $ln) = (undef, $prefix) unless defined $ln;
541	my $ns; ## TODO:
542	my $el = $self->{document}->create_element_ns ($ns, [$prefix, $ln]);
543	$self->{document}->append_child ($el);
544
545	if ($self->{self_closing}) {
546	delete $self->{self_closing};
547	## Stay in the mode.
548	} else {
549	push @{$self->{open_elements}}, [$el, $token->{tag_name}];
550	$self->{insertion_mode} = IN_ELEMENT_IM;
551	}
552
553	#delete $self->{tainted};
554
555	$token = $self->_get_next_token;
556	return;
557	} elsif ($token->{type} == COMMENT_TOKEN) {
558	my $comment = $self->{document}->create_comment ($token->{data});
559	$self->{document}->append_child ($comment);
560
561	## Stay in the mode.
562	$token = $self->_get_next_token;
563	next B;
564	} elsif ($token->{type} == PI_TOKEN) {
565	my $pi = $self->{document}->create_processing_instruction
566	($token->{target}, $token->{data});
567	$self->{document}->append_child ($pi);
568
569	## Stay in the mode.
570	$token = $self->_get_next_token;
571	next B;
572	} elsif ($token->{type} == CHARACTER_TOKEN) {
573	if (not $self->{tainted} and
574	$token->{data} =~ s/^([\x09\x0A\x0C\x20]+)//) {
575	#
576	}
577
578	if (length $token->{data}) {
579	## XML5: Ignore the token.
580
581	unless ($self->{tainted}) {
582	$self->{parse_error}->(level => $self->{level}->{must}, type => 'text outside of root element',
583	token => $token);
584	$self->{tainted} = 1;
585	}
586
587	$self->{document}->manakai_append_text ($token->{data});
588	}
589
590	## Stay in the mode.
591	$token = $self->_get_next_token;
592	next B;
593	} elsif ($token->{type} == END_OF_FILE_TOKEN) {
594	## Stop parsing.
595
596	## TODO: implement "stop parsing".
597
598	$token = {type => ABORT_TOKEN};
599	return;
600	} elsif ($token->{type} == END_TAG_TOKEN) {
601	$self->{parse_error}->(level => $self->{level}->{must}, type => 'unmatched end tag',
602	text => $token->{tag_name},
603	token => $token);
604	## Ignore the token.
605
606	## Stay in the mode.
607	$token = $self->_get_next_token;
608	next B;
609	} elsif ($token->{type} == DOCTYPE_TOKEN) {
610	$self->{parse_error}->(level => $self->{level}->{must}, type => 'in html:#doctype',
611	token => $token);
612	## Ignore the token.
613
614	## Stay in the mode.
615	$token = $self->_get_next_token;
616	next B;
617	} elsif ($token->{type} == ABORT_TOKEN) {
618	return;
619	} else {
620	die "$0: XML parser initial: Unknown token type $token->{type}";
621	}
622	} # B
623	} # _tree_after_root_element
624
625	}
626
627	1;