--- markup/html/scripting-parser/parser.html	2008/04/29 02:50:00	1.14
+++ markup/html/scripting-parser/parser.html	2008/09/20 13:32:45	1.20
@@ -65,9 +65,11 @@
 
   var logIndentLevel = 0;
   function log (s) {
+    var indent = '';
     for (var i = 0; i < logIndentLevel; i++) {
-      s = '  ' + s;
+      indent += '  ';
     }
+    s = indent + s.replace (/\n/g, "\n" + indent);
     document.logElement.appendChild (document.createTextNode (s + "\n"));
   } // log
 
@@ -81,6 +83,7 @@
       doc = new JSDocument (this);
       doc.manakaiIsHTML = true;
     }
+    this.nextToken = [];
     this.doc = doc;
     this.openElements = [doc];
     this.input = i;
@@ -90,6 +93,10 @@
   } // Parser
 
   Parser.prototype.getNextToken = function () {
+    if (this.nextToken.length) {
+      return this.nextToken.shift ();
+    }
+
     var p = this;
     var i = this.input;
     if (this.parseMode == 'cdata') {
@@ -144,7 +151,7 @@
     i.s = i.s.replace (/^<\/([^>]+)(?:>|$)/, function (s, e) {
       if (p.insertionPoint < s.length ||
           (p.insertionPoint <= s.length &&
-           s.substring (s.length - 1, 1) != '>')) {
+           s.substring (s.length - 1, s.length) != '>')) {
         token = {type: 'abort'};
         return s;
       }
@@ -156,7 +163,7 @@
     i.s = i.s.replace (/^<([^>]+)(?:>|$)/, function (s, e) {
       if (p.insertionPoint < s.length ||
           (p.insertionPoint <= s.length &&
-           s.substring (s.length - 1, 1) != '>')) {
+           s.substring (s.length - 1, s.length) != '>')) {
         token = {type: 'abort'};
         return s;
       }
@@ -219,6 +226,24 @@
       var token = this.getNextToken ();
       log ('token: ' + token.type + ' "' + token.value + '"');
 
+      if (this.cdataEndTagRequired) {
+        // Generic CDATA parsing algorithm
+
+        if (token.type != 'abort') {
+          // 7.
+          if (token.type == 'end-tag' && token.value == this.endTagName) {
+            // 7.1. Ignores it.
+            //
+          } else {
+            // 7.2. Parse error.
+            log ('Parse error: no </' + this.endTagName + '>');
+            this.nextToken.unshift (token);
+          }
+          this.cdataEndTagRequired = false;
+          continue;
+        }
+      }
+
       if (token.type == 'start-tag') {
         if (token.value == 'script') {
           // 1. Create an element for the token in the HTML namespace.
@@ -255,6 +280,7 @@
               if (!(token.type == 'end-tag' && token.value == 'script')) {
                 // 7.2. This is a parse error.
                 log ('Parse error: no </' + 'script>');
+                this.nextToken.unshift (token);
 
                 // 7.3. Mark the script element as "already executed".
                 el.manakaiAlreadyExecuted = true;
@@ -269,7 +295,7 @@
           // 8.1. If the parser were originally created for the ...
           if (this.fragmentParsingMode) {
             // 8.2. Mark the script element as "already executed" and ...
-            el.alreadyExecuted = true;
+            el.manakaiAlreadyExecuted = true;
             continue;
           }
 
@@ -286,8 +312,8 @@
           oldInsertionPoint += this.insertionPoint;
           this.setInsertionPoint (oldInsertionPoint);
 
-          // 12. If there is a script that will execute as soon as ...
-          while (this.scriptExecutedWhenParserResumes) {
+          // 12. If there is a pending external script
+          while (this.pendingExternalScript) {
             // 12.1. If the tree construction stage is being called reentrantly
             if (this.reentrant) {
               log ('parse: abort (reentrance)');
@@ -297,8 +323,8 @@
             // 12.2. Otherwise
             } else {
               // 1.
-              var script = this.scriptExecutedWhenParserResumes;
-              this.scriptExecutedWhenParserResumes = null;
+              var script = this.pendingExternalScript;
+              this.pendingExternalScript = null;
 
               // 2. Pause until the script has completed loading.
               // 
@@ -346,11 +372,17 @@
               // 6. Switched back to the PCDATA state.
               this.parseMode = 'pcdata';
 
+              if (token.type == 'abort') {
+                this.cdataEndTagRequired = true;
+                break;
+              }
+
               // 7.1. If the next token is not an end tag token with ...
               if (!(token.type == 'end-tag' &&
                     token.value == this.endTagName)) {
                 // 7.2. This is a parse error.
                 log ('Parse error: no </' + this.endTagName + '>');
+                this.nextToken.unshift (token);
 
                 // 7.3. Mark the script element as "already executed".
                 el.manakaiAlreadyExecuted = true;
@@ -489,54 +521,60 @@
       var doc = this.ownerDocument || this;
       var p = doc._parser;
 
-      // 1. Script type
+      // 1.The script's type
+      //
+
+      // 2. The cript's character encoding
       //
 
-      // 2.1. If scripting is disabled
+      // 3.1. If without script
       //
       // 2.2. If the script element was created by an XML ... innerHTML ...
       //
       // 2.3. If the user agent does not support the scripting language ...
       //
-      // 2.4. If the script element has its "already executed" flag set
-      if (e.manakaiAlreadyExecuted) {
+      if (false) {
         // 2.5. Abort these steps at this point.
-        log ('Running a script: aborted');
+        log ('Running a script: aborted (noscript)');
         logIndentLevel--;
         return e;
       }
 
-      // 3. Set the element's "already executed" flag.
+      // 4. Set the element's "already executed" flag.
       e.manakaiAlreadyExecuted = true;
 
-      // 4. If the element has a src attribute, then a load for ...
+      // 5. If the element has a src attribute, then a load for ...
       // TODO: load an external resource
 
       // 5. The first of the following options:
 
-      // 5.1. 
       if (/* TODO: If the document is still being parsed && */
           e.defer && !e.async) {
+        // 6.1. 
         p.scriptsExecutedAfterParsing.push (e);
         log ('Running a script: aborted (defer)');
       } else if (e.async && e.src != null) {
+        // 6.2.
         p.scriptsExecutedAsynchronously.push (e);
         log ('Running a script: aborted (async src)');
       } else if (e.async && e.src == null &&
                  p.scriptsExecutedAsynchronously.length > 0) {
+        // 6.3.
         p.scriptsExecutedAsynchronously.push (e);
         log ('Running a script: aborted (async)');
-        // ISSUE: What is the difference with the case above?
       } else if (e.src != null && e.manakaiParserInserted) {
-        if (p.scriptExecutedWhenParserResumes) {
-          log ('Error: There is a script that will execute as soon as the parser resumes.');
+        // 6.4.
+        if (p.pendingExternalScript) {
+          log ('Error: There is a pending external script.');
         }
-        p.scriptExecutedWhenParserResumes = e;
+        p.pendingExternalScript = e;
         log ('Running a script: aborted (src parser-inserted)');
       } else if (e.src != null) {
+        // 6.5.
         p.scriptsExecutedSoon.push (e);
         log ('Running a script: aborted (src)');
       } else {
+        // 6.6.
         executeScript (doc, e); // even if other scripts are already executing.
       }
 
@@ -566,7 +604,6 @@
     }
 
     // If the load was successful
-    log ('load event fired at the script element');
 
     if (true) {
     // Scripting is enabled, Document.designMode is disabled,
@@ -575,6 +612,8 @@
       parseAndRunScript (doc, s);
     }
 
+    log ('load event fired at the script element');
+
     log ('executing a script block: end');
   } // executeScript
 
@@ -612,11 +651,22 @@
         doc.write.apply (doc, args);
         return '';
       });
-      s = s.replace (/^\s*var\s+s\s*=\s*document\.createElement\s*\(\s*['"]script['"]\s*\)\s*;\s*s\.src\s*=\s*(?:'(javascript:[^']*)'|"(javascript:[^"]*)")\s*;\s*document\.documentElement\.appendChild\s*\(\s*s\s*\)\s*;\s*/,
+      var noDocumentElement = false;
+      s = s.replace (/^\s*var\s+s\s*=\s*document\.createElement\s*\(\s*['"]script['"]\s*\)\s*;\s*s\.src\s*=\s*(?:'([^']*)'|"([^"]*)")\s*;\s*document\.documentElement\.appendChild\s*\(\s*s\s*\)\s*;\s*/,
       function (s, t, u) {
         matched = true;
         var args = [unescapeJSLiteral (t ? t : u)];
-        doc._insertExternalScript.apply (doc, args);
+        noDocumentElement = !doc._insertExternalScript.apply (doc, args);
+        return '';
+      });
+      if (noDocumentElement) {
+        log ('Script error: documentElement is null');
+        break;
+      }
+      s = s.replace (/^\s*w\s*\(\s*document\.documentElement\.innerHTML\s*\)\s*;\s*/,
+      function (s, t) {
+        matched = true;
+        log (dumpTree (doc, ''));
         return '';
       });
       if (s == '') break;
@@ -701,6 +751,7 @@
   }; // document.open
 
   JSDocument.prototype.write = function () {
+    log ('document.write: start');
     logIndentLevel++;
 
     var p = this._parser;
@@ -720,10 +771,11 @@
         + p.input.s.substring (p.insertionPoint, p.input.s.length);
     p.insertionPoint += s.length;
 
-    // 3. If there is a script that will execute as soon as the parser resumes
-    if (p.scriptExecutedAfterParserResumes) {
+    // 3. If there is a pending external script
+    if (p.pendingExternalScript) {
       log ('document.write: processed later (there is an unprocessed <script src>)');
       logIndentLevel--;
+      log ('document.write: return');
       return;
     }
 
@@ -737,16 +789,21 @@
     // to do something here?
 
     // 5. Return
+    logIndentLevel--;
     log ('document.write: return');
 
-    logIndentLevel--;
     return;
   }; // document.write
 
   JSDocument.prototype._insertExternalScript = function (uri) {
     var s = new JSElement (this, 'script');
     s.src = uri;
-    this.documentElement.appendChild (s);
+    if (this.documentElement) {
+      this.documentElement.appendChild (s);
+      return true;
+    } else {
+      return false;
+    }
   }; // _insertExternalScript
 
   JSDocument.prototype.__defineGetter__ ('documentElement', function () {
@@ -846,11 +903,12 @@
 algorithm, and so on.
 <li>Does not raise parse errors for invalid attribute specifications in start 
 or end tags.
-<li>Does not support PCDATA elements (<code>title</code> and
+<li>Does not support RCDATA elements (<code>title</code> and
 <code>textarea</code>).
-<li>Does not strip the first newline in <code>pre</code> elements.
+<li>Does not strip the first newline in <code>pre</code>,
+<code>listing</code>, and <code>textarea</code> elements.
 <li>Does not support <code>&lt;!--</code>..<code>--></code> parsing rule
-in <code>script</code> element.
+in CDATA/RCDATA elements.
 <li>Does not support foreign (SVG or MathML) elements.
 <li>Only supports <code>script</code> <code>type</code>
 <code>text/javascript</code>.  <code>type</code> and <code>language</code>
@@ -863,6 +921,10 @@
   <li><code>var s = document.createElement ("script");
             s.src = "<var>string</var>";
             document.documentElement.appendChild (s);</code>
+  <li><code>w (document.documentElement.innerHTML);</code> (This statement
+  can be used to dump the document, even when the document has no
+  document element.  The output format is the tree dump format used
+  in html5lib test data, not <abbr>HTML</abbr>.)
   </ul>
 Note that strings may be delimited by <code>'</code>s instead of
 <code>"</code>s.
@@ -888,9 +950,13 @@
 executed.  Currently it does not matter, since we don't allow dynamic
 modification to the |src| content/DOM attribute value yet. -->
 
+<p>See also
+<a href="http://suika.fam.cx/gate/2005/sw/Live%20Scripting%20HTML%20Parser">SuikaWiki:
+Live Scripting HTML Parser</a>.
+
 </body>
 </html>
-<!-- $Date: 2008/04/29 02:50:00 $ -->
+<!-- $Date: 2008/09/20 13:32:45 $ -->
 <!--
 
 Copyright 2008 Wakaba <w@suika.fam.cx>