--- markup/html/scripting-parser/parser.html	2008/04/27 09:16:11	1.9
+++ markup/html/scripting-parser/parser.html	2008/08/31 09:46:14	1.18
@@ -2,13 +2,20 @@
 <html lang=en>
 <head>
 <title>Live Scripting HTML Parser</title>
+<link rel=author href="http://suika.fam.cx/~wakaba/who?">
+<link rel=license href="http://suika.fam.cx/c/gnu/gpl"
+    title="GNU GPL2 or later">
 <style>
-  h1, h2 {
+  h1 {
+    margin: 0;
+    font-size: 150%;
+  } 
+  h2 {
     margin: 0;
     font-size: 100%;
   }
-  p, pre {
-    margin: 0;
+  p {
+    margin: 0 1em;
   }
   textarea {
     width: 100%;
@@ -47,15 +54,22 @@
       var p = new Parser (new InputStream (v));
       var doc = p.doc;
       p.parse ();
+      
       log (dumpTree (doc, ''));
+      
+      if (p.hasAsyncScript) {
+        log ('Some script codes are executed asynchronously; it means that the document might be rendered in different ways depending on the network condition and other factors');
+      }
     }
   } // update2
 
   var logIndentLevel = 0;
   function log (s) {
+    var indent = '';
     for (var i = 0; i < logIndentLevel; i++) {
-      s = '  ' + s;
+      indent += '  ';
     }
+    s = indent + s.replace (/\n/g, "\n" + indent);
     document.logElement.appendChild (document.createTextNode (s + "\n"));
   } // log
 
@@ -69,16 +83,24 @@
       doc = new JSDocument (this);
       doc.manakaiIsHTML = true;
     }
+    this.nextToken = [];
     this.doc = doc;
     this.openElements = [doc];
     this.input = i;
     this.scriptsExecutedAfterParsing = [];
+    this.scriptsExecutedSoon = [];
+    this.scriptsExecutedAsynchronously = [];
   } // Parser
 
   Parser.prototype.getNextToken = function () {
+    if (this.nextToken.length) {
+      return this.nextToken.shift ();
+    }
+
     var p = this;
     var i = this.input;
-    if (this.parseMode == 'script') {
+    if (this.parseMode == 'cdata') {
+      var tagName = this.endTagName;
       var token;
       if (p.insertionPoint <= 0) {
         return {type: 'abort'};
@@ -96,21 +118,22 @@
         return '';
       });
       if (token) return token;
-      i.s = i.s.replace (/^<\/[Ss][Cc][Rr][Ii][Pp][Tt]>/, function (s) {
+      var pattern = new RegExp ('^</' + tagName + '>', 'i');
+      i.s = i.s.replace (pattern, function (s) {
         if (p.insertionPoint < s.length) {
           token = {type: 'abort'};
           return s;
         }
-        token = {type: 'end-tag', value: 'script'};
+        token = {type: 'end-tag', value: tagName};
         p.insertionPoint -= s.length;
         return '';
       });
       if (token) return token;
       var m;
-      if ((p.insertionPoint < '</script'.length) &&
-          (m = i.s.match (/^<\/([SCRIPTscript]+)/))) {
+      if ((p.insertionPoint < ('</' + tagName).length) &&
+          (m = i.s.match (/^<\/([A-Za-z]+)/))) {
         var v = m[1].substring (0, p.insertionPoint).toLowerCase ();
-        if (v == 'script'.substring (0, p.insertionPoint - '</'.length)) {
+        if (v == tagName.substring (0, p.insertionPoint - '</'.length)) {
           return {type: 'abort'};
         }
       }
@@ -128,7 +151,7 @@
     i.s = i.s.replace (/^<\/([^>]+)(?:>|$)/, function (s, e) {
       if (p.insertionPoint < s.length ||
           (p.insertionPoint <= s.length &&
-           s.substring (s.length - 1, 1) != '>')) {
+           s.substring (s.length - 1, s.length) != '>')) {
         token = {type: 'abort'};
         return s;
       }
@@ -140,7 +163,7 @@
     i.s = i.s.replace (/^<([^>]+)(?:>|$)/, function (s, e) {
       if (p.insertionPoint < s.length ||
           (p.insertionPoint <= s.length &&
-           s.substring (s.length - 1, 1) != '>')) {
+           s.substring (s.length - 1, s.length) != '>')) {
         token = {type: 'abort'};
         return s;
       }
@@ -203,6 +226,24 @@
       var token = this.getNextToken ();
       log ('token: ' + token.type + ' "' + token.value + '"');
 
+      if (this.cdataEndTagRequired) {
+        // Generic CDATA parsing algorithm
+
+        if (token.type != 'abort') {
+          // 7.
+          if (token.type == 'end-tag' && token.value == this.endTagName) {
+            // 7.1. Ignores it.
+            //
+          } else {
+            // 7.2. Parse error.
+            log ('Parse error: no </' + this.endTagName + '>');
+            this.nextToken.unshift (token);
+          }
+          this.cdataEndTagRequired = false;
+          continue;
+        }
+      }
+
       if (token.type == 'start-tag') {
         if (token.value == 'script') {
           // 1. Create an element for the token in the HTML namespace.
@@ -215,7 +256,8 @@
           el.manakaiParserInserted = true;
 
           // 3. Switch the tokeniser's content model flag to the CDATA state.
-          this.parseMode = 'script';
+          this.parseMode = 'cdata';
+          this.endTagName = 'script';
 
           // 4.1. Collect all the character tokens.
           while (true) {
@@ -229,15 +271,16 @@
             // 4.2. Until it returns a token that is not a character token, or
             // until it stops tokenising.
             } else if (token.type == 'eof' ||
-                       (token.type == 'end-tag' && token.value == 'script') ||
+                       token.type == 'end-tag' ||
                        token.type == 'abort') {
               // 6. Switched back to the PCDATA state.
               this.parseMode = 'pcdata';
 
               // 7.1. If the next token is not an end tag token with ...
-              if (token.type != 'end-tag') {
+              if (!(token.type == 'end-tag' && token.value == 'script')) {
                 // 7.2. This is a parse error.
                 log ('Parse error: no </' + 'script>');
+                this.nextToken.unshift (token);
 
                 // 7.3. Mark the script element as "already executed".
                 el.manakaiAlreadyExecuted = true;
@@ -269,8 +312,8 @@
           oldInsertionPoint += this.insertionPoint;
           this.setInsertionPoint (oldInsertionPoint);
 
-          // 12. If there is a script that will execute as soon as ...
-          while (this.scriptExecutedWhenParserResumes) {
+          // 12. If there is a pending external script
+          while (this.pendingExternalScript) {
             // 12.1. If the tree construction stage is being called reentrantly
             if (this.reentrant) {
               log ('parse: abort (reentrance)');
@@ -280,8 +323,8 @@
             // 12.2. Otherwise
             } else {
               // 1.
-              var script = this.scriptExecutedWhenParserResumes;
-              this.scriptExecutedWhenParserResumes = null;
+              var script = this.pendingExternalScript;
+              this.pendingExternalScript = null;
 
               // 2. Pause until the script has completed loading.
               // 
@@ -299,6 +342,57 @@
               //
             }
           }
+        } else if (token.value == 'style' ||
+                   token.value == 'noscript' ||
+                   token.value == 'xmp') {
+          // 1. Create an element for the token in the HTML namespace.
+          var el = new JSElement (this.doc, token.value);
+
+          // 2. Append the new element to the current node.
+          this.openElements[this.openElements.length - 1].appendChild (el);
+
+          // 3. Switch the tokeniser's content model flag to the CDATA state.
+          this.parseMode = 'cdata';
+          this.endTagName = token.value;
+
+          // 4.1. Collect all the character tokens.
+          while (true) {
+            var token = this.getNextToken ();
+            log ('token: ' + token.type + ' "' + token.value + '"');
+
+            if (token.type == 'char') {
+              // 5. Append a single Text node to the script element node.
+              el.manakaiAppendText (token.value);
+
+            // 4.2. Until it returns a token that is not a character token, or
+            // until it stops tokenising.
+            } else if (token.type == 'eof' ||
+                       token.type == 'end-tag' ||
+                       token.type == 'abort') {
+              // 6. Switched back to the PCDATA state.
+              this.parseMode = 'pcdata';
+
+              if (token.type == 'abort') {
+                this.cdataEndTagRequired = true;
+                break;
+              }
+
+              // 7.1. If the next token is not an end tag token with ...
+              if (!(token.type == 'end-tag' &&
+                    token.value == this.endTagName)) {
+                // 7.2. This is a parse error.
+                log ('Parse error: no </' + this.endTagName + '>');
+                this.nextToken.unshift (token);
+
+                // 7.3. Mark the script element as "already executed".
+                el.manakaiAlreadyExecuted = true;
+              } else {
+                // 7.4. Ignore it.
+                //
+              }
+              break;
+            }
+          }
         } else {
           var el = new JSElement (this.doc, token.value);
           this.openElements[this.openElements.length - 1].appendChild (el);
@@ -329,8 +423,41 @@
 
     // "When a script completes loading" rules start applying.
 
-    // TODO: Handles "list of scripts that will execute as soon as possible"
-    // and "list of scripts that will execute asynchronously"
+    while (this.scriptsExecutedSoon.length > 0 ||
+           this.scriptsExecutedAsynchronously.length > 0) {
+      // Handle "list of scripts that will execute as soon as possible".
+      while (this.scriptsExecutedSoon.length > 0) {
+        var e = this.scriptsExecutedSoon.shift ();
+  
+        // If it has completed loading
+        log ('Execute an external script not inserted by parser...');
+        executeScript (this.doc, e);
+
+        // NOTE: It MAY be executed before the end of the parsing, according
+        // to the spec.
+        this.hasAsyncScript = true;
+      }
+
+      // Handle "list of scripts that will execute asynchronously".
+      while (this.scriptsExecutedAsynchronously.length > 0) {
+        var e = this.scriptsExecutedAsynchronously.shift ();
+
+        // Step 1.
+        // We assume that all scripts have been loaded at this time.
+  
+        // Step 2.
+        log ('Execute an asynchronous script...');
+        executeScript (this.doc, e);
+
+        // Step 3.
+        //
+ 
+        // Step 4.
+        //
+
+        this.hasAsyncScript = true;
+      }
+    }
 
     // Handle "list of scripts that will execute when the document has finished
     // parsing".
@@ -350,7 +477,7 @@
 
     log ('DOMContentLoaded event fired');
 
-    // "delays tha load event" things has completed:
+    // "delays the load event" things has completed:
     // readyState = 'complete'
     log ('load event fired');
 
@@ -406,7 +533,7 @@
       // 2.4. If the script element has its "already executed" flag set
       if (e.manakaiAlreadyExecuted) {
         // 2.5. Abort these steps at this point.
-        log ('Running a script: aborted');
+        log ('Running a script: aborted (already executed)');
         logIndentLevel--;
         return e;
       }
@@ -425,18 +552,22 @@
         p.scriptsExecutedAfterParsing.push (e);
         log ('Running a script: aborted (defer)');
       } else if (e.async && e.src != null) {
-        // TODO
-      } else if (e.async && e.src == null
-                 /* && list of scripts that will execute asynchronously is not empty */) {
-        // TODO
+        p.scriptsExecutedAsynchronously.push (e);
+        log ('Running a script: aborted (async src)');
+      } else if (e.async && e.src == null &&
+                 p.scriptsExecutedAsynchronously.length > 0) {
+        p.scriptsExecutedAsynchronously.push (e);
+        log ('Running a script: aborted (async)');
+        // ISSUE: What is the difference with the case above?
       } else if (e.src != null && e.manakaiParserInserted) {
-        if (p.scriptExecutedWhenParserResumes) {
-          log ('Error: There is a script that will execute as soon as the parser resumes.');
+        if (p.pendingExternalScript) {
+          log ('Error: There is a pending external script.');
         }
-        p.scriptExecutedWhenParserResumes = e;
-        log ('Running a script: aborted (src)');
+        p.pendingExternalScript = e;
+        log ('Running a script: aborted (src parser-inserted)');
       } else if (e.src != null) {
-        // TODO
+        p.scriptsExecutedSoon.push (e);
+        log ('Running a script: aborted (src)');
       } else {
         executeScript (doc, e); // even if other scripts are already executing.
       }
@@ -467,7 +598,6 @@
     }
 
     // If the load was successful
-    log ('load event fired at the script element');
 
     if (true) {
     // Scripting is enabled, Document.designMode is disabled,
@@ -476,6 +606,8 @@
       parseAndRunScript (doc, s);
     }
 
+    log ('load event fired at the script element');
+
     log ('executing a script block: end');
   } // executeScript
 
@@ -484,9 +616,9 @@
       var m;
       if (m = uri.match (/^javascript:\s*(?:'([^']*)'|"([^"]+)")\s*$/i)) {
         if (m[1]) {
-          return m[1];
+          return unescapeJSLiteral (m[1]);
         } else if (m[2]) {
-          return m[2];
+          return unescapeJSLiteral (m[2]);
         } else {
           return null;
         }
@@ -507,12 +639,30 @@
         matched = true;
         var args = [];
         t.replace (/('[^']*'|"[^"]*")/g, function (s, v) {
-          args.push (v.substring (1, v.length - 1));
+          args.push (unescapeJSLiteral (v.substring (1, v.length - 1)));
           return '';
         });
         doc.write.apply (doc, args);
         return '';
       });
+      var noDocumentElement = false;
+      s = s.replace (/^\s*var\s+s\s*=\s*document\.createElement\s*\(\s*['"]script['"]\s*\)\s*;\s*s\.src\s*=\s*(?:'([^']*)'|"([^"]*)")\s*;\s*document\.documentElement\.appendChild\s*\(\s*s\s*\)\s*;\s*/,
+      function (s, t, u) {
+        matched = true;
+        var args = [unescapeJSLiteral (t ? t : u)];
+        noDocumentElement = !doc._insertExternalScript.apply (doc, args);
+        return '';
+      });
+      if (noDocumentElement) {
+        log ('Script error: documentElement is null');
+        break;
+      }
+      s = s.replace (/^\s*w\s*\(\s*document\.documentElement\.innerHTML\s*\)\s*;\s*/,
+      function (s, t) {
+        matched = true;
+        log (dumpTree (doc, ''));
+        return '';
+      });
       if (s == '') break;
       if (!matched) {
         log ('Script parse error: "' + s + '"');
@@ -521,6 +671,12 @@
     }
   } // parseAndRunScript
 
+  function unescapeJSLiteral (s) {
+    return s.replace (/\\u([0-9A-Fa-f]{4})/g, function (t, v) {
+      return String.fromCharCode (parseInt ('0x' + v));
+    });
+  } // unescapeJSLiteral
+
   function JSText (data) {
     this.data = data;
   } // JSText
@@ -589,6 +745,7 @@
   }; // document.open
 
   JSDocument.prototype.write = function () {
+    log ('document.write: start');
     logIndentLevel++;
 
     var p = this._parser;
@@ -608,10 +765,11 @@
         + p.input.s.substring (p.insertionPoint, p.input.s.length);
     p.insertionPoint += s.length;
 
-    // 3. If there is a script that will execute as soon as the parser resumes
-    if (p.scriptExecutedAfterParserResumes) {
+    // 3. If there is a pending external script
+    if (p.pendingExternalScript) {
       log ('document.write: processed later (there is an unprocessed <script src>)');
       logIndentLevel--;
+      log ('document.write: return');
       return;
     }
 
@@ -625,12 +783,33 @@
     // to do something here?
 
     // 5. Return
+    logIndentLevel--;
     log ('document.write: return');
 
-    logIndentLevel--;
     return;
   }; // document.write
 
+  JSDocument.prototype._insertExternalScript = function (uri) {
+    var s = new JSElement (this, 'script');
+    s.src = uri;
+    if (this.documentElement) {
+      this.documentElement.appendChild (s);
+      return true;
+    } else {
+      return false;
+    }
+  }; // _insertExternalScript
+
+  JSDocument.prototype.__defineGetter__ ('documentElement', function () {
+    var cn = this.childNodes;
+    for (var i = 0; i < cn.length; i++) {
+      if (cn[i] instanceof JSElement) {
+        return cn[i]
+      }
+    }
+    return null;
+  });
+
   JSElement.prototype.__defineGetter__ ('text', function () {
     var r = '';
     for (var i = 0; i < this.childNodes.length; i++) {
@@ -700,10 +879,10 @@
 &lt;p>
 </textarea>
 
-<h2>Log</h2>
+<h2 id=log>Log</h2>
 <p><output></output>
 
-<h2>Note</h2>
+<h2 id=notes>Notes</h2>
 
 <p>This is a <em>simplified</em> implementation of 
 <a href="http://www.whatwg.org/specs/web-apps/current-work/#parsing">HTML5 
@@ -718,26 +897,72 @@
 algorithm, and so on.
 <li>Does not raise parse errors for invalid attribute specifications in start 
 or end tags.
-<li>Does not support CDATA/PCDATA element other than <code>script</code>.
+<li>Does not support RCDATA elements (<code>title</code> and
+<code>textarea</code>).
+<li>Does not strip the first newline in <code>pre</code>,
+<code>listing</code>, and <code>textarea</code> elements.
 <li>Does not support <code>&lt;!--</code>..<code>--></code> parsing rule
-in <code>script</code> element.
+in CDATA/RCDATA elements.
 <li>Does not support foreign (SVG or MathML) elements.
 <li>Only supports <code>script</code> <code>type</code>
 <code>text/javascript</code>.  <code>type</code> and <code>language</code>
 attributes are ignored.
-<li>Only supports <code>document.write</code>.
-The script code must be match to the regular expression
-<code>^\s*(?:document\.write\s*\(<var>v</var>\s*(?:,\s*<var>v</var>\s*)*\)\s*;\s*)*$</code>
-where <var>v</var> is <code>"[^"]*"|'[^']*'</code>.
+<li>Only supports limited statements.  It must consist of zero or more
+of statements looking similar to the following statements, possibly
+introduced, followed, or separated by white space characters:
+  <ul>
+  <li><code>document.write ("<var>string</var>", ["<var>string</var>", ...]);</code>.
+  <li><code>var s = document.createElement ("script");
+            s.src = "<var>string</var>";
+            document.documentElement.appendChild (s);</code>
+  <li><code>w (document.documentElement.innerHTML);</code> (This statement
+  can be used to dump the document, even when the document has no
+  document element.  The output format is the tree dump format used
+  in html5lib test data, not <abbr>HTML</abbr>.)
+  </ul>
+Note that strings may be delimited by <code>'</code>s instead of
+<code>"</code>s.
 <li>Only supports <code>javascript:</code> 
 <abbr title="Uniform Resourace Identifiers">URI</abbr> scheme in the 
 <code>src</code> attribute of the <code>script</code> element.  In addition, 
 the <abbr title="Uniform Resource Identifiers">URI</abbr> must be conform to 
 the regular expression <code>^javascript:\s*(?:"[^"]*"|'[^']*')\s*$</code>.
+<li>Only supports <code>\u<var>HHHH</var></code> escapes in JavaScript 
+string literals.
+<li>Does not handle <i>stop parsing</i> phase correctly if the document is 
+replaced by <code>document.open ()</code> call.  In other word, delayed 
+(deferred or asynchronous) script executions and event firings might be
+treated in a wrong way if a <code>document.open ()</code> invocation
+is implicitly done by <code>document.write ()</code> in a delayed script.
 </ul>
 
 <p>For some reason, this parser does not work in browsers that do
 not support JavaScript 1.5.
 
+<!-- TODO: |src| attribute value should refer the value at the time
+when it is inserted into the document, not the value when the script is 
+executed.  Currently it does not matter, since we don't allow dynamic
+modification to the |src| content/DOM attribute value yet. -->
+
 </body>
-</html>
\ No newline at end of file
+</html>
+<!-- $Date: 2008/08/31 09:46:14 $ -->
+<!--
+
+Copyright 2008 Wakaba <w@suika.fam.cx>
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+-->
