{"tests": [ {"description": "empty", "input": "", "output": []}, {"description": "length=1 IDENT (lowercase)", "input": "a", "output": [["IDENT", "a"]]}, {"description": "length=1 IDENT (uppercase)", "input": "A", "output": [["IDENT", "A"]]}, {"description": "length=2 IDENT (uppercase)", "input": "AB", "output": [["IDENT", "AB"]]}, {"description": "length=2 IDENT (lowercase)", "input": "ab", "output": [["IDENT", "ab"]]}, {"description": "\\", "input": "\\", "output": [["DELIM", "\\"]]}, {"description": "IDENT started by escape 1", "input": "\\41", "output": [["IDENT", "A"]]}, {"description": "IDENT started by escape 2", "input": "\\041", "output": [["IDENT", "A"]]}, {"description": "IDENT started by escape 3", "input": "\\0041", "output": [["IDENT", "A"]]}, {"description": "IDENT started by escape 4", "input": "\\00041", "output": [["IDENT", "A"]]}, {"description": "IDENT started by escape 5", "input": "\\000041", "output": [["IDENT", "A"]]}, {"description": "IDENT started by escape 6", "input": "\\0000041", "output": [["IDENT", "\u00041"]]}, {"description": "IDENT started by escape 7", "input": "\\00000041", "output": [["IDENT", "\u000041"]]}, {"description": "IDENT started by escape followed by a space 1", "input": "\\41 ", "output": [["IDENT", "A"]]}, {"description": "IDENT started by escape followed by a space 2", "input": "\\41 a", "output": [["IDENT", "Aa"]]}, {"description": "IDENT started by escape followed by a space 3", "input": "\\41\u0009a", "output": [["IDENT", "Aa"]]}, {"description": "IDENT started by escape followed by a space 4", "input": "\\000041 ", "output": [["IDENT", "A"]]}, {"description": "IDENT started by escape followed by a space 5", "input": "\\000041\u000D\u000A", "output": [["IDENT", "A"]]}, {"description": "\\\u000D", "input": "\\\u000D", "output": [["DELIM", "\\"], ["S"]]}, {"description": "\\\u000D\u000A", "input": "\\\u000D\u000A", "output": [["DELIM", "\\"], ["S"]]}, {"description": "\\\u000D\\\u000A", "input": "\\\u000D\\\u000A", "output": [["DELIM", "\\"], ["S"], ["DELIM", "\\"], ["S"]]}, {"description": "\\\u000C", "input": "\\\u000C", "output": [["DELIM", "\\"], ["S"]]}, {"description": "\\\u000A", "input": "\\\u000A", "output": [["DELIM", "\\"], ["S"]]}, {"description": "\\ ", "input": "\\ ", "output": [["IDENT", " "]]}, {"description": "\\\\", "input": "\\\\", "output": [["IDENT", "\\"]]}, {"description": "\\\\\\", "input": "\\\\\\", "output": [["IDENT", "\\"], ["DELIM", "\\"]]}, {"description": "\\\u0009", "input": "\\\u0009", "output": [["IDENT", "\u0009"]]}, {"description": "\\X", "input": "\\X", "output": [["IDENT", "X"]]}, {"description": "a\\", "input": "a\\", "output": [["IDENT", "a"], ["DELIM", "\\"]]}, {"description": "a\\f1", "input": "a\\f1", "output": [["IDENT", "a\u00f1"]]}, {"description": "a\\F1", "input": "a\\F1", "output": [["IDENT", "a\u00f1"]]}, {"description": "a\\0f1", "input": "a\\0f1", "output": [["IDENT", "a\u00f1"]]}, {"description": "a\\0F1", "input": "a\\0F1", "output": [["IDENT", "a\u00f1"]]}, {"description": "a\\41", "input": "a\\41", "output": [["IDENT", "aA"]]}, {"description": "a\\41 b", "input": "a\\41 b", "output": [["IDENT", "aAb"]]}, {"description": "a\\41\u0009X", "input": "a\\41\u0009X", "output": [["IDENT", "aAX"]]}, {"description": "a\\4e00", "input": "a\\4e00", "output": [["IDENT", "a\u4e00"]]}, {"description": "a\\41 X", "input": "a\\41 X", "output": [["IDENT", "aA"], ["S"], ["IDENT", "X"]]}, {"description": "a\\ ", "input": "a\\ ", "output": [["IDENT", "a "]]}, {"description": "a\\ b", "input": "a\\ b", "output": [["IDENT", "a b"]]}, {"description": "a\\\\b", "input": "a\\\\b", "output": [["IDENT", "a\\b"]]}, {"description": "a\\\u0009b", "input": "a\\\u0009b", "output": [["IDENT", "a\u0009b"]]}, {"description": "a\\GB", "input": "a\\GB", "output": [["IDENT", "aGB"]]}, {"description": "a\\\u000D", "input": "a\\\u000D", "output": [["IDENT", "a"], ["DELIM", "\\"], ["S"]]}, {"description": "a\\\u000A", "input": "a\\\u000A", "output": [["IDENT", "a"], ["DELIM", "\\"], ["S"]]}, {"description": "a\\\u000D\u000A", "input": "a\\\u000D\u000A", "output": [["IDENT", "a"], ["DELIM", "\\"], ["S"]]}, {"description": "a\\\u000D\\\u000A", "input": "a\\\u000D\\\u000A", "output": [["IDENT", "a"], ["DELIM", "\\"], ["S"], ["DELIM", "\\"], ["S"]]}, {"description": "a\\\u000C", "input": "a\\\u000C", "output": [["IDENT", "a"], ["DELIM", "\\"], ["S"]]}, {"description": "-", "input": "-", "output": [["DELIM", "-"]]}, {"description": "\\-", "input": "\\-", "output": [["IDENT", "-"]]}, {"description": "-i", "input": "-i", "output": [["IDENT", "-i"]]}, {"description": "\\-i", "input": "\\-i", "output": [["IDENT", "-i"]]}, {"description": "-ident", "input": "-ident", "output": [["IDENT", "-ident"]]}, {"description": "\\-ident", "input": "\\-ident", "output": [["IDENT", "-ident"]]}, {"description": "-ident-ident", "input": "-ident-ident", "output": [["IDENT", "-ident-ident"]]}, {"description": "\\-ident-ident", "input": "\\-ident-ident", "output": [["IDENT", "-ident-ident"]]}, {"description": "-1", "input": "-1", "output": [["DELIM", "-"], ["NUMBER", "1"]]}, {"description": "\\-1", "input": "\\-1", "output": [["IDENT", "-1"]]}, {"description": "-1a", "input": "-1a", "output": [["DELIM", "-"], ["DIMENSION", "1", "a"]]}, {"description": "\\-1a", "input": "\\-1a", "output": [["IDENT", "-1a"]]}, {"description": "-\\31", "input": "-\\31", "output": [["IDENT", "-1"]]}, {"description": "\\-\\31", "input": "\\-\\31", "output": [["IDENT", "-1"]]}, {"description": "-\\41", "input": "-\\41", "output": [["IDENT", "-A"]]}, {"description": "\\-\\41", "input": "\\-\\41", "output": [["IDENT", "-A"]]}, {"description": "-\\\u000D", "input": "-\\\u000D", "output": [["DELIM", "-"], ["DELIM", "\\"], ["S"]]}, {"description": "\\-\\\u000D", "input": "\\-\\\u000D", "output": [["IDENT", "-"], ["DELIM", "\\"], ["S"]]}, {"description": "\u4e00", "input": "\u4e00", "output": [["IDENT", "\u4e00"]]}, {"description": "a\u4e00b", "input": "a\u4e00b", "output": [["IDENT", "a\u4e00b"]]}, {"description": "@", "input": "@", "output": [["DELIM", "@"]]}, {"description": "@a", "input": "@a", "output": [["ATKEYWORD", "a"]]}, {"description": "@A", "input": "@A", "output": [["ATKEYWORD", "A"]]}, {"description": "@abc", "input": "@abc", "output": [["ATKEYWORD", "abc"]]}, {"description": "@-", "input": "@-", "output": [["DELIM", "@"], ["DELIM", "-"]]}, {"description": "@-a", "input": "@-a", "output": [["ATKEYWORD", "-a"]]}, {"description": "@1", "input": "@1", "output": [["DELIM", "@"], ["NUMBER", "1"]]}, {"description": "@-1", "input": "@-1", "output": [["DELIM", "@"], ["DELIM", "-"], ["NUMBER", "1"]]}, {"description": "@\\41", "input": "@\\41", "output": [["ATKEYWORD", "A"]]}, {"description": "@\\61 b", "input": "@\\61 b", "output": [["ATKEYWORD", "ab"]]}, {"description": "@\\61 b", "input": "@\\61 b", "output": [["ATKEYWORD", "a"], ["S"], ["IDENT", "b"]]}, {"description": "@a\\41", "input": "@aA", "output": [["ATKEYWORD", "aA"]]}, {"description": "@a\\4e00X", "input": "@a\\4e00X", "output": [["ATKEYWORD", "a\u4e00X"]]}, {"description": "\"", "input": "\"", "output": [["INVALID", ""]]}, {"description": "'", "input": "'", "output": [["INVALID", ""]]}, {"description": "\"\"", "input": "\"\"", "output": [["STRING", ""]]}, {"description": "''", "input": "''", "output": [["STRING", ""]]}, {"description": "\"a", "input": "\"a", "output": [["INVALID", "a"]]}, {"description": "'a", "input": "'a", "output": [["INVALID", "a"]]}, {"description": "'a'", "input": "'a'", "output": [["STRING", "a"]]}, {"description": "\"a\"", "input": "\"a\"", "output": [["STRING", "a"]]}, {"description": "'a'", "input": "'a'", "output": [["STRING", "a"]]}, {"description": "\"a'", "input": "\"a'", "output": [["INVALID", "a'"]]}, {"description": "'a\"", "input": "'a\"", "output": [["INVALID", "a\""]]}, {"description": "\"A", "input": "\"A", "output": [["INVALID", "A"]]}, {"description": "'A", "input": "'A", "output": [["INVALID", "A"]]}, {"description": "\"A\"", "input": "\"A\"", "output": [["STRING", "A"]]}, {"description": "'A'", "input": "'A'", "output": [["STRING", "A"]]}, {"description": "\"A'", "input": "\"A'", "output": [["INVALID", "A'"]]}, {"description": "'A\"", "input": "'A\"", "output": [["INVALID", "A\""]]}, {"description": "\"abc", "input": "\"abc", "output": [["INVALID", "abc"]]}, {"description": "'abc", "input": "'abc", "output": [["INVALID", "abc"]]}, {"description": "\"abc\"", "input": "\"abc\"", "output": [["STRING", "abc"]]}, {"description": "'abc'", "input": "'abc'", "output": [["STRING", "abc"]]}, {"description": "\"a\\", "input": "\"a\\", "output": [["INVALID", "a"], ["DELIM", "\\"]]}, {"description": "'a\\", "input": "'a\\", "output": [["INVALID", "a"], ["DELIM", "\\"]]}, {"description": "\"\\", "input": "\"\\", "output": [["INVALID", ""], ["DELIM", "\\"]]}, {"description": "'\\", "input": "'\\", "output": [["INVALID", ""], ["DELIM", "\\"]]}, {"description": "\"a\\\"", "input": "\"a\\\"", "output": [["INVALID", "a\""]]}, {"description": "'a\\\"", "input": "'a\\\"", "output": [["INVALID", "a\""]]}, {"description": "'a\\'", "input": "'a\\'", "output": [["INVALID", "a'"]]}, {"description": "\"\\\"", "input": "\"\\\"", "output": [["INVALID", "\""]]}, {"description": "'\\\"", "input": "'\\\"", "output": [["INVALID", "\""]]}, {"description": "'\\'", "input": "'\\'", "output": [["INVALID", "'"]]}, {"description": "'\\\\\"", "input": "'\\\\\"", "output": [["INVALID", "\\\""]]}, {"description": "\"a\\'", "input": "\"a\\'", "output": [["INVALID", "a'"]]}, {"description": "\"a\\x", "input": "\"a\\x", "output": [["INVALID", "ax"]]}, {"description": "\"\\x", "input": "\"\\x", "output": [["INVALID", "x"]]}, {"description": "'\\x", "input": "'\\x", "output": [["INVALID", "x"]]}, {"description": "\"a\\x\"", "input": "\"a\\x\"", "output": [["STRING", "ax"]]}, {"description": "\"\\x \"", "input": "\"\\x \"", "output": [["STRING", "x "]]}, {"description": "\"a\\x \"", "input": "\"a\\x \"", "output": [["STRING", "ax "]]}, {"description": "\"\\31", "input": "\"\\31", "output": [["INVALID", "1"]]}, {"description": "\"a\\31", "input": "\"a\\31", "output": [["INVALID", "a1"]]}, {"description": "'a\\31", "input": "'a\\31", "output": [["INVALID", "a1"]]}, {"description": "\"\\31\"", "input": "\"\\31\"", "output": [["STRING", "1"]]}, {"description": "'\\31'", "input": "'\\31'", "output": [["STRING", "1"]]}, {"description": "\"a\\31\"", "input": "\"a\\31\"", "output": [["STRING", "a1"]]}, {"description": "\"\\31 ", "input": "\"\\31 ", "output": [["INVALID", "1"]]}, {"description": "\"a\\31 ", "input": "\"a\\31 ", "output": [["INVALID", "a1"]]}, {"description": "\"a\\31 \"", "input": "\"a\\31 \"", "output": [["STRING", "a1"]]}, {"description": "\"\\\u000D", "input": "\"\\\u000D", "output": [["INVALID", "\u000D"]]}, {"description": "'\\\u000D", "input": "'\\\u000D", "output": [["INVALID", "\u000D"]]}, {"description": "\"a\\\u000D", "input": "\"a\\\u000D", "output": [["INVALID", "a\u000D"]]}, {"description": "\"a\\\u000D\"", "input": "\"a\\\u000D\"", "output": [["STRING", "a\u000D"]]}, {"description": "'a\\\u000D'", "input": "'a\\\u000D'", "output": [["STRING", "a\u000D"]]}, {"description": "\"a\\\u000D\u000A", "input": "\"a\\\u000D\u000A", "output": [["INVALID", "a\u000D\u000A"]]}, {"description": "\"a\\\u000D\u000A\"", "input": "\"a\\\u000D\u000A\"", "output": [["STRING", "a\u000D\u000A"]]}, {"description": "\"a\\\u000D\\\u000A", "input": "\"a\\\u000D\\\u000A", "output": [["INVALID", "a\u000D\u000A"]]}, {"description": "'a\\\u000D\\\u000A", "input": "'a\\\u000D\\\u000A", "output": [["INVALID", "a\u000D\u000A"]]}, {"description": "\"a\\\u000D\\\u000A\"", "input": "\"a\\\u000D\\\u000A\"", "output": [["STRING", "a\u000D\u000A"]]}, {"description": "\"a\\\u000A", "input": "\"a\\\u000A", "output": [["INVALID", "a\u000A"]]}, {"description": "\"a\\\u000A\"", "input": "\"a\\\u000A\"", "output": [["STRING", "a\u000A"]]}, {"description": "'a\\\u000A'", "input": "'a\\\u000A'", "output": [["STRING", "a\u000A"]]}, {"description": "\"a\\\u000C", "input": "\"a\\\u000C", "output": [["INVALID", "a\u000C"]]}, {"description": "\"a\\\u000C\"", "input": "\"a\\\u000C\"", "output": [["STRING", "a\u000C"]]}, {"description": "'a\\\u000C'", "input": "'a\\\u000C'", "output": [["STRING", "a\u000C"]]}, {"description": "\"a\\\u000Daa", "input": "\"a\\\u000Daa", "output": [["INVALID", "a\u000Daa"]]}, {"description": "\"a\\\u000D\u000Ab", "input": "\"a\\\u000D\u000Ab", "output": [["INVALID", "a\u000D\u000Ab"]]}, {"description": "'a\\\u000D\u000Ab", "input": "'a\\\u000D\u000Ab", "output": [["INVALID", "a\u000D\u000Ab"]]}, {"description": "#", "input": "#", "output": [["DELIM", "#"]]}, {"description": "\\#", "input": "\\#", "output": [["IDENT", "#"]]}, {"description": "#a", "input": "#a", "output": [["HASH", "a"]]}, {"description": "#A", "input": "#A", "output": [["HASH", "A"]]}, {"description": "##", "input": "##", "output": [["DELIM", "#"], ["DELIM", "#"]]}, {"description": "#\\#", "input": "#\\#", "output": [["HASH", "#"]]}, {"description": "#\\X", "input": "#\\X", "output": [["HASH", "X"]]}, {"description": "#\\a", "input": "#\\a", "output": [["HASH", "\u000a"]]}, {"description": "#\\-", "input": "#\\-", "output": [["HASH", "-"]]}, {"description": "#_", "input": "#_", "output": [["HASH", "_"]]}, {"description": "#0", "input": "#0", "output": [["HASH", "0"]]}, {"description": "#123456", "input": "#123456", "output": [["HASH", "123456"]]}, {"description": "#abc", "input": "#abc", "output": [["HASH", "abc"]]}, {"description": "#\\4e00XYZ", "input": "#\\4e00XYZ", "output": [["HASH", "\u4e00XYZ"]]}, {"description": "#\u4e00", "input": "#\u4e00", "output": [["HASH", "\u4e00"]]}, {"description": "#a\\100", "input": "#a\\100", "output": [["HASH", "a\u0100"]]}, {"description": "#1234567890", "input": "#1234567890", "output": [["HASH", "1234567890"]]}, {"description": "#1a", "input": "#1a", "output": [["HASH", "1a"]]}, {"description": "#\\a b", "input": "#\\a b", "output": [["HASH", "\u000ab"]]}, {"description": "#!", "input": "#!", "output": [["DELIM", "#"], ["DELIM", "!"]]}, {"description": "#-", "input": "#-", "output": [["HASH", "-"]]}, {"description": "#--", "input": "#--", "output": [["HASH", "--"]]}, {"description": "#-->", "input": "#-->", "output": [["HASH", "--"], ["GREATER"]]}, {"description": "0", "input": "0", "output": [["NUMBER", "0"]]}, {"description": "1", "input": "1", "output": [["NUMBER", "1"]]}, {"description": "111", "input": "111", "output": [["NUMBER", "111"]]}, {"description": "0001", "input": "0001", "output": [["NUMBER", "0001"]]}, {"description": "1e", "input": "1e", "output": [["DIMENSION", "1", "e"]]}, {"description": "1e4", "input": "1e4", "output": [["DIMENSION", "1", "e4"]]}, {"description": "1n+2", "input": "1n+2", "output": [["DIMENSION", "1", "n"], ["PLUS"], ["NUMBER", "2"]]}, {"description": "0.", "input": "0.", "output": [["NUMBER", "0"], ["DELIM", "."]]}, {"description": "1.", "input": "1.", "output": [["NUMBER", "1"], ["DELIM", "."]]}, {"description": "144.", "input": "144.", "output": [["NUMBER", "144"], ["DELIM", "."]]}, {"description": "0.1", "input": "0.1", "output": [["NUMBER", "0.1"]]}, {"description": "1.1", "input": "1.1", "output": [["NUMBER", "1.1"]]}, {"description": "0.1000", "input": "0.1000", "output": [["NUMBER", "0.1000"]]}, {"description": "0.0001", "input": "0.0001", "output": [["NUMBER", "0.0001"]]}, {"description": ".", "input": ".", "output": [["DELIM", "."]]}, {"description": ".1", "input": ".1", "output": [["NUMBER", "0.1"]]}, {"description": ".0", "input": ".0", "output": [["NUMBER", "0.0"]]}, {"description": ".1234", "input": ".1234", "output": [["NUMBER", "0.1234"]]}, {"description": ".1ab", "input": ".1ab", "output": [["DIMENSION", "0.1", "ab"]]}, {"description": ".1.2", "input": ".1.2", "output": [["NUMBER", "0.1"], ["NUMBER", "0.2"]]}, {"description": ".a", "input": ".a", "output": [["DELIM", "."], ["IDENT", "a"]]}, {"description": "3.11.66", "input": "3.11.66", "output": [["NUMBER", "3.11"], ["NUMBER", "0.66"]]}, {"description": "..", "input": "..", "output": [["DELIM", "."], ["DELIM", "."]]}, {"description": "...", "input": "...", "output": [["DELIM", "."], ["DELIM", "."], ["DELIM", "."]]}, {"description": "...1", "input": "...1", "output": [["DELIM", "."], ["DELIM", "."], ["NUMBER", "0.1"]]}, {"description": "...a", "input": "...a", "output": [["DELIM", "."], ["DELIM", "."], ["DELIM", "."], ["IDENT", "a"]]}, {"description": "+0", "input": "+0", "output": [["PLUS"], ["NUMBER", "0"]]}, {"description": "+1", "input": "+1", "output": [["PLUS"], ["NUMBER", "1"]]}, {"description": "+1.15", "input": "+1.15", "output": [["PLUS"], ["NUMBER", "1.15"]]}, {"description": "-.55", "input": "-.55", "output": [["DELIM", "-"], ["NUMBER", "0.55"]]}, {"description": "-0", "input": "-0", "output": [["DELIM", "-"], ["NUMBER", "0"]]}, {"description": "-0.2", "input": "-0.2", "output": [["DELIM", "-"], ["NUMBER", "0.2"]]}, {"description": "-.1", "input": "-.1", "output": [["DELIM", "-"], ["NUMBER", "0.1"]]}, {"description": "+-0", "input": "+-0", "output": [["PLUS"], ["DELIM", "-"], ["NUMBER", "0"]]}, {"description": "a-0", "input": "a-0", "output": [["IDENT", "a-0"]]}, {"description": "@a-0", "input": "@a-0", "output": [["ATKEYWORD", "a-0"]]}, {"description": "#a-0", "input": "#a-0", "output": [["HASH", "a-0"]]}, {"description": "