3 // (c) 2006 Douglas Crockford
5 // Produce an array of simple token objects from a string.
6 // A simple token object contains these members:
7 // type: 'name', 'string', 'number', 'operator'
8 // value: string or number value of the token
9 // from: index of first character of the token
10 // to: index of the last character + 1
12 // Comments of the ; type are ignored.
14 // Operators are by default single characters. Multicharacter
15 // operators can be made by supplying a string of prefix and
17 // characters. For example,
19 // will match any of these:
20 // <= >> >>> <> >= +: -: &: &&: &&
24 String.prototype.tokens = function (prefix, suffix) {
25 var c; // The current character.
26 var from; // The index of the start of the token.
27 var i = 0; // The index of the current character.
28 var length = this.length;
29 var n; // The number value.
30 var q; // The quote character.
31 var str; // The string value.
33 var result = []; // An array to hold the results.
35 var make = function (type, value) {
37 // Make a token object.
47 // Begin tokenization. If the source string is empty, return nothing.
53 // If prefix and suffix strings are not provided, supply defaults.
55 if (typeof prefix !== 'string') {
58 if (typeof suffix !== 'string') {
63 // Loop through this text, one character at a time.
77 } else if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z') {
82 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
83 (c >= '0' && c <= '9') || c === '_') {
90 result.push(make('name', str));
94 // A number cannot start with a decimal point. It must start with a digit,
97 } else if (c >= '0' && c <= '9') {
101 // Look for more digits.
105 if (c < '0' || c > '9') {
112 // Make sure the next character is not a letter.
114 if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_') {
117 make('number', str).error("Bad number");
120 // Convert the string value to a number. If it is finite, then it is a good
125 result.push(make('number', n));
127 make('number', str).error("Bad number");
132 } else if (c === ';') {
135 if (c === '\n' || c === '\r' || c === '') {
141 // multi-char operator.
143 } else if (prefix.indexOf(c) >= 0) {
148 if (suffix.indexOf(c) < 0) {
154 result.push(make('operator', str));
156 // single-character operator.
160 result.push(make('operator', c));