// Based on tokens.js
-// 2009-05-17
-// (c) 2006 Douglas Crockford
-
-// Produce an array of simple token objects from a string.
-// A simple token object contains these members:
-// type: 'name', 'string', 'number', 'operator'
-// value: string or number value of the token
-// from: index of first character of the token
-// to: index of the last character + 1
-
-// Comments of the ; type are ignored.
-
-// Operators are by default single characters. Multicharacter
-// operators can be made by supplying a string of prefix and
-// suffix characters.
-// characters. For example,
-// '<>+-&', '=>&:'
-// will match any of these:
-// <= >> >>> <> >= +: -: &: &&: &&
-
-
-
-String.prototype.tokens = function (prefix, suffix) {
+// 2009-05-17
+// (c) 2006 Douglas Crockford
+
+// Produce an array of simple token objects from a string.
+// A simple token object contains these members:
+// type: 'name', 'string', 'number', 'operator'
+// value: string or number value of the token
+// from: index of first character of the token
+// to: index of the last character + 1
+
+// Comments of the ; type are ignored.
+
+// Operators are by default single characters. Multicharacter
+// operators can be made by supplying a string of multi_start and
+// multi_continue characters.
+// characters. For example,
+// '<>+-&', '=>&:'
+// will match any of these:
+// <= >> >>> <> >= +: -: &: &&: &&
+
+/*jslint onevar: false
+ */
+
+String.prototype.tokens = function (multi_start, multi_continue) {
var c; // The current character.
var from; // The index of the start of the token.
var i = 0; // The index of the current character.
return;
}
-// If prefix and suffix strings are not provided, supply defaults.
+// If multi_start and multi_continue strings are not provided, supply defaults.
- if (typeof prefix !== 'string') {
- prefix = '';
+ if (typeof multi_start !== 'string') {
+ multi_start = '';
}
- if (typeof suffix !== 'string') {
- suffix = '';
+ if (typeof multi_continue !== 'string') {
+ multi_continue = '';
}
for (;;) {
c = this.charAt(i);
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
- (c >= '0' && c <= '9') || c === '_') {
+ (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/') {
+ str += c;
+ i += 1;
+ } else if (c === '?' || c === '!') {
+ // should only be terminal
str += c;
i += 1;
- } else {
+ c = this.charAt(i);
+ // make sure next character is not an identifier
+ if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+ (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/' || c === '?' || c === '!') {
+ str += c;
+ i += 1;
+ make('name', str).error("Bad identifier");
+ }
+ } else {
break;
}
}
// Make sure the next character is not a letter.
- if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_') {
+ if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c === '_') {
str += c;
i += 1;
make('number', str).error("Bad number");
// comment.
- } else if (c === ';') {
+ } else if (c === ';') {
for (;;) {
c = this.charAt(i);
if (c === '\n' || c === '\r' || c === '') {
// multi-char operator.
- } else if (prefix.indexOf(c) >= 0) {
+ } else if (multi_start.indexOf(c) >= 0) {
str = c;
i += 1;
while (i < length) {
c = this.charAt(i);
- if (suffix.indexOf(c) < 0) {
+ if (multi_continue.indexOf(c) < 0) {
break;
}
str += c;