// Based on tokens.js
-// 2009-05-17
-// (c) 2006 Douglas Crockford
+// 2009-05-17
+// (c) 2006 Douglas Crockford
-// Produce an array of simple token objects from a string.
-// A simple token object contains these members:
-// type: 'name', 'string', 'number', 'operator'
-// value: string or number value of the token
-// from: index of first character of the token
-// to: index of the last character + 1
+// Produce an array of simple token objects from a string.
+// A simple token object contains these members:
+// type: 'name', 'string', 'number', 'operator'
+// value: string or number value of the token
+// from: index of first character of the token
+// to: index of the last character + 1
-// Comments of the ; type are ignored.
-
-// Operators are by default single characters. Multicharacter
-// operators can be made by supplying a string of prefix and
-// suffix characters.
-// characters. For example,
-// '<>+-&', '=>&:'
-// will match any of these:
-// <= >> >>> <> >= +: -: &: &&: &&
+// Comments of the ; type are ignored.
+// Operators are by default single characters. Multicharacter
+// operators can be made by supplying a string of prefix and
+// suffix characters.
+// characters. For example,
+// '<>+-&', '=>&:'
+// will match any of these:
+// <= >> >>> <> >= +: -: &: &&: &&
+/*jslint onevar: false
+ */
String.prototype.tokens = function (prefix, suffix) {
var c; // The current character.
// If prefix and suffix strings are not provided, supply defaults.
if (typeof prefix !== 'string') {
- prefix = '';
+ prefix = '';
}
if (typeof suffix !== 'string') {
- suffix = '';
+ suffix = '';
}
for (;;) {
c = this.charAt(i);
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
- (c >= '0' && c <= '9') || c === '_') {
+ (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/') {
+ str += c;
+ i += 1;
+ } else if (c === '?' || c === '!') {
+ // should only be terminal
str += c;
i += 1;
- } else {
+ c = this.charAt(i);
+ // make sure next character is not an identifier
+ if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+ (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/' || c === '?' || c === '!') {
+ str += c;
+ i += 1;
+ make('name', str).error("Bad identifier");
+ }
+ } else {
break;
}
}
// Make sure the next character is not a letter.
- if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_') {
+ if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c === '_') {
str += c;
i += 1;
make('number', str).error("Bad number");
// comment.
- } else if (c === ';') {
+ } else if (c === ';') {
for (;;) {
c = this.charAt(i);
if (c === '\n' || c === '\r' || c === '') {