X-Git-Url: http://lambda.jimpryor.net/git/gitweb.cgi?p=lambda.git;a=blobdiff_plain;f=code%2Ftokens.js;h=c6630fa54a7cb98bd9ee0c4c2c71c0ec67540a19;hp=98277a0f6144dd6a0b4c0b444233596515fe94ae;hb=45bfd0c44c531bd94e00b5a11285691c48062157;hpb=42859c95fe5548d1fc3c3f5b03980b13a35ebf35

diff --git a/code/tokens.js b/code/tokens.js
index 98277a0f..c6630fa5 100644
--- a/code/tokens.js
+++ b/code/tokens.js
@@ -1,27 +1,28 @@
 // Based on tokens.js
-// 		2009-05-17
-// 		(c) 2006 Douglas Crockford
-
-// 		Produce an array of simple token objects from a string.
-// 		A simple token object contains these members:
-// 		     type: 'name', 'string', 'number', 'operator'
-// 		     value: string or number value of the token
-// 		     from: index of first character of the token
-// 		     to: index of the last character + 1
-
-// 		Comments of the ; type are ignored.
-
-// 		Operators are by default single characters. Multicharacter
-// 		operators can be made by supplying a string of prefix and
-// 		suffix characters.
-// 		characters. For example,
-// 		     '<>+-&', '=>&:'
-// 		will match any of these:
-// 		     <=  >>  >>>  <>  >=  +: -: &: &&: &&
-
-
-
-String.prototype.tokens = function (prefix, suffix) {
+//      2009-05-17
+//      (c) 2006 Douglas Crockford
+
+//      Produce an array of simple token objects from a string.
+//      A simple token object contains these members:
+//           type: 'name', 'string', 'number', 'operator'
+//           value: string or number value of the token
+//           from: index of first character of the token
+//           to: index of the last character + 1
+
+//      Comments of the ; type are ignored.
+
+//      Operators are by default single characters. Multicharacter
+//      operators can be made by supplying a string of multi_start and
+//      multi_continue characters.
+//      characters. For example,
+//           '<>+-&', '=>&:'
+//      will match any of these:
+//           <=  >>  >>>  <>  >=  +: -: &: &&: &&
+
+/*jslint onevar: false
+ */
+
+String.prototype.tokens = function (multi_start, multi_continue) {
     var c;                      // The current character.
     var from;                   // The index of the start of the token.
     var i = 0;                  // The index of the current character.
@@ -50,13 +51,13 @@ String.prototype.tokens = function (prefix, suffix) {
         return;
     }
 
-// If prefix and suffix strings are not provided, supply defaults.
+// If multi_start and multi_continue strings are not provided, supply defaults.
 
-    if (typeof prefix !== 'string') {
-		prefix = '';
+    if (typeof multi_start !== 'string') {
+        multi_start = '';
     }
-    if (typeof suffix !== 'string') {
-		suffix = '';
+    if (typeof multi_continue !== 'string') {
+        multi_continue = '';
     }
 
 
@@ -80,10 +81,22 @@ String.prototype.tokens = function (prefix, suffix) {
             for (;;) {
                 c = this.charAt(i);
                 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
-                        (c >= '0' && c <= '9') || c === '_') {
+                        (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/') {
+                    str += c;
+                    i += 1;
+				} else if (c === '?' || c === '!') {
+				// should only be terminal
                     str += c;
                     i += 1;
-                } else {
+					c = this.charAt(i);
+				// make sure next character is not an identifier
+					if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+						(c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/' || c === '?' || c === '!') {
+						str += c;
+						i += 1;
+						make('name', str).error("Bad identifier");
+					}
+				} else {
                     break;
                 }
             }
@@ -111,7 +124,7 @@ String.prototype.tokens = function (prefix, suffix) {
 
 // Make sure the next character is not a letter.
 
-            if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_') {
+            if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c === '_') {
                 str += c;
                 i += 1;
                 make('number', str).error("Bad number");
@@ -129,7 +142,7 @@ String.prototype.tokens = function (prefix, suffix) {
 
 // comment.
 
-		} else if (c === ';') {
+        } else if (c === ';') {
             for (;;) {
                 c = this.charAt(i);
                 if (c === '\n' || c === '\r' || c === '') {
@@ -140,12 +153,12 @@ String.prototype.tokens = function (prefix, suffix) {
 
 // multi-char operator.
 
-        } else if (prefix.indexOf(c) >= 0) {
+        } else if (multi_start.indexOf(c) >= 0) {
             str = c;
             i += 1;
             while (i < length) {
                 c = this.charAt(i);
-                if (suffix.indexOf(c) < 0) {
+                if (multi_continue.indexOf(c) < 0) {
                     break;
                 }
                 str += c;