X-Git-Url: http://lambda.jimpryor.net/git/gitweb.cgi?p=lambda.git;a=blobdiff_plain;f=code%2Ftokens.js;h=c6630fa54a7cb98bd9ee0c4c2c71c0ec67540a19;hp=53639a8b6ade4449a75725e9e94ce3af30c63a26;hb=7936567be1fea69bb7e1bb299bd3d1616b6e2dcc;hpb=71bfc3a4fcf9ae9eddc1854bce2ff6d5f6869644

diff --git a/code/tokens.js b/code/tokens.js
index 53639a8b..c6630fa5 100644
--- a/code/tokens.js
+++ b/code/tokens.js
@@ -12,8 +12,8 @@
 //      Comments of the ; type are ignored.
 
 //      Operators are by default single characters. Multicharacter
-//      operators can be made by supplying a string of prefix and
-//      suffix characters.
+//      operators can be made by supplying a string of multi_start and
+//      multi_continue characters.
 //      characters. For example,
 //           '<>+-&', '=>&:'
 //      will match any of these:
@@ -22,7 +22,7 @@
 /*jslint onevar: false
  */
 
-String.prototype.tokens = function (prefix, suffix) {
+String.prototype.tokens = function (multi_start, multi_continue) {
     var c;                      // The current character.
     var from;                   // The index of the start of the token.
     var i = 0;                  // The index of the current character.
@@ -51,13 +51,13 @@ String.prototype.tokens = function (prefix, suffix) {
         return;
     }
 
-// If prefix and suffix strings are not provided, supply defaults.
+// If multi_start and multi_continue strings are not provided, supply defaults.
 
-    if (typeof prefix !== 'string') {
-        prefix = '';
+    if (typeof multi_start !== 'string') {
+        multi_start = '';
     }
-    if (typeof suffix !== 'string') {
-        suffix = '';
+    if (typeof multi_continue !== 'string') {
+        multi_continue = '';
     }
 
 
@@ -81,16 +81,17 @@ String.prototype.tokens = function (prefix, suffix) {
             for (;;) {
                 c = this.charAt(i);
                 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
-                        (c >= '0' && c <= '9') || c === '_' || c === '-') {
+                        (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/') {
                     str += c;
                     i += 1;
 				} else if (c === '?' || c === '!') {
 				// should only be terminal
                     str += c;
                     i += 1;
+					c = this.charAt(i);
 				// make sure next character is not an identifier
 					if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
-						(c >= '0' && c <= '9') || c === '_' || c === '-' || c === '?' || c === '!') {
+						(c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/' || c === '?' || c === '!') {
 						str += c;
 						i += 1;
 						make('name', str).error("Bad identifier");
@@ -152,12 +153,12 @@ String.prototype.tokens = function (prefix, suffix) {
 
 // multi-char operator.
 
-        } else if (prefix.indexOf(c) >= 0) {
+        } else if (multi_start.indexOf(c) >= 0) {
             str = c;
             i += 1;
             while (i < length) {
                 c = this.charAt(i);
-                if (suffix.indexOf(c) < 0) {
+                if (multi_continue.indexOf(c) < 0) {
                     break;
                 }
                 str += c;