edit

[lambda.git] / code / tokens.js

diff --git a/code/tokens.js b/code/tokens.js
index 98277a0..ddf1e89 100644 (file)
--- a/code/tokens.js
+++ b/code/tokens.js
@@ -1,25 +1,26 @@
 // Based on tokens.js
-//             2009-05-17
-//             (c) 2006 Douglas Crockford
+//      2009-05-17
+//      (c) 2006 Douglas Crockford
 
-//             Produce an array of simple token objects from a string.
-//             A simple token object contains these members:
-//                  type: 'name', 'string', 'number', 'operator'
-//                  value: string or number value of the token
-//                  from: index of first character of the token
-//                  to: index of the last character + 1
+//      Produce an array of simple token objects from a string.
+//      A simple token object contains these members:
+//           type: 'name', 'string', 'number', 'operator'
+//           value: string or number value of the token
+//           from: index of first character of the token
+//           to: index of the last character + 1
 
-//             Comments of the ; type are ignored.
-
-//             Operators are by default single characters. Multicharacter
-//             operators can be made by supplying a string of prefix and
-//             suffix characters.
-//             characters. For example,
-//                  '<>+-&', '=>&:'
-//             will match any of these:
-//                  <=  >>  >>>  <>  >=  +: -: &: &&: &&
+//      Comments of the ; type are ignored.
 
+//      Operators are by default single characters. Multicharacter
+//      operators can be made by supplying a string of prefix and
+//      suffix characters.
+//      characters. For example,
+//           '<>+-&', '=>&:'
+//      will match any of these:
+//           <=  >>  >>>  <>  >=  +: -: &: &&: &&
 
+/*jslint onevar: false
+ */
 
 String.prototype.tokens = function (prefix, suffix) {
     var c;                      // The current character.
@@ -53,10 +54,10 @@ String.prototype.tokens = function (prefix, suffix) {
 // If prefix and suffix strings are not provided, supply defaults.
 
     if (typeof prefix !== 'string') {
-               prefix = '';
+        prefix = '';
     }
     if (typeof suffix !== 'string') {
-               suffix = '';
+        suffix = '';
     }
 
 
@@ -80,10 +81,21 @@ String.prototype.tokens = function (prefix, suffix) {
             for (;;) {
                 c = this.charAt(i);
                 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
-                        (c >= '0' && c <= '9') || c === '_') {
+                        (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/') {
+                    str += c;
+                    i += 1;
+                               } else if (c === '?' || c === '!') {
+                               // should only be terminal
                     str += c;
                     i += 1;
-                } else {
+                               // make sure next character is not an identifier
+                                       if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+                                               (c >= '0' && c <= '9') || c === '_' || c === '-' || c === '/' || c === '?' || c === '!') {
+                                               str += c;
+                                               i += 1;
+                                               make('name', str).error("Bad identifier");
+                                       }
+                               } else {
                     break;
                 }
             }
@@ -111,7 +123,7 @@ String.prototype.tokens = function (prefix, suffix) {
 
 // Make sure the next character is not a letter.
 
-            if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '_') {
+            if (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c === '_') {
                 str += c;
                 i += 1;
                 make('number', str).error("Bad number");
@@ -129,7 +141,7 @@ String.prototype.tokens = function (prefix, suffix) {
 
 // comment.
 
-               } else if (c === ';') {
+        } else if (c === ';') {
             for (;;) {
                 c = this.charAt(i);
                 if (c === '\n' || c === '\r' || c === '') {