client: remove whitespace around tokens: Fixes #292

1 files changed, 2 insertions, 0 deletions

diff --git a/src/client/textanalysis.js b/src/client/textanalysis.js
index 138425a5..e0b57dbc 100644
--- a/src/client/textanalysis.js
+++ b/src/client/textanalysis.js
@@ -101,6 +101,8 @@ function tokenize(text, node_token, quote)
         prev_whitespace = prev === null || prev === ' ' || prev === '\t';
     }
     next();
+    // Remove whitespace around tokens. Can be done with lookahead but less code this way
+    tokens.forEach(function (obj) { obj.token = obj.token.trim(); });
     return tokens;
 }