diff options
Diffstat (limited to 'src/client')
| -rw-r--r-- | src/client/textanalysis.js | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/src/client/textanalysis.js b/src/client/textanalysis.js index 138425a5..e0b57dbc 100644 --- a/src/client/textanalysis.js +++ b/src/client/textanalysis.js @@ -101,6 +101,8 @@ function tokenize(text, node_token, quote) prev_whitespace = prev === null || prev === ' ' || prev === '\t'; } next(); + // Remove whitespace around tokens. Can be done with lookahead but less code this way + tokens.forEach(function (obj) { obj.token = obj.token.trim(); }); return tokens; } |