From bb5bd7f158dc791d16cde816fa25dd0dd7f62ac4 Mon Sep 17 00:00:00 2001
From: Alon Levy <alon@pobox.com>
Date: Tue, 6 Jan 2015 21:41:51 +0200
Subject: textanalysis: new_tokenise: return node start/end

---
 src/client/textanalysis.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/client/textanalysis.js b/src/client/textanalysis.js
index 2c59bea1..67778b81 100644
--- a/src/client/textanalysis.js
+++ b/src/client/textanalysis.js
@@ -19,10 +19,12 @@ function new_tokenize(text, node_token, quote)
         inquote = false,
         prev = null,
         prev_whitespace = true,
+        start = 0,
         next = function() {
             if (token.length > 0) {
-                tokens.push(token.join(''));
+                tokens.push({start: start, end: i, token: token.join('')});
                 token = [];
+                start = i;
             }
         };
     for (i = 0 ; i < text.length; ++i) {
@@ -47,7 +49,7 @@ function new_tokenize(text, node_token, quote)
             break;
         default:
             if (c == node_token && prev_whitespace) {
-                tokens.push(node_token);
+                tokens.push({start: i, end: i + 1, token: node_token});
             } else {
                 token.push(c);
             }
-- 
cgit v1.3.1