diff options
| author | Alon Levy <alon@pobox.com> | 2015-02-11 14:45:41 +0200 |
|---|---|---|
| committer | Alon Levy <alon@pobox.com> | 2015-02-11 14:45:41 +0200 |
| commit | 6746a8fd949c401e5b2c1b00f3a7f69a3c209f36 (patch) | |
| tree | 1d31b27674c3c5694160f280375b8cbb70be7f1b /src/client/textanalysis.js | |
| parent | 0dc89aa6b16412b02d76151a777c2346409bb78f (diff) | |
client/textanalysis: sanitize quotes, fixes #293
Diffstat (limited to 'src/client/textanalysis.js')
| -rw-r--r-- | src/client/textanalysis.js | 13 |
1 files changed, 12 insertions, 1 deletions
diff --git a/src/client/textanalysis.js b/src/client/textanalysis.js index e0b57dbc..9a4cd412 100644 --- a/src/client/textanalysis.js +++ b/src/client/textanalysis.js @@ -28,6 +28,17 @@ function selectedType() return nodetypes[typeindex]; } +/* + * Place to do general cleanup of the text. Right now just replacing all quotation types + * with a single type, separately for double quotes and single quotes. + */ +function cleanup(text) +{ + return text + .replace(/[״”“„‟″‶]/, '"') // Convert typographic double quotes + .replace(/[`׳’‘‚‛′‵]/, "'"); // Convert typographic single quotes +} + /** * Tokenizer for input. * @@ -251,7 +262,7 @@ var textAnalyser = function (spec) { //Sentence Sequencing //Build the words and cuts the main elements - tokens = tokenize(newtext, '#', '"'); + tokens = tokenize(cleanup(newtext), '#', '"'); sentence = tokens.map(function (d) { return d.token; }); // build new node,link arrays in order of appearance |