Calculate begin and end position of a word in a document in JavaScript
I have a text document represented as an array
of sentences and for each sentence I have an array
of word tokens.
I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum
five times, I must get the right position in that sentence for each occurrence.
I wrote this function
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
but there is something wrong. The calculated characterOffsetBegin
and characterOffsetEnd
are wrong.
A document has the following structure
{
"sentences": [
"index": 0,
"text": "Lorem ipsum dolor sit amet,",
"tokens": [
"index": 1,
"word": "Lorem",
"characterOffsetBegin": 0,
"characterOffsetEnd": 5
,
"index": 2,
"word": "ipsum",
"characterOffsetBegin": 5,
"characterOffsetEnd": 10
,
...
]
,
"index": 1,
"text": " consectetur adipiscing elit,",
"tokens": [
"index": 1,
"word": "",
"characterOffsetBegin": 24,
"characterOffsetEnd": 24
,
...
This is an example using this method. Then calculateTokenBeginEnd
should then calculate the token begin and end indexes, while the text2SentencesTokens
created the document structure above. The calculateTokenBeginEnd
does not work as expected.
text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"
// to map a text to sentences and tokens
text2SentencesTokens = function(text)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
if (typeof(tokenP) == 'function')
return tokenP.apply(self, [item]);
else
return new Promise((resolve, _) =>
resolve(item);
);
);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
text2SentencesTokens(text)
.then(sentences =>
sentences = calculateTokenBeginEnd(sentences);
console.log(sentences);
)
[UPDATE]
According to the suggestion I rewrote the function as follows:
function calculateTokenBeginEnd(textArray)
var wordStart=-1;
for (var j = 0; j < textArray.sentences.length; ++j)
var sentence=textArray.sentences[j];
wordStart +=1;
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
var match = wordRegex.exec(sentence.text);
var previousEnd = 0;
wordStart += match.index + previousEnd;
var wordEnd = wordStart + word.length - 1;
token.characterOffsetBegin = wordStart;
token.characterOffsetEnd = wordEnd;
//calculateTokenBeginEnd
Any better solution to that?
[UPDATE 2]
I have updated the text2SentencesTokens
according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token
in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down
here will get the last matched positions:
"index": 2,
"word": "down",
"characterOffsetBegin": 70,
"characterOffsetEnd": 73
in the first occurrence of the first sentence, while it should have had the first matched position.
// convert a text document into a sentences array and a token array for each sentence
function text2SentencesTokens(text, tokenP)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
var escaped = token.replace(/[-[]()*+?.,\^$);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
text = "Steve down walks warily down the street downnWith the brim pulled way down low";
text2SentencesTokens(text)
.then(res => console.log(JSON.stringify(res, null, 2)))
javascript text-processing
add a comment |
I have a text document represented as an array
of sentences and for each sentence I have an array
of word tokens.
I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum
five times, I must get the right position in that sentence for each occurrence.
I wrote this function
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
but there is something wrong. The calculated characterOffsetBegin
and characterOffsetEnd
are wrong.
A document has the following structure
{
"sentences": [
"index": 0,
"text": "Lorem ipsum dolor sit amet,",
"tokens": [
"index": 1,
"word": "Lorem",
"characterOffsetBegin": 0,
"characterOffsetEnd": 5
,
"index": 2,
"word": "ipsum",
"characterOffsetBegin": 5,
"characterOffsetEnd": 10
,
...
]
,
"index": 1,
"text": " consectetur adipiscing elit,",
"tokens": [
"index": 1,
"word": "",
"characterOffsetBegin": 24,
"characterOffsetEnd": 24
,
...
This is an example using this method. Then calculateTokenBeginEnd
should then calculate the token begin and end indexes, while the text2SentencesTokens
created the document structure above. The calculateTokenBeginEnd
does not work as expected.
text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"
// to map a text to sentences and tokens
text2SentencesTokens = function(text)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
if (typeof(tokenP) == 'function')
return tokenP.apply(self, [item]);
else
return new Promise((resolve, _) =>
resolve(item);
);
);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
text2SentencesTokens(text)
.then(sentences =>
sentences = calculateTokenBeginEnd(sentences);
console.log(sentences);
)
[UPDATE]
According to the suggestion I rewrote the function as follows:
function calculateTokenBeginEnd(textArray)
var wordStart=-1;
for (var j = 0; j < textArray.sentences.length; ++j)
var sentence=textArray.sentences[j];
wordStart +=1;
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
var match = wordRegex.exec(sentence.text);
var previousEnd = 0;
wordStart += match.index + previousEnd;
var wordEnd = wordStart + word.length - 1;
token.characterOffsetBegin = wordStart;
token.characterOffsetEnd = wordEnd;
//calculateTokenBeginEnd
Any better solution to that?
[UPDATE 2]
I have updated the text2SentencesTokens
according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token
in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down
here will get the last matched positions:
"index": 2,
"word": "down",
"characterOffsetBegin": 70,
"characterOffsetEnd": 73
in the first occurrence of the first sentence, while it should have had the first matched position.
// convert a text document into a sentences array and a token array for each sentence
function text2SentencesTokens(text, tokenP)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
var escaped = token.replace(/[-[]()*+?.,\^$);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
text = "Steve down walks warily down the street downnWith the brim pulled way down low";
text2SentencesTokens(text)
.then(res => console.log(JSON.stringify(res, null, 2)))
javascript text-processing
add a comment |
I have a text document represented as an array
of sentences and for each sentence I have an array
of word tokens.
I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum
five times, I must get the right position in that sentence for each occurrence.
I wrote this function
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
but there is something wrong. The calculated characterOffsetBegin
and characterOffsetEnd
are wrong.
A document has the following structure
{
"sentences": [
"index": 0,
"text": "Lorem ipsum dolor sit amet,",
"tokens": [
"index": 1,
"word": "Lorem",
"characterOffsetBegin": 0,
"characterOffsetEnd": 5
,
"index": 2,
"word": "ipsum",
"characterOffsetBegin": 5,
"characterOffsetEnd": 10
,
...
]
,
"index": 1,
"text": " consectetur adipiscing elit,",
"tokens": [
"index": 1,
"word": "",
"characterOffsetBegin": 24,
"characterOffsetEnd": 24
,
...
This is an example using this method. Then calculateTokenBeginEnd
should then calculate the token begin and end indexes, while the text2SentencesTokens
created the document structure above. The calculateTokenBeginEnd
does not work as expected.
text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"
// to map a text to sentences and tokens
text2SentencesTokens = function(text)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
if (typeof(tokenP) == 'function')
return tokenP.apply(self, [item]);
else
return new Promise((resolve, _) =>
resolve(item);
);
);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
text2SentencesTokens(text)
.then(sentences =>
sentences = calculateTokenBeginEnd(sentences);
console.log(sentences);
)
[UPDATE]
According to the suggestion I rewrote the function as follows:
function calculateTokenBeginEnd(textArray)
var wordStart=-1;
for (var j = 0; j < textArray.sentences.length; ++j)
var sentence=textArray.sentences[j];
wordStart +=1;
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
var match = wordRegex.exec(sentence.text);
var previousEnd = 0;
wordStart += match.index + previousEnd;
var wordEnd = wordStart + word.length - 1;
token.characterOffsetBegin = wordStart;
token.characterOffsetEnd = wordEnd;
//calculateTokenBeginEnd
Any better solution to that?
[UPDATE 2]
I have updated the text2SentencesTokens
according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token
in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down
here will get the last matched positions:
"index": 2,
"word": "down",
"characterOffsetBegin": 70,
"characterOffsetEnd": 73
in the first occurrence of the first sentence, while it should have had the first matched position.
// convert a text document into a sentences array and a token array for each sentence
function text2SentencesTokens(text, tokenP)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
var escaped = token.replace(/[-[]()*+?.,\^$);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
text = "Steve down walks warily down the street downnWith the brim pulled way down low";
text2SentencesTokens(text)
.then(res => console.log(JSON.stringify(res, null, 2)))
javascript text-processing
I have a text document represented as an array
of sentences and for each sentence I have an array
of word tokens.
I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum
five times, I must get the right position in that sentence for each occurrence.
I wrote this function
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
but there is something wrong. The calculated characterOffsetBegin
and characterOffsetEnd
are wrong.
A document has the following structure
{
"sentences": [
"index": 0,
"text": "Lorem ipsum dolor sit amet,",
"tokens": [
"index": 1,
"word": "Lorem",
"characterOffsetBegin": 0,
"characterOffsetEnd": 5
,
"index": 2,
"word": "ipsum",
"characterOffsetBegin": 5,
"characterOffsetEnd": 10
,
...
]
,
"index": 1,
"text": " consectetur adipiscing elit,",
"tokens": [
"index": 1,
"word": "",
"characterOffsetBegin": 24,
"characterOffsetEnd": 24
,
...
This is an example using this method. Then calculateTokenBeginEnd
should then calculate the token begin and end indexes, while the text2SentencesTokens
created the document structure above. The calculateTokenBeginEnd
does not work as expected.
text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"
// to map a text to sentences and tokens
text2SentencesTokens = function(text)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
if (typeof(tokenP) == 'function')
return tokenP.apply(self, [item]);
else
return new Promise((resolve, _) =>
resolve(item);
);
);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
text2SentencesTokens(text)
.then(sentences =>
sentences = calculateTokenBeginEnd(sentences);
console.log(sentences);
)
[UPDATE]
According to the suggestion I rewrote the function as follows:
function calculateTokenBeginEnd(textArray)
var wordStart=-1;
for (var j = 0; j < textArray.sentences.length; ++j)
var sentence=textArray.sentences[j];
wordStart +=1;
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
var match = wordRegex.exec(sentence.text);
var previousEnd = 0;
wordStart += match.index + previousEnd;
var wordEnd = wordStart + word.length - 1;
token.characterOffsetBegin = wordStart;
token.characterOffsetEnd = wordEnd;
//calculateTokenBeginEnd
Any better solution to that?
[UPDATE 2]
I have updated the text2SentencesTokens
according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token
in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down
here will get the last matched positions:
"index": 2,
"word": "down",
"characterOffsetBegin": 70,
"characterOffsetEnd": 73
in the first occurrence of the first sentence, while it should have had the first matched position.
// convert a text document into a sentences array and a token array for each sentence
function text2SentencesTokens(text, tokenP)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
var escaped = token.replace(/[-[]()*+?.,\^$);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
text = "Steve down walks warily down the street downnWith the brim pulled way down low";
text2SentencesTokens(text)
.then(res => console.log(JSON.stringify(res, null, 2)))
text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"
// to map a text to sentences and tokens
text2SentencesTokens = function(text)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
if (typeof(tokenP) == 'function')
return tokenP.apply(self, [item]);
else
return new Promise((resolve, _) =>
resolve(item);
);
);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
text2SentencesTokens(text)
.then(sentences =>
sentences = calculateTokenBeginEnd(sentences);
console.log(sentences);
)
text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"
// to map a text to sentences and tokens
text2SentencesTokens = function(text)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
if (typeof(tokenP) == 'function')
return tokenP.apply(self, [item]);
else
return new Promise((resolve, _) =>
resolve(item);
);
);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');
token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);
token.characterOffsetEnd = currentText.length;
currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd
text2SentencesTokens(text)
.then(sentences =>
sentences = calculateTokenBeginEnd(sentences);
console.log(sentences);
)
// convert a text document into a sentences array and a token array for each sentence
function text2SentencesTokens(text, tokenP)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
var escaped = token.replace(/[-[]()*+?.,\^$);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
text = "Steve down walks warily down the street downnWith the brim pulled way down low";
text2SentencesTokens(text)
.then(res => console.log(JSON.stringify(res, null, 2)))
// convert a text document into a sentences array and a token array for each sentence
function text2SentencesTokens(text, tokenP)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token
var escaped = token.replace(/[-[]()*+?.,\^$);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens
text = "Steve down walks warily down the street downnWith the brim pulled way down low";
text2SentencesTokens(text)
.then(res => console.log(JSON.stringify(res, null, 2)))
javascript text-processing
javascript text-processing
edited Nov 20 '18 at 23:17
loretoparisi
asked Nov 13 '18 at 16:26
loretoparisiloretoparisi
7,66254772
7,66254772
add a comment |
add a comment |
1 Answer
1
active
oldest
votes
This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help
var word = "Lorem";
var reg = RegExp(word, 'g');
var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
var match;
console.log(sentence);
console.log(word);
while ((match = reg.exec(sentence)) !== null)
var wordStart = match.index;
var wordEnd = wordStart + word.length - 1;
console.log(wordStart + ' -start index');
console.log(word.length + ' -length of word');
console.log(wordEnd + ' -last character index, need to +1 to use with substring');
console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');
thanks this suggestion helped to rewrote the question adding a possibile solution.
– loretoparisi
Nov 13 '18 at 18:08
Also there is a problem with theRegExp
that is we should take in account multiple matches of atoken
within the currentsentence
.
– loretoparisi
Nov 13 '18 at 18:47
1
updated code to take into consideration for multiple matches.
– XTOTHEL
Nov 13 '18 at 19:10
hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.
– loretoparisi
Nov 20 '18 at 23:14
add a comment |
Your Answer
StackExchange.ifUsing("editor", function ()
StackExchange.using("externalEditor", function ()
StackExchange.using("snippets", function ()
StackExchange.snippets.init();
);
);
, "code-snippets");
StackExchange.ready(function()
var channelOptions =
tags: "".split(" "),
id: "1"
;
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function()
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled)
StackExchange.using("snippets", function()
createEditor();
);
else
createEditor();
);
function createEditor()
StackExchange.prepareEditor(
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader:
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
,
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
);
);
Sign up or log in
StackExchange.ready(function ()
StackExchange.helpers.onClickDraftSave('#login-link');
);
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function ()
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53285370%2fcalculate-begin-and-end-position-of-a-word-in-a-document-in-javascript%23new-answer', 'question_page');
);
Post as a guest
Required, but never shown
1 Answer
1
active
oldest
votes
1 Answer
1
active
oldest
votes
active
oldest
votes
active
oldest
votes
This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help
var word = "Lorem";
var reg = RegExp(word, 'g');
var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
var match;
console.log(sentence);
console.log(word);
while ((match = reg.exec(sentence)) !== null)
var wordStart = match.index;
var wordEnd = wordStart + word.length - 1;
console.log(wordStart + ' -start index');
console.log(word.length + ' -length of word');
console.log(wordEnd + ' -last character index, need to +1 to use with substring');
console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');
thanks this suggestion helped to rewrote the question adding a possibile solution.
– loretoparisi
Nov 13 '18 at 18:08
Also there is a problem with theRegExp
that is we should take in account multiple matches of atoken
within the currentsentence
.
– loretoparisi
Nov 13 '18 at 18:47
1
updated code to take into consideration for multiple matches.
– XTOTHEL
Nov 13 '18 at 19:10
hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.
– loretoparisi
Nov 20 '18 at 23:14
add a comment |
This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help
var word = "Lorem";
var reg = RegExp(word, 'g');
var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
var match;
console.log(sentence);
console.log(word);
while ((match = reg.exec(sentence)) !== null)
var wordStart = match.index;
var wordEnd = wordStart + word.length - 1;
console.log(wordStart + ' -start index');
console.log(word.length + ' -length of word');
console.log(wordEnd + ' -last character index, need to +1 to use with substring');
console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');
thanks this suggestion helped to rewrote the question adding a possibile solution.
– loretoparisi
Nov 13 '18 at 18:08
Also there is a problem with theRegExp
that is we should take in account multiple matches of atoken
within the currentsentence
.
– loretoparisi
Nov 13 '18 at 18:47
1
updated code to take into consideration for multiple matches.
– XTOTHEL
Nov 13 '18 at 19:10
hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.
– loretoparisi
Nov 20 '18 at 23:14
add a comment |
This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help
var word = "Lorem";
var reg = RegExp(word, 'g');
var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
var match;
console.log(sentence);
console.log(word);
while ((match = reg.exec(sentence)) !== null)
var wordStart = match.index;
var wordEnd = wordStart + word.length - 1;
console.log(wordStart + ' -start index');
console.log(word.length + ' -length of word');
console.log(wordEnd + ' -last character index, need to +1 to use with substring');
console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');
This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help
var word = "Lorem";
var reg = RegExp(word, 'g');
var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
var match;
console.log(sentence);
console.log(word);
while ((match = reg.exec(sentence)) !== null)
var wordStart = match.index;
var wordEnd = wordStart + word.length - 1;
console.log(wordStart + ' -start index');
console.log(word.length + ' -length of word');
console.log(wordEnd + ' -last character index, need to +1 to use with substring');
console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');
var word = "Lorem";
var reg = RegExp(word, 'g');
var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
var match;
console.log(sentence);
console.log(word);
while ((match = reg.exec(sentence)) !== null)
var wordStart = match.index;
var wordEnd = wordStart + word.length - 1;
console.log(wordStart + ' -start index');
console.log(word.length + ' -length of word');
console.log(wordEnd + ' -last character index, need to +1 to use with substring');
console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');
var word = "Lorem";
var reg = RegExp(word, 'g');
var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
var match;
console.log(sentence);
console.log(word);
while ((match = reg.exec(sentence)) !== null)
var wordStart = match.index;
var wordEnd = wordStart + word.length - 1;
console.log(wordStart + ' -start index');
console.log(word.length + ' -length of word');
console.log(wordEnd + ' -last character index, need to +1 to use with substring');
console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');
edited Nov 13 '18 at 19:10
answered Nov 13 '18 at 16:52
XTOTHELXTOTHEL
2,0821210
2,0821210
thanks this suggestion helped to rewrote the question adding a possibile solution.
– loretoparisi
Nov 13 '18 at 18:08
Also there is a problem with theRegExp
that is we should take in account multiple matches of atoken
within the currentsentence
.
– loretoparisi
Nov 13 '18 at 18:47
1
updated code to take into consideration for multiple matches.
– XTOTHEL
Nov 13 '18 at 19:10
hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.
– loretoparisi
Nov 20 '18 at 23:14
add a comment |
thanks this suggestion helped to rewrote the question adding a possibile solution.
– loretoparisi
Nov 13 '18 at 18:08
Also there is a problem with theRegExp
that is we should take in account multiple matches of atoken
within the currentsentence
.
– loretoparisi
Nov 13 '18 at 18:47
1
updated code to take into consideration for multiple matches.
– XTOTHEL
Nov 13 '18 at 19:10
hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.
– loretoparisi
Nov 20 '18 at 23:14
thanks this suggestion helped to rewrote the question adding a possibile solution.
– loretoparisi
Nov 13 '18 at 18:08
thanks this suggestion helped to rewrote the question adding a possibile solution.
– loretoparisi
Nov 13 '18 at 18:08
Also there is a problem with the
RegExp
that is we should take in account multiple matches of a token
within the current sentence
.– loretoparisi
Nov 13 '18 at 18:47
Also there is a problem with the
RegExp
that is we should take in account multiple matches of a token
within the current sentence
.– loretoparisi
Nov 13 '18 at 18:47
1
1
updated code to take into consideration for multiple matches.
– XTOTHEL
Nov 13 '18 at 19:10
updated code to take into consideration for multiple matches.
– XTOTHEL
Nov 13 '18 at 19:10
hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.
– loretoparisi
Nov 20 '18 at 23:14
hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.
– loretoparisi
Nov 20 '18 at 23:14
add a comment |
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function ()
StackExchange.helpers.onClickDraftSave('#login-link');
);
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function ()
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53285370%2fcalculate-begin-and-end-position-of-a-word-in-a-document-in-javascript%23new-answer', 'question_page');
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function ()
StackExchange.helpers.onClickDraftSave('#login-link');
);
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function ()
StackExchange.helpers.onClickDraftSave('#login-link');
);
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function ()
StackExchange.helpers.onClickDraftSave('#login-link');
);
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown