Calculate begin and end position of a word in a document in JavaScript










1















I have a text document represented as an array of sentences and for each sentence I have an array of word tokens.



I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum five times, I must get the right position in that sentence for each occurrence.



I wrote this function



// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');


token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);

token.characterOffsetEnd = currentText.length;

currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd


but there is something wrong. The calculated characterOffsetBegin and characterOffsetEnd are wrong.
A document has the following structure



{
"sentences": [

"index": 0,
"text": "Lorem ipsum dolor sit amet,",
"tokens": [

"index": 1,
"word": "Lorem",
"characterOffsetBegin": 0,
"characterOffsetEnd": 5
,

"index": 2,
"word": "ipsum",
"characterOffsetBegin": 5,
"characterOffsetEnd": 10
,
...
]
,

"index": 1,
"text": " consectetur adipiscing elit,",
"tokens": [

"index": 1,
"word": "",
"characterOffsetBegin": 24,
"characterOffsetEnd": 24
,
...



This is an example using this method. Then calculateTokenBeginEnd should then calculate the token begin and end indexes, while the text2SentencesTokens created the document structure above. The calculateTokenBeginEnd does not work as expected.






text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"

// to map a text to sentences and tokens
text2SentencesTokens = function(text)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token

if (typeof(tokenP) == 'function')
return tokenP.apply(self, [item]);
else
return new Promise((resolve, _) =>
resolve(item);
);

);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens

// calculate begin and end to each token in a sentence
function calculateTokenBeginEnd(textArray)
var currentText = ;
textArray.sentences.forEach(function(sentence)
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
if (i > 0)
var thisBegin = token.characterOffsetBegin;
var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
if (thisBegin > previousEnd)
currentText.push(' ');


token.characterOffsetBegin = currentText.length;
for (var j = 0; j < word.length; ++j)
currentText.push(word[j]);

token.characterOffsetEnd = currentText.length;

currentText.push('n');
);
return textArray;
//calculateTokenBeginEnd

text2SentencesTokens(text)
.then(sentences =>
sentences = calculateTokenBeginEnd(sentences);
console.log(sentences);

)





[UPDATE]



According to the suggestion I rewrote the function as follows:



 function calculateTokenBeginEnd(textArray) 
var wordStart=-1;
for (var j = 0; j < textArray.sentences.length; ++j)
var sentence=textArray.sentences[j];
wordStart +=1;
for (var i = 0; i < sentence.tokens.length; ++i)
var token = sentence.tokens[i];
var word = token.word;
var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
var match = wordRegex.exec(sentence.text);
var previousEnd = 0;
wordStart += match.index + previousEnd;
var wordEnd = wordStart + word.length - 1;
token.characterOffsetBegin = wordStart;
token.characterOffsetEnd = wordEnd;


//calculateTokenBeginEnd


Any better solution to that?



[UPDATE 2]
I have updated the text2SentencesTokens according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down here will get the last matched positions:



 
"index": 2,
"word": "down",
"characterOffsetBegin": 70,
"characterOffsetEnd": 73



in the first occurrence of the first sentence, while it should have had the first matched position.






// convert a text document into a sentences array and a token array for each sentence
function text2SentencesTokens(text, tokenP)
var self = this;
return new Promise((resolve, _) =>
let sentences = text.split(/n+/g);
let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
return new Promise((resolve, _) =>
let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
let tokensP = tokens.map((token, tokenIndex) => // for each token
let item =
"index": (tokenIndex + 1),
"word": token

var escaped = token.replace(/[-[]()*+?.,\^$);
Promise.all(tokensP)
.then(res =>
resolve(
index: lineIndex,
text: sentence,
tokens: res
);
)
.catch(err => console.error(err))
);
);
Promise.all(sentencesP)
.then(res =>
resolve(
sentences: res
)
)
.catch(err => console.error(err))
);
//text2SentencesTokens

text = "Steve down walks warily down the street downnWith the brim pulled way down low";
text2SentencesTokens(text)
.then(res => console.log(JSON.stringify(res, null, 2)))












share|improve this question




























    1















    I have a text document represented as an array of sentences and for each sentence I have an array of word tokens.



    I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum five times, I must get the right position in that sentence for each occurrence.



    I wrote this function



    // calculate begin and end to each token in a sentence
    function calculateTokenBeginEnd(textArray)
    var currentText = ;
    textArray.sentences.forEach(function(sentence)
    for (var i = 0; i < sentence.tokens.length; ++i)
    var token = sentence.tokens[i];
    var word = token.word;
    if (i > 0)
    var thisBegin = token.characterOffsetBegin;
    var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
    if (thisBegin > previousEnd)
    currentText.push(' ');


    token.characterOffsetBegin = currentText.length;
    for (var j = 0; j < word.length; ++j)
    currentText.push(word[j]);

    token.characterOffsetEnd = currentText.length;

    currentText.push('n');
    );
    return textArray;
    //calculateTokenBeginEnd


    but there is something wrong. The calculated characterOffsetBegin and characterOffsetEnd are wrong.
    A document has the following structure



    {
    "sentences": [

    "index": 0,
    "text": "Lorem ipsum dolor sit amet,",
    "tokens": [

    "index": 1,
    "word": "Lorem",
    "characterOffsetBegin": 0,
    "characterOffsetEnd": 5
    ,

    "index": 2,
    "word": "ipsum",
    "characterOffsetBegin": 5,
    "characterOffsetEnd": 10
    ,
    ...
    ]
    ,

    "index": 1,
    "text": " consectetur adipiscing elit,",
    "tokens": [

    "index": 1,
    "word": "",
    "characterOffsetBegin": 24,
    "characterOffsetEnd": 24
    ,
    ...



    This is an example using this method. Then calculateTokenBeginEnd should then calculate the token begin and end indexes, while the text2SentencesTokens created the document structure above. The calculateTokenBeginEnd does not work as expected.






    text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"

    // to map a text to sentences and tokens
    text2SentencesTokens = function(text)
    var self = this;
    return new Promise((resolve, _) =>
    let sentences = text.split(/n+/g);
    let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
    return new Promise((resolve, _) =>
    let tokens = sentence.split(/s+/g);
    let tokensP = tokens.map((token, tokenIndex) => // for each token
    let item =
    "index": (tokenIndex + 1),
    "word": token

    if (typeof(tokenP) == 'function')
    return tokenP.apply(self, [item]);
    else
    return new Promise((resolve, _) =>
    resolve(item);
    );

    );
    Promise.all(tokensP)
    .then(res =>
    resolve(
    index: lineIndex,
    text: sentence,
    tokens: res
    );
    )
    .catch(err => console.error(err))
    );
    );
    Promise.all(sentencesP)
    .then(res =>
    resolve(
    sentences: res
    )
    )
    .catch(err => console.error(err))
    );
    //text2SentencesTokens

    // calculate begin and end to each token in a sentence
    function calculateTokenBeginEnd(textArray)
    var currentText = ;
    textArray.sentences.forEach(function(sentence)
    for (var i = 0; i < sentence.tokens.length; ++i)
    var token = sentence.tokens[i];
    var word = token.word;
    if (i > 0)
    var thisBegin = token.characterOffsetBegin;
    var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
    if (thisBegin > previousEnd)
    currentText.push(' ');


    token.characterOffsetBegin = currentText.length;
    for (var j = 0; j < word.length; ++j)
    currentText.push(word[j]);

    token.characterOffsetEnd = currentText.length;

    currentText.push('n');
    );
    return textArray;
    //calculateTokenBeginEnd

    text2SentencesTokens(text)
    .then(sentences =>
    sentences = calculateTokenBeginEnd(sentences);
    console.log(sentences);

    )





    [UPDATE]



    According to the suggestion I rewrote the function as follows:



     function calculateTokenBeginEnd(textArray) 
    var wordStart=-1;
    for (var j = 0; j < textArray.sentences.length; ++j)
    var sentence=textArray.sentences[j];
    wordStart +=1;
    for (var i = 0; i < sentence.tokens.length; ++i)
    var token = sentence.tokens[i];
    var word = token.word;
    var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
    var match = wordRegex.exec(sentence.text);
    var previousEnd = 0;
    wordStart += match.index + previousEnd;
    var wordEnd = wordStart + word.length - 1;
    token.characterOffsetBegin = wordStart;
    token.characterOffsetEnd = wordEnd;


    //calculateTokenBeginEnd


    Any better solution to that?



    [UPDATE 2]
    I have updated the text2SentencesTokens according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down here will get the last matched positions:



     
    "index": 2,
    "word": "down",
    "characterOffsetBegin": 70,
    "characterOffsetEnd": 73



    in the first occurrence of the first sentence, while it should have had the first matched position.






    // convert a text document into a sentences array and a token array for each sentence
    function text2SentencesTokens(text, tokenP)
    var self = this;
    return new Promise((resolve, _) =>
    let sentences = text.split(/n+/g);
    let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
    return new Promise((resolve, _) =>
    let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
    let tokensP = tokens.map((token, tokenIndex) => // for each token
    let item =
    "index": (tokenIndex + 1),
    "word": token

    var escaped = token.replace(/[-[]()*+?.,\^$);
    Promise.all(tokensP)
    .then(res =>
    resolve(
    index: lineIndex,
    text: sentence,
    tokens: res
    );
    )
    .catch(err => console.error(err))
    );
    );
    Promise.all(sentencesP)
    .then(res =>
    resolve(
    sentences: res
    )
    )
    .catch(err => console.error(err))
    );
    //text2SentencesTokens

    text = "Steve down walks warily down the street downnWith the brim pulled way down low";
    text2SentencesTokens(text)
    .then(res => console.log(JSON.stringify(res, null, 2)))












    share|improve this question


























      1












      1








      1


      1






      I have a text document represented as an array of sentences and for each sentence I have an array of word tokens.



      I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum five times, I must get the right position in that sentence for each occurrence.



      I wrote this function



      // calculate begin and end to each token in a sentence
      function calculateTokenBeginEnd(textArray)
      var currentText = ;
      textArray.sentences.forEach(function(sentence)
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      if (i > 0)
      var thisBegin = token.characterOffsetBegin;
      var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
      if (thisBegin > previousEnd)
      currentText.push(' ');


      token.characterOffsetBegin = currentText.length;
      for (var j = 0; j < word.length; ++j)
      currentText.push(word[j]);

      token.characterOffsetEnd = currentText.length;

      currentText.push('n');
      );
      return textArray;
      //calculateTokenBeginEnd


      but there is something wrong. The calculated characterOffsetBegin and characterOffsetEnd are wrong.
      A document has the following structure



      {
      "sentences": [

      "index": 0,
      "text": "Lorem ipsum dolor sit amet,",
      "tokens": [

      "index": 1,
      "word": "Lorem",
      "characterOffsetBegin": 0,
      "characterOffsetEnd": 5
      ,

      "index": 2,
      "word": "ipsum",
      "characterOffsetBegin": 5,
      "characterOffsetEnd": 10
      ,
      ...
      ]
      ,

      "index": 1,
      "text": " consectetur adipiscing elit,",
      "tokens": [

      "index": 1,
      "word": "",
      "characterOffsetBegin": 24,
      "characterOffsetEnd": 24
      ,
      ...



      This is an example using this method. Then calculateTokenBeginEnd should then calculate the token begin and end indexes, while the text2SentencesTokens created the document structure above. The calculateTokenBeginEnd does not work as expected.






      text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"

      // to map a text to sentences and tokens
      text2SentencesTokens = function(text)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      if (typeof(tokenP) == 'function')
      return tokenP.apply(self, [item]);
      else
      return new Promise((resolve, _) =>
      resolve(item);
      );

      );
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      // calculate begin and end to each token in a sentence
      function calculateTokenBeginEnd(textArray)
      var currentText = ;
      textArray.sentences.forEach(function(sentence)
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      if (i > 0)
      var thisBegin = token.characterOffsetBegin;
      var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
      if (thisBegin > previousEnd)
      currentText.push(' ');


      token.characterOffsetBegin = currentText.length;
      for (var j = 0; j < word.length; ++j)
      currentText.push(word[j]);

      token.characterOffsetEnd = currentText.length;

      currentText.push('n');
      );
      return textArray;
      //calculateTokenBeginEnd

      text2SentencesTokens(text)
      .then(sentences =>
      sentences = calculateTokenBeginEnd(sentences);
      console.log(sentences);

      )





      [UPDATE]



      According to the suggestion I rewrote the function as follows:



       function calculateTokenBeginEnd(textArray) 
      var wordStart=-1;
      for (var j = 0; j < textArray.sentences.length; ++j)
      var sentence=textArray.sentences[j];
      wordStart +=1;
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
      var match = wordRegex.exec(sentence.text);
      var previousEnd = 0;
      wordStart += match.index + previousEnd;
      var wordEnd = wordStart + word.length - 1;
      token.characterOffsetBegin = wordStart;
      token.characterOffsetEnd = wordEnd;


      //calculateTokenBeginEnd


      Any better solution to that?



      [UPDATE 2]
      I have updated the text2SentencesTokens according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down here will get the last matched positions:



       
      "index": 2,
      "word": "down",
      "characterOffsetBegin": 70,
      "characterOffsetEnd": 73



      in the first occurrence of the first sentence, while it should have had the first matched position.






      // convert a text document into a sentences array and a token array for each sentence
      function text2SentencesTokens(text, tokenP)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      var escaped = token.replace(/[-[]()*+?.,\^$);
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      text = "Steve down walks warily down the street downnWith the brim pulled way down low";
      text2SentencesTokens(text)
      .then(res => console.log(JSON.stringify(res, null, 2)))












      share|improve this question
















      I have a text document represented as an array of sentences and for each sentence I have an array of word tokens.



      I have to calculate the absolute begin and the end of the token position in the document for each token position, hence if in a sentence I have ipsum five times, I must get the right position in that sentence for each occurrence.



      I wrote this function



      // calculate begin and end to each token in a sentence
      function calculateTokenBeginEnd(textArray)
      var currentText = ;
      textArray.sentences.forEach(function(sentence)
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      if (i > 0)
      var thisBegin = token.characterOffsetBegin;
      var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
      if (thisBegin > previousEnd)
      currentText.push(' ');


      token.characterOffsetBegin = currentText.length;
      for (var j = 0; j < word.length; ++j)
      currentText.push(word[j]);

      token.characterOffsetEnd = currentText.length;

      currentText.push('n');
      );
      return textArray;
      //calculateTokenBeginEnd


      but there is something wrong. The calculated characterOffsetBegin and characterOffsetEnd are wrong.
      A document has the following structure



      {
      "sentences": [

      "index": 0,
      "text": "Lorem ipsum dolor sit amet,",
      "tokens": [

      "index": 1,
      "word": "Lorem",
      "characterOffsetBegin": 0,
      "characterOffsetEnd": 5
      ,

      "index": 2,
      "word": "ipsum",
      "characterOffsetBegin": 5,
      "characterOffsetEnd": 10
      ,
      ...
      ]
      ,

      "index": 1,
      "text": " consectetur adipiscing elit,",
      "tokens": [

      "index": 1,
      "word": "",
      "characterOffsetBegin": 24,
      "characterOffsetEnd": 24
      ,
      ...



      This is an example using this method. Then calculateTokenBeginEnd should then calculate the token begin and end indexes, while the text2SentencesTokens created the document structure above. The calculateTokenBeginEnd does not work as expected.






      text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"

      // to map a text to sentences and tokens
      text2SentencesTokens = function(text)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      if (typeof(tokenP) == 'function')
      return tokenP.apply(self, [item]);
      else
      return new Promise((resolve, _) =>
      resolve(item);
      );

      );
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      // calculate begin and end to each token in a sentence
      function calculateTokenBeginEnd(textArray)
      var currentText = ;
      textArray.sentences.forEach(function(sentence)
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      if (i > 0)
      var thisBegin = token.characterOffsetBegin;
      var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
      if (thisBegin > previousEnd)
      currentText.push(' ');


      token.characterOffsetBegin = currentText.length;
      for (var j = 0; j < word.length; ++j)
      currentText.push(word[j]);

      token.characterOffsetEnd = currentText.length;

      currentText.push('n');
      );
      return textArray;
      //calculateTokenBeginEnd

      text2SentencesTokens(text)
      .then(sentences =>
      sentences = calculateTokenBeginEnd(sentences);
      console.log(sentences);

      )





      [UPDATE]



      According to the suggestion I rewrote the function as follows:



       function calculateTokenBeginEnd(textArray) 
      var wordStart=-1;
      for (var j = 0; j < textArray.sentences.length; ++j)
      var sentence=textArray.sentences[j];
      wordStart +=1;
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      var wordRegex = new RegExp("\b(" + word + ")\b", "gi");
      var match = wordRegex.exec(sentence.text);
      var previousEnd = 0;
      wordStart += match.index + previousEnd;
      var wordEnd = wordStart + word.length - 1;
      token.characterOffsetBegin = wordStart;
      token.characterOffsetEnd = wordEnd;


      //calculateTokenBeginEnd


      Any better solution to that?



      [UPDATE 2]
      I have updated the text2SentencesTokens according to the proposed solution. The problem is that this solution will not work properly when there are multiple matches of the same token in one or more sentences, because it will overwrite the start and end positions with the last matched position, so the token down here will get the last matched positions:



       
      "index": 2,
      "word": "down",
      "characterOffsetBegin": 70,
      "characterOffsetEnd": 73



      in the first occurrence of the first sentence, while it should have had the first matched position.






      // convert a text document into a sentences array and a token array for each sentence
      function text2SentencesTokens(text, tokenP)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      var escaped = token.replace(/[-[]()*+?.,\^$);
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      text = "Steve down walks warily down the street downnWith the brim pulled way down low";
      text2SentencesTokens(text)
      .then(res => console.log(JSON.stringify(res, null, 2)))








      text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"

      // to map a text to sentences and tokens
      text2SentencesTokens = function(text)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      if (typeof(tokenP) == 'function')
      return tokenP.apply(self, [item]);
      else
      return new Promise((resolve, _) =>
      resolve(item);
      );

      );
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      // calculate begin and end to each token in a sentence
      function calculateTokenBeginEnd(textArray)
      var currentText = ;
      textArray.sentences.forEach(function(sentence)
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      if (i > 0)
      var thisBegin = token.characterOffsetBegin;
      var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
      if (thisBegin > previousEnd)
      currentText.push(' ');


      token.characterOffsetBegin = currentText.length;
      for (var j = 0; j < word.length; ++j)
      currentText.push(word[j]);

      token.characterOffsetEnd = currentText.length;

      currentText.push('n');
      );
      return textArray;
      //calculateTokenBeginEnd

      text2SentencesTokens(text)
      .then(sentences =>
      sentences = calculateTokenBeginEnd(sentences);
      console.log(sentences);

      )





      text = "Lorem ipsum dolor sit amet,n consectetur adipiscing elit,nsed do eiusmod tempor incididuntnut labore et dolore magna aliqua.nUt enim ad minim veniam,nquis nostrud exercitation ullamco laboris nisinut aliquip ex ea commodo consequat.nDuis aute irure dolor in reprehenderit in voluptate velit essencillum dolore eu fugiat nulla pariatur.nExcepteur sint occaecat cupidatat non proident,nLorem ipsum dolor sit amet etwas,nsunt in culpa qui officia deserunt mollit anim id est laborum"

      // to map a text to sentences and tokens
      text2SentencesTokens = function(text)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      if (typeof(tokenP) == 'function')
      return tokenP.apply(self, [item]);
      else
      return new Promise((resolve, _) =>
      resolve(item);
      );

      );
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      // calculate begin and end to each token in a sentence
      function calculateTokenBeginEnd(textArray)
      var currentText = ;
      textArray.sentences.forEach(function(sentence)
      for (var i = 0; i < sentence.tokens.length; ++i)
      var token = sentence.tokens[i];
      var word = token.word;
      if (i > 0)
      var thisBegin = token.characterOffsetBegin;
      var previousEnd = sentence.tokens[i - 1].characterOffsetEnd;
      if (thisBegin > previousEnd)
      currentText.push(' ');


      token.characterOffsetBegin = currentText.length;
      for (var j = 0; j < word.length; ++j)
      currentText.push(word[j]);

      token.characterOffsetEnd = currentText.length;

      currentText.push('n');
      );
      return textArray;
      //calculateTokenBeginEnd

      text2SentencesTokens(text)
      .then(sentences =>
      sentences = calculateTokenBeginEnd(sentences);
      console.log(sentences);

      )





      // convert a text document into a sentences array and a token array for each sentence
      function text2SentencesTokens(text, tokenP)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      var escaped = token.replace(/[-[]()*+?.,\^$);
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      text = "Steve down walks warily down the street downnWith the brim pulled way down low";
      text2SentencesTokens(text)
      .then(res => console.log(JSON.stringify(res, null, 2)))





      // convert a text document into a sentences array and a token array for each sentence
      function text2SentencesTokens(text, tokenP)
      var self = this;
      return new Promise((resolve, _) =>
      let sentences = text.split(/n+/g);
      let sentencesP = sentences.map((sentence, lineIndex) => // for each sentence
      return new Promise((resolve, _) =>
      let tokens = sentence.replace(/[\+;:?!»«><][)(,.‘'“”"]/g, '').split(/s+/g);
      let tokensP = tokens.map((token, tokenIndex) => // for each token
      let item =
      "index": (tokenIndex + 1),
      "word": token

      var escaped = token.replace(/[-[]()*+?.,\^$);
      Promise.all(tokensP)
      .then(res =>
      resolve(
      index: lineIndex,
      text: sentence,
      tokens: res
      );
      )
      .catch(err => console.error(err))
      );
      );
      Promise.all(sentencesP)
      .then(res =>
      resolve(
      sentences: res
      )
      )
      .catch(err => console.error(err))
      );
      //text2SentencesTokens

      text = "Steve down walks warily down the street downnWith the brim pulled way down low";
      text2SentencesTokens(text)
      .then(res => console.log(JSON.stringify(res, null, 2)))






      javascript text-processing






      share|improve this question















      share|improve this question













      share|improve this question




      share|improve this question








      edited Nov 20 '18 at 23:17







      loretoparisi

















      asked Nov 13 '18 at 16:26









      loretoparisiloretoparisi

      7,66254772




      7,66254772






















          1 Answer
          1






          active

          oldest

          votes


















          1














          This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help






          var word = "Lorem";
          var reg = RegExp(word, 'g');
          var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
          var match;

          console.log(sentence);
          console.log(word);

          while ((match = reg.exec(sentence)) !== null)
          var wordStart = match.index;
          var wordEnd = wordStart + word.length - 1;
          console.log(wordStart + ' -start index');
          console.log(word.length + ' -length of word');
          console.log(wordEnd + ' -last character index, need to +1 to use with substring');
          console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');








          share|improve this answer

























          • thanks this suggestion helped to rewrote the question adding a possibile solution.

            – loretoparisi
            Nov 13 '18 at 18:08











          • Also there is a problem with the RegExp that is we should take in account multiple matches of a token within the current sentence.

            – loretoparisi
            Nov 13 '18 at 18:47






          • 1





            updated code to take into consideration for multiple matches.

            – XTOTHEL
            Nov 13 '18 at 19:10











          • hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.

            – loretoparisi
            Nov 20 '18 at 23:14










          Your Answer






          StackExchange.ifUsing("editor", function ()
          StackExchange.using("externalEditor", function ()
          StackExchange.using("snippets", function ()
          StackExchange.snippets.init();
          );
          );
          , "code-snippets");

          StackExchange.ready(function()
          var channelOptions =
          tags: "".split(" "),
          id: "1"
          ;
          initTagRenderer("".split(" "), "".split(" "), channelOptions);

          StackExchange.using("externalEditor", function()
          // Have to fire editor after snippets, if snippets enabled
          if (StackExchange.settings.snippets.snippetsEnabled)
          StackExchange.using("snippets", function()
          createEditor();
          );

          else
          createEditor();

          );

          function createEditor()
          StackExchange.prepareEditor(
          heartbeatType: 'answer',
          autoActivateHeartbeat: false,
          convertImagesToLinks: true,
          noModals: true,
          showLowRepImageUploadWarning: true,
          reputationToPostImages: 10,
          bindNavPrevention: true,
          postfix: "",
          imageUploader:
          brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
          contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
          allowUrls: true
          ,
          onDemand: true,
          discardSelector: ".discard-answer"
          ,immediatelyShowMarkdownHelp:true
          );



          );













          draft saved

          draft discarded


















          StackExchange.ready(
          function ()
          StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53285370%2fcalculate-begin-and-end-position-of-a-word-in-a-document-in-javascript%23new-answer', 'question_page');

          );

          Post as a guest















          Required, but never shown

























          1 Answer
          1






          active

          oldest

          votes








          1 Answer
          1






          active

          oldest

          votes









          active

          oldest

          votes






          active

          oldest

          votes









          1














          This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help






          var word = "Lorem";
          var reg = RegExp(word, 'g');
          var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
          var match;

          console.log(sentence);
          console.log(word);

          while ((match = reg.exec(sentence)) !== null)
          var wordStart = match.index;
          var wordEnd = wordStart + word.length - 1;
          console.log(wordStart + ' -start index');
          console.log(word.length + ' -length of word');
          console.log(wordEnd + ' -last character index, need to +1 to use with substring');
          console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');








          share|improve this answer

























          • thanks this suggestion helped to rewrote the question adding a possibile solution.

            – loretoparisi
            Nov 13 '18 at 18:08











          • Also there is a problem with the RegExp that is we should take in account multiple matches of a token within the current sentence.

            – loretoparisi
            Nov 13 '18 at 18:47






          • 1





            updated code to take into consideration for multiple matches.

            – XTOTHEL
            Nov 13 '18 at 19:10











          • hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.

            – loretoparisi
            Nov 20 '18 at 23:14















          1














          This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help






          var word = "Lorem";
          var reg = RegExp(word, 'g');
          var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
          var match;

          console.log(sentence);
          console.log(word);

          while ((match = reg.exec(sentence)) !== null)
          var wordStart = match.index;
          var wordEnd = wordStart + word.length - 1;
          console.log(wordStart + ' -start index');
          console.log(word.length + ' -length of word');
          console.log(wordEnd + ' -last character index, need to +1 to use with substring');
          console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');








          share|improve this answer

























          • thanks this suggestion helped to rewrote the question adding a possibile solution.

            – loretoparisi
            Nov 13 '18 at 18:08











          • Also there is a problem with the RegExp that is we should take in account multiple matches of a token within the current sentence.

            – loretoparisi
            Nov 13 '18 at 18:47






          • 1





            updated code to take into consideration for multiple matches.

            – XTOTHEL
            Nov 13 '18 at 19:10











          • hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.

            – loretoparisi
            Nov 20 '18 at 23:14













          1












          1








          1







          This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help






          var word = "Lorem";
          var reg = RegExp(word, 'g');
          var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
          var match;

          console.log(sentence);
          console.log(word);

          while ((match = reg.exec(sentence)) !== null)
          var wordStart = match.index;
          var wordEnd = wordStart + word.length - 1;
          console.log(wordStart + ' -start index');
          console.log(word.length + ' -length of word');
          console.log(wordEnd + ' -last character index, need to +1 to use with substring');
          console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');








          share|improve this answer















          This might be an easier way to calculate the start/end of a word in a sentence, hopefully it will help






          var word = "Lorem";
          var reg = RegExp(word, 'g');
          var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
          var match;

          console.log(sentence);
          console.log(word);

          while ((match = reg.exec(sentence)) !== null)
          var wordStart = match.index;
          var wordEnd = wordStart + word.length - 1;
          console.log(wordStart + ' -start index');
          console.log(word.length + ' -length of word');
          console.log(wordEnd + ' -last character index, need to +1 to use with substring');
          console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');








          var word = "Lorem";
          var reg = RegExp(word, 'g');
          var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
          var match;

          console.log(sentence);
          console.log(word);

          while ((match = reg.exec(sentence)) !== null)
          var wordStart = match.index;
          var wordEnd = wordStart + word.length - 1;
          console.log(wordStart + ' -start index');
          console.log(word.length + ' -length of word');
          console.log(wordEnd + ' -last character index, need to +1 to use with substring');
          console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');





          var word = "Lorem";
          var reg = RegExp(word, 'g');
          var sentence = "Lore ipsum Lorem dolor sit Lorem amet,";
          var match;

          console.log(sentence);
          console.log(word);

          while ((match = reg.exec(sentence)) !== null)
          var wordStart = match.index;
          var wordEnd = wordStart + word.length - 1;
          console.log(wordStart + ' -start index');
          console.log(word.length + ' -length of word');
          console.log(wordEnd + ' -last character index, need to +1 to use with substring');
          console.log(sentence.substring(wordStart, wordEnd + 1) + '-using substring with calculated to find the word and verify');






          share|improve this answer














          share|improve this answer



          share|improve this answer








          edited Nov 13 '18 at 19:10

























          answered Nov 13 '18 at 16:52









          XTOTHELXTOTHEL

          2,0821210




          2,0821210












          • thanks this suggestion helped to rewrote the question adding a possibile solution.

            – loretoparisi
            Nov 13 '18 at 18:08











          • Also there is a problem with the RegExp that is we should take in account multiple matches of a token within the current sentence.

            – loretoparisi
            Nov 13 '18 at 18:47






          • 1





            updated code to take into consideration for multiple matches.

            – XTOTHEL
            Nov 13 '18 at 19:10











          • hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.

            – loretoparisi
            Nov 20 '18 at 23:14

















          • thanks this suggestion helped to rewrote the question adding a possibile solution.

            – loretoparisi
            Nov 13 '18 at 18:08











          • Also there is a problem with the RegExp that is we should take in account multiple matches of a token within the current sentence.

            – loretoparisi
            Nov 13 '18 at 18:47






          • 1





            updated code to take into consideration for multiple matches.

            – XTOTHEL
            Nov 13 '18 at 19:10











          • hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.

            – loretoparisi
            Nov 20 '18 at 23:14
















          thanks this suggestion helped to rewrote the question adding a possibile solution.

          – loretoparisi
          Nov 13 '18 at 18:08





          thanks this suggestion helped to rewrote the question adding a possibile solution.

          – loretoparisi
          Nov 13 '18 at 18:08













          Also there is a problem with the RegExp that is we should take in account multiple matches of a token within the current sentence.

          – loretoparisi
          Nov 13 '18 at 18:47





          Also there is a problem with the RegExp that is we should take in account multiple matches of a token within the current sentence.

          – loretoparisi
          Nov 13 '18 at 18:47




          1




          1





          updated code to take into consideration for multiple matches.

          – XTOTHEL
          Nov 13 '18 at 19:10





          updated code to take into consideration for multiple matches.

          – XTOTHEL
          Nov 13 '18 at 19:10













          hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.

          – loretoparisi
          Nov 20 '18 at 23:14





          hey @xtothel I have updated the question with your solution. The problem is that when cycling on tokens in a sentence, only the last match will be taken in account, so the matched position will be wrong.

          – loretoparisi
          Nov 20 '18 at 23:14

















          draft saved

          draft discarded
















































          Thanks for contributing an answer to Stack Overflow!


          • Please be sure to answer the question. Provide details and share your research!

          But avoid


          • Asking for help, clarification, or responding to other answers.

          • Making statements based on opinion; back them up with references or personal experience.

          To learn more, see our tips on writing great answers.




          draft saved


          draft discarded














          StackExchange.ready(
          function ()
          StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53285370%2fcalculate-begin-and-end-position-of-a-word-in-a-document-in-javascript%23new-answer', 'question_page');

          );

          Post as a guest















          Required, but never shown





















































          Required, but never shown














          Required, but never shown












          Required, but never shown







          Required, but never shown

































          Required, but never shown














          Required, but never shown












          Required, but never shown







          Required, but never shown







          這個網誌中的熱門文章

          How to read a connectionString WITH PROVIDER in .NET Core?

          In R, how to develop a multiplot heatmap.2 figure showing key labels successfully

          Museum of Modern and Contemporary Art of Trento and Rovereto