%0 Journal Article %1 VanAken2011 %A Van Aken, J. R. %D 2011 %J ArXiv e-prints %K natural_language %T A statistical learning algorithm for word segmentation %X In natural speech, the speaker does not pause between words, yet a human listener somehow perceives this continuous stream of phonemes as a series of distinct words. The detection of boundaries between spoken words is an instance of a general capability of the human neocortex to remember and to recognize recurring sequences. This paper describes a computer algorithm that is designed to solve the problem of locating word boundaries in blocks of English text from which the spaces have been removed. This problem avoids the complexities of processing speech but requires similar capabilities for detecting recurring sequences. The algorithm that is described in this paper relies entirely on statistical relationships between letters in the input stream to infer the locations of word boundaries. The source code for a C++ version of this algorithm is presented in an appendix.

@article{VanAken2011, abstract = {In natural speech, the speaker does not pause between words, yet a human listener somehow perceives this continuous stream of phonemes as a series of distinct words. The detection of boundaries between spoken words is an instance of a general capability of the human neocortex to remember and to recognize recurring sequences. This paper describes a computer algorithm that is designed to solve the problem of locating word boundaries in blocks of English text from which the spaces have been removed. This problem avoids the complexities of processing speech but requires similar capabilities for detecting recurring sequences. The algorithm that is described in this paper relies entirely on statistical relationships between letters in the input stream to infer the locations of word boundaries. The source code for a C++ version of this algorithm is presented in an appendix.}, added-at = {2011-06-02T10:35:58.000+0200}, adsnote = {Provided by the SAO/NASA Astrophysics Data System}, adsurl = {http://adsabs.harvard.edu/abs/2011arXiv1105.6162V}, archiveprefix = {arXiv}, author = {{Van Aken}, J. R.}, biburl = {https://www.bibsonomy.org/bibtex/26072207ae20ef0625a95bfcb50d7eaca/sjbutler}, eprint = {1105.6162}, interhash = {5da53d844e1295fb979ed58cee27ba84}, intrahash = {6072207ae20ef0625a95bfcb50d7eaca}, journal = {ArXiv e-prints}, keywords = {natural_language}, month = May, primaryclass = {cs.CL}, timestamp = {2011-06-02T10:35:58.000+0200}, title = {{A statistical learning algorithm for word segmentation}}, year = 2011 }