package reader import ( "strings" "unicode" ) type Word struct { Text string `json:"text"` Prefix string `json:"prefix"` Focus string `json:"focus"` Suffix string `json:"suffix"` } // Tokenize splits text into words while ignoring repeated whitespace. func Tokenize(text string) []Word { fields := strings.Fields(text) words := make([]Word, 0, len(fields)) for _, field := range fields { words = append(words, SplitWord(field)) } return words } // SplitWord separates a word into its prefix, focus character and suffix. func SplitWord(word string) Word { if word == "" { return Word{} } runes := []rune(word) position := ORPIndex(runes) // Keep the focus index inside the word. if position >= len(runes) { position = len(runes) - 1 } return Word{ Text: word, Prefix: string(runes[:position]), Focus: string(runes[position : position+1]), Suffix: string(runes[position+1:]), } } // lastMeaningfulRune returns the last rune after ignoring // closing quotes and brackets. func lastMeaningfulRune(word string) rune { word = strings.TrimRightFunc(word, func(r rune) bool { switch r { case '"', '\'', '”', '’', ')', ']', '}', '»': return true default: return false } }) runes := []rune(word) if len(runes) == 0 { return 0 } return runes[len(runes)-1] } // IsSentenceEnding checks whether a word ends a sentence. func IsSentenceEnding(word string) bool { switch lastMeaningfulRune(word) { case '.', '!', '?', '…': return true default: return false } } // IsPausePunctuation checks for punctuation that should // introduce a short pause during playback. func IsPausePunctuation(word string) bool { switch lastMeaningfulRune(word) { case ',', ';', ':': return true default: return false } } // RuneLength returns the number of Unicode characters in a word. func RuneLength(word string) int { return len([]rune(word)) } // IsPunctuation reports whether the final meaningful character is punctuation. func IsPunctuation(word string) bool { return unicode.IsPunct(lastMeaningfulRune(word)) }