99 lines
2.0 KiB
Go
99 lines
2.0 KiB
Go
package reader
|
||
|
||
import (
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
type Word struct {
|
||
Text string `json:"text"`
|
||
Prefix string `json:"prefix"`
|
||
Focus string `json:"focus"`
|
||
Suffix string `json:"suffix"`
|
||
}
|
||
|
||
// Tokenize splits text into words while ignoring repeated whitespace.
|
||
func Tokenize(text string) []Word {
|
||
fields := strings.Fields(text)
|
||
words := make([]Word, 0, len(fields))
|
||
|
||
for _, field := range fields {
|
||
words = append(words, SplitWord(field))
|
||
}
|
||
|
||
return words
|
||
}
|
||
|
||
// SplitWord separates a word into its prefix, focus character and suffix.
|
||
func SplitWord(word string) Word {
|
||
if word == "" {
|
||
return Word{}
|
||
}
|
||
|
||
runes := []rune(word)
|
||
position := ORPIndex(runes)
|
||
|
||
// Keep the focus index inside the word.
|
||
if position >= len(runes) {
|
||
position = len(runes) - 1
|
||
}
|
||
|
||
return Word{
|
||
Text: word,
|
||
Prefix: string(runes[:position]),
|
||
Focus: string(runes[position : position+1]),
|
||
Suffix: string(runes[position+1:]),
|
||
}
|
||
}
|
||
|
||
// lastMeaningfulRune returns the last rune after ignoring
|
||
// closing quotes and brackets.
|
||
func lastMeaningfulRune(word string) rune {
|
||
word = strings.TrimRightFunc(word, func(r rune) bool {
|
||
switch r {
|
||
case '"', '\'', '”', '’', ')', ']', '}', '»':
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
})
|
||
|
||
runes := []rune(word)
|
||
if len(runes) == 0 {
|
||
return 0
|
||
}
|
||
|
||
return runes[len(runes)-1]
|
||
}
|
||
|
||
// IsSentenceEnding checks whether a word ends a sentence.
|
||
func IsSentenceEnding(word string) bool {
|
||
switch lastMeaningfulRune(word) {
|
||
case '.', '!', '?', '…':
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// IsPausePunctuation checks for punctuation that should
|
||
// introduce a short pause during playback.
|
||
func IsPausePunctuation(word string) bool {
|
||
switch lastMeaningfulRune(word) {
|
||
case ',', ';', ':':
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// RuneLength returns the number of Unicode characters in a word.
|
||
func RuneLength(word string) int {
|
||
return len([]rune(word))
|
||
}
|
||
|
||
// IsPunctuation reports whether the final meaningful character is punctuation.
|
||
func IsPunctuation(word string) bool {
|
||
return unicode.IsPunct(lastMeaningfulRune(word))
|
||
}
|