99 lines
2.0 KiB
Go
99 lines
2.0 KiB
Go
|
package reader
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"strings"
|
|||
|
|
"unicode"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
type Word struct {
|
|||
|
|
Text string `json:"text"`
|
|||
|
|
Prefix string `json:"prefix"`
|
|||
|
|
Focus string `json:"focus"`
|
|||
|
|
Suffix string `json:"suffix"`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Tokenize splits text into words while ignoring repeated whitespace.
|
|||
|
|
func Tokenize(text string) []Word {
|
|||
|
|
fields := strings.Fields(text)
|
|||
|
|
words := make([]Word, 0, len(fields))
|
|||
|
|
|
|||
|
|
for _, field := range fields {
|
|||
|
|
words = append(words, SplitWord(field))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return words
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// SplitWord separates a word into its prefix, focus character and suffix.
|
|||
|
|
func SplitWord(word string) Word {
|
|||
|
|
if word == "" {
|
|||
|
|
return Word{}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
runes := []rune(word)
|
|||
|
|
position := ORPIndex(runes)
|
|||
|
|
|
|||
|
|
// Keep the focus index inside the word.
|
|||
|
|
if position >= len(runes) {
|
|||
|
|
position = len(runes) - 1
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return Word{
|
|||
|
|
Text: word,
|
|||
|
|
Prefix: string(runes[:position]),
|
|||
|
|
Focus: string(runes[position : position+1]),
|
|||
|
|
Suffix: string(runes[position+1:]),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// lastMeaningfulRune returns the last rune after ignoring
|
|||
|
|
// closing quotes and brackets.
|
|||
|
|
func lastMeaningfulRune(word string) rune {
|
|||
|
|
word = strings.TrimRightFunc(word, func(r rune) bool {
|
|||
|
|
switch r {
|
|||
|
|
case '"', '\'', '”', '’', ')', ']', '}', '»':
|
|||
|
|
return true
|
|||
|
|
default:
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
runes := []rune(word)
|
|||
|
|
if len(runes) == 0 {
|
|||
|
|
return 0
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return runes[len(runes)-1]
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// IsSentenceEnding checks whether a word ends a sentence.
|
|||
|
|
func IsSentenceEnding(word string) bool {
|
|||
|
|
switch lastMeaningfulRune(word) {
|
|||
|
|
case '.', '!', '?', '…':
|
|||
|
|
return true
|
|||
|
|
default:
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// IsPausePunctuation checks for punctuation that should
|
|||
|
|
// introduce a short pause during playback.
|
|||
|
|
func IsPausePunctuation(word string) bool {
|
|||
|
|
switch lastMeaningfulRune(word) {
|
|||
|
|
case ',', ';', ':':
|
|||
|
|
return true
|
|||
|
|
default:
|
|||
|
|
return false
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// RuneLength returns the number of Unicode characters in a word.
|
|||
|
|
func RuneLength(word string) int {
|
|||
|
|
return len([]rune(word))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// IsPunctuation reports whether the final meaningful character is punctuation.
|
|||
|
|
func IsPunctuation(word string) bool {
|
|||
|
|
return unicode.IsPunct(lastMeaningfulRune(word))
|
|||
|
|
}
|