First real commit. Setting up for full CI/CD and testing pipeline

internal/reader/tokenizer.go

@@ -0,0 +1,98 @@
+package reader
+
+import (
+	"strings"
+	"unicode"
+)
+
+type Word struct {
+	Text   string `json:"text"`
+	Prefix string `json:"prefix"`
+	Focus  string `json:"focus"`
+	Suffix string `json:"suffix"`
+}
+
+// Tokenize splits text into words while ignoring repeated whitespace.
+func Tokenize(text string) []Word {
+	fields := strings.Fields(text)
+	words := make([]Word, 0, len(fields))
+
+	for _, field := range fields {
+		words = append(words, SplitWord(field))
+	}
+
+	return words
+}
+
+// SplitWord separates a word into its prefix, focus character and suffix.
+func SplitWord(word string) Word {
+	if word == "" {
+		return Word{}
+	}
+
+	runes := []rune(word)
+	position := ORPIndex(runes)
+
+	// Keep the focus index inside the word.
+	if position >= len(runes) {
+		position = len(runes) - 1
+	}
+
+	return Word{
+		Text:   word,
+		Prefix: string(runes[:position]),
+		Focus:  string(runes[position : position+1]),
+		Suffix: string(runes[position+1:]),
+	}
+}
+
+// lastMeaningfulRune returns the last rune after ignoring
+// closing quotes and brackets.
+func lastMeaningfulRune(word string) rune {
+	word = strings.TrimRightFunc(word, func(r rune) bool {
+		switch r {
+		case '"', '\'', '”', '’', ')', ']', '}', '»':
+			return true
+		default:
+			return false
+		}
+	})
+
+	runes := []rune(word)
+	if len(runes) == 0 {
+		return 0
+	}
+
+	return runes[len(runes)-1]
+}
+
+// IsSentenceEnding checks whether a word ends a sentence.
+func IsSentenceEnding(word string) bool {
+	switch lastMeaningfulRune(word) {
+	case '.', '!', '?', '…':
+		return true
+	default:
+		return false
+	}
+}
+
+// IsPausePunctuation checks for punctuation that should
+// introduce a short pause during playback.
+func IsPausePunctuation(word string) bool {
+	switch lastMeaningfulRune(word) {
+	case ',', ';', ':':
+		return true
+	default:
+		return false
+	}
+}
+
+// RuneLength returns the number of Unicode characters in a word.
+func RuneLength(word string) int {
+	return len([]rune(word))
+}
+
+// IsPunctuation reports whether the final meaningful character is punctuation.
+func IsPunctuation(word string) bool {
+	return unicode.IsPunct(lastMeaningfulRune(word))
+}