Tokens

gosom · Jun 7, 2020 · 04469da · 04469da
1 parent daf9705
commit 04469da
Showing 1 changed file with 1 addition and 5 deletions.
diff --git a/pkg/spellcorrect/frequencies.go b/pkg/spellcorrect/frequencies.go
@@ -34,9 +34,6 @@ func (o *Frequencies) Load(tokens []string) error {
 	unigrams := make(map[uint64]int)
 	for i := range tokens {
 		hashes[i] = hashString(tokens[i])
-		if _, ok := unigrams[hashes[i]]; !ok {
-			unigrams[hashes[i]] = 0
-		}
 		unigrams[hashes[i]]++
 		if len(tokens[i]) < o.minWord {
 			bl[hashes[i]] = true
@@ -47,10 +44,9 @@ func (o *Frequencies) Load(tokens []string) error {
 		if v < o.minFreq {
 			bl[k] = true
 		} else {
-			o.uniGramProbs[k] = float64(v) / float64(len(unigrams))
+			o.uniGramProbs[k] = float64(v) / float64(len(tokens))
 		}
 	}
-	unigrams = nil
 
 	t2 := time.Now()
 	fmt.Println("time to hash and map", t2.Sub(t1))