Move text normalization outside chunking method

NatLibFi · Feb 4, 2019 · 794bfb5 · 794bfb5
1 parent 3e46e99
commit 794bfb5
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 6 deletions.
diff --git a/annif/backend/fasttext.py b/annif/backend/fasttext.py
@@ -107,9 +107,18 @@ def train(self, corpus, project):
         self._create_train_file(corpus, project)
         self._create_model()
 
+    def _predict_chunks(self, chunktexts, project, limit):
+        normalized_chunks = []
+        for chunktext in chunktexts:
+            normalized = self._normalize_text(project, chunktext)
+            if normalized != '':
+                normalized_chunks.append(normalized)
+        return self._model.predict(normalized_chunks, limit)
+
     def _analyze_chunks(self, chunktexts, project):
         limit = int(self.params['limit'])
-        chunklabels, chunkscores = self._model.predict(chunktexts, limit)
+        chunklabels, chunkscores = self._predict_chunks(
+            chunktexts, project, limit)
         label_scores = collections.defaultdict(float)
         for labels, scores in zip(chunklabels, chunkscores):
             for label, score in zip(labels, scores):

diff --git a/annif/backend/mixins.py b/annif/backend/mixins.py
@@ -24,10 +24,7 @@ def _analyze(self, text, project, params):
         chunksize = int(params['chunksize'])
         chunktexts = []
         for i in range(0, len(sentences), chunksize):
-            chunktext = ' '.join(sentences[i:i + chunksize])
-            normalized = self._normalize_text(project, chunktext)
-            if normalized != '':
-                chunktexts.append(normalized)
+            chunktexts.append(' '.join(sentences[i:i + chunksize]))
         self.debug('Split sentences into {} chunks'.format(len(chunktexts)))
         if len(chunktexts) == 0:  # nothing to analyze, empty result
             return ListAnalysisResult(hits=[], subject_index=project.subjects)

diff --git a/annif/backend/vw_multi.py b/annif/backend/vw_multi.py
@@ -152,7 +152,10 @@ def train(self, corpus, project):
     def _analyze_chunks(self, chunktexts, project):
         results = []
         for chunktext in chunktexts:
-            example = ' | {}'.format(chunktext)
+            normalized = self._normalize_text(project, chunktext)
+            if normalized == '':
+                continue
+            example = ' | {}'.format(normalized)
             result = self._model.predict(example)
             if self.algorithm == 'multilabel_oaa':
                 # result is a list of subject IDs - need to vectorize