Do not remove newline in Wayback download script

apple · Jan 4, 2021 · 0c5e226 · 0c5e226
1 parent dc9cf6b
commit 0c5e226
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/collection/download_wayback_passages.py b/collection/download_wayback_passages.py
@@ -76,7 +76,7 @@ def extract_text(html_text: str) -> str:
     output = ''
     for t in text:
         if t.parent.name not in blacklist:
-            output += f'{t.strip()} '
+            output += f'{t} '
 
     return output
 

diff --git a/collection/paragraph_chunker.py b/collection/paragraph_chunker.py
@@ -16,7 +16,7 @@
 
 
 def chunk_doc(content: str) -> List[str]:
-    """Given a document, return a list of passages of no more than MIN_PASSAGE_TOKENS tokens."""
+    """Given a document, return a list of passages of no fewer than MIN_PASSAGE_TOKENS tokens / passage until EOF."""
     passages = []
     passage_tokens = []
     lines = content.split('\n')