Reduce Docker memory limit to 2g and implement section processing opt…

…imization in ActSectionUpdaterTask
miskibin · Nov 3, 2024 · 9d5f8b4 · 9d5f8b4
1 parent 6ffd0c3
commit 9d5f8b4
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -99,7 +99,7 @@ services:
       - C_FORCE_ROOT=true
     env_file:
       - .env
-    mem_limit: 4g
+    mem_limit: 2g
   certbot:
     image: certbot/dns-cloudflare
     volumes:

diff --git a/src/eli_app/db_update/update_law.py b/src/eli_app/db_update/update_law.py
@@ -56,6 +56,8 @@ def fetch_metadata_and_text(self, eli: str):
         return metadata, full_text
 
     def process_chapters(self, eli: str, chapters: list[ActSection], metadata: dict):
+        new_sections_count = 0  # Counter for new sections created
+
         for chapter in chapters:
             if self.should_skip_section(eli, chapter):
                 logger.info(
@@ -81,9 +83,15 @@ def process_chapters(self, eli: str, chapters: list[ActSection], metadata: dict)
                     "summary": summary,
                 },
             )
-            sleep(8)
+            new_sections_count += 1  # Increment the counter
+            sleep(5)
             logger.info(f"Updated section {chapter.chapters}")
 
+            # Run process_sections_without_embeddings every 20 new sections
+            if new_sections_count >= 20:
+                self.process_sections_without_embeddings()
+                new_sections_count = 0  # Reset the counter
+
     def should_skip_section(self, eli: str, chapter: ActSection) -> bool:
         existing_section = ActSection.objects.filter(
             act=Act.objects.get(ELI=eli),