Offset Github by 3 hours to not lose updates (onyx-dot-app#1073)

ToniHil · Feb 12, 2024 · 1b55e61 · 1b55e61
1 parent 1c4f7fe
commit 1b55e61
Showing 1 changed file with 11 additions and 1 deletion.
diff --git a/backend/danswer/connectors/github/connector.py b/backend/danswer/connectors/github/connector.py
@@ -214,7 +214,17 @@ def poll_source(
     ) -> GenerateDocumentsOutput:
         start_datetime = datetime.utcfromtimestamp(start)
         end_datetime = datetime.utcfromtimestamp(end)
-        return self._fetch_from_github(start_datetime, end_datetime)
+
+        # Move start time back by 3 hours, since some Issues/PRs are getting dropped
+        # Could be due to delayed processing on GitHub side
+        # The non-updated issues since last poll will be shortcut-ed and not embedded
+        adjusted_start_datetime = start_datetime - timedelta(hours=3)
+
+        epoch = datetime.utcfromtimestamp(0)
+        if adjusted_start_datetime < epoch:
+            adjusted_start_datetime = epoch
+
+        return self._fetch_from_github(adjusted_start_datetime, end_datetime)
 
 
 if __name__ == "__main__":