feat: progress, but have encoutered another problem with invidious, w…

…ill update issue
Kariton · Feb 3, 2023 · 311f6c0 · 311f6c0
1 parent cfebe1e
commit 311f6c0
Show file tree

Hide file tree

Showing 8 changed files with 108 additions and 37 deletions.
diff --git a/rss-ntfy.py b/rss-ntfy.py
@@ -17,26 +17,34 @@
 
 ## Code:
 
+from bs4 import BeautifulSoup
+from pathlib import Path
 import os
+import re
 import requests
-from bs4 import BeautifulSoup
 
 SCRIPT_DIR    = os.path.dirname(os.path.realpath(__file__)) + "/rss-ntfy/"
 SERVICES      = [
                  {
                   "service": "nitter", 
-                  "rss-url": "https://uk.unofficialbird.com/{username}/rss", 
-                  "descriptor": "🎶 Tweet"
+                  "rss-url": "https://uk.unofficialbird.com/{{ custom }}/rss", 
+                  "descriptor": "🐦 Tweet"
                  },
                  {
                   "service": "proxitok", 
-                  "rss-url": "https://proxitok.pabloferreiro.es/@{username}/rss", 
+                  "rss-url": "https://proxitok.pabloferreiro.es/@{{ custom }}/rss", 
                   "descriptor": "🎶 TikTok"
                  },
                  {
                   "service": "invidious", 
-                  "rss-url": "https://invidious.snopyta.org/feed/channel/{username}", 
-                  "descriptor": "📽 YouTube video "
+                  "rss-url": "https://invidious.snopyta.org/feed/channel/{{ custom }}",
+                  "descriptor": "📽 YouTube video"
+
+                 },
+                 { 
+                  "service": "teddit",
+                  "rss-url": "https://teddit.net/r/{{ custom }}?api&type=rss",
+                  "descriptor": "🎩 Reddit post" 
                  }
                 ]
 NTFY_INSTANCE = "https://ntfy.julian.rocks/"
@@ -82,40 +90,76 @@ def get_user_list(user_list_file):
         user_list = [l.rstrip() for l in f]
     return user_list
 
+def handlebar_replace(input, replacement):
+    '''
+    Very simple Handlebar style replace:
+    https://handlebarsjs.com
+
+    Takes the input URL and replaces the {{ custom }}
+    part, which will be the current user part.
+    '''
+    return re.sub('\{\{.*\}\}', replacement, input)
+
+def check_file_list_exists(file_list):
+    '''
+    Takes a list of files, checks if they exist, 
+    creates them if they do not!
+
+    I'm using this function instead of just relying on
+    'w+', because we 'r+' the History file, at one point,
+    and 'r+' doesn't create the file if it doesn't exist 
+    (unlike 'w+' and 'a+').
+    '''
+    for file in file_list:
+        Path(file).touch(exist_ok=True)
+
 def main():
     '''
     This article by Matthew Wimberly got me along the right lines with things:
     https://codeburst.io/building-an-rss-feed-scraper-with-python-73715ca06e1f
     '''
     for service in SERVICES:
+        # Follow File and History File
         user_list_file = f"{SCRIPT_DIR}{service['service']}-follow-list.txt"
-        instance       = f"{service['rss-url']}"
-        ntfy_topic     = f"{service['service']}"
-        service_log    = f"{SCRIPT_DIR}{service['service']}.log"
-        descriptor     = service['descriptor']
-        user_list = get_user_list(user_list_file)
+        service_hist   = f"{SCRIPT_DIR}{service['service']}_hist"
+        check_file_list_exists([user_list_file, service_hist])
+
+        # Instance, Topic, Descriptor
+        instance    = f"{service['rss-url']}"
+        ntfy_topic  = f"{service['service']}"
+        descriptor  = service['descriptor']
+
+        # TODO: Rename everything with 'user', as it's more generally an
+        #       account? Not sure if account is the best name, either.
+        user_list   = get_user_list(user_list_file)
+
         for username in user_list:
+            current_instance = handlebar_replace(instance, username)
             try:
-                req = requests.get(f"{instance}")
+                req = requests.get(f"{current_instance}")
                 rss_content = BeautifulSoup(req.content, "lxml-xml")
                 articles = rss_content.findAll('item')
                 for a in articles:
                     title     = a.find('title').text
                     link      = a.find('link').text
                     published = a.find('pubDate').text
 
-                    with open(service_log, "r+") as f:
-                        data = f.read()
+                    with open(service_hist, "r+") as hist_file:
+                        data = hist_file.read()
+                        # If the link isn't in data, not only
+                        # do we want to add it to the Hist file,
+                        # we also want to, of course, ntfy:
                         if not link in data:
                             ntfyr_complex(ntfy_topic, 
                                           username, 
                                           title, 
                                           link, 
                                           published,
                                           descriptor)
-                            f.write(f"{link}\n")
+                            hist_file.write(f"{link}\n")
 
             except Exception as e:
+                # TODO: Just use the ntfy JSON request format
                 ntfyr(f"Error with scraping {username}, '{e}'.", ntfy_topic)
 
 if __name__ == '__main__':

diff --git a/rss-ntfy/invidious-follow-list.txt b/rss-ntfy/invidious-follow-list.txt
@@ -1 +1,2 @@
 UCPdzbBejsLqZRl4vDclPCnw
+UCXuqSBlHAE6Xw-yeJA0Tunw
diff --git a/rss-ntfy/invidious.log → rss-ntfy/invidious_hist b/rss-ntfy/invidious.log → rss-ntfy/invidious_hist
diff --git a/rss-ntfy/nitter.log b/rss-ntfy/nitter.log
diff --git a/rss-ntfy/nitter_hist b/rss-ntfy/nitter_hist
@@ -0,0 +1,20 @@
+https://uk.unofficialbird.com/joinmastodon/status/1594121627083493378#m
+https://uk.unofficialbird.com/BBCBweaking/status/1593548583671824385#m
+https://uk.unofficialbird.com/ByDonkeys/status/1582303415576715265#m
+https://uk.unofficialbird.com/brian_bilston/status/1581553049843404801#m
+https://uk.unofficialbird.com/jdorchard/status/1580634441889697792#m
+https://uk.unofficialbird.com/jdorchard/status/1580634433136123904#m
+https://uk.unofficialbird.com/brian_bilston/status/1580106651729698816#m
+https://uk.unofficialbird.com/CHAINSAWMAN_PR/status/1579856860651233281#m
+https://uk.unofficialbird.com/mizabitha/status/1579382959660232705#m
+https://uk.unofficialbird.com/POTUS/status/1578097875480895489#m
+https://uk.unofficialbird.com/Rocket_Sledge/status/1576948986334048257#m
+https://uk.unofficialbird.com/poisonjr/status/1575322819772096512#m
+https://uk.unofficialbird.com/WessexLifts/status/1574700560284094464#m
+https://uk.unofficialbird.com/jdorchard/status/1565636919282786305#m
+https://uk.unofficialbird.com/ThreatNotation/status/1512804148886056969#m
+https://uk.unofficialbird.com/h3h3productions/status/1508285488616337408#m
+https://uk.unofficialbird.com/jdorchard/status/1475156805407981574#m
+https://uk.unofficialbird.com/highprogressive/status/1471720721495867396#m
+https://uk.unofficialbird.com/Rizex45/status/1468687563447799808#m
+https://uk.unofficialbird.com/ai_curio_bot/status/1465164092243644416#m
diff --git a/rss-ntfy/proxitok.log → rss-ntfy/proxitok_hist b/rss-ntfy/proxitok.log → rss-ntfy/proxitok_hist
@@ -1,3 +1,2 @@
-
 https://proxitok.pabloferreiro.es/@0x6a75/video/6638307464917814533
 https://proxitok.pabloferreiro.es/@0x6a75/video/6606359306138815750
diff --git a/rss-ntfy/teddit-follow-list.txt b/rss-ntfy/teddit-follow-list.txt
@@ -0,0 +1 @@
+funny
diff --git a/rss-ntfy/teddit_hist b/rss-ntfy/teddit_hist
@@ -0,0 +1,27 @@
+https://teddit.net/r/funny/comments/uq9pjw/going_forward_comics_may_only_be_posted_on/
+https://teddit.net/r/funny/comments/10qqy3c/subreddit_of_the_month_february_2023/
+https://teddit.net/r/funny/comments/10s9i7j/my_wife_and_her_class_made_ground_hogs_i_cant/
+https://teddit.net/r/funny/comments/10s4s8a/what_am_i_looking_at/
+https://teddit.net/r/funny/comments/10rw2oj/pain_of_labor/
+https://teddit.net/r/funny/comments/10rt8k3/i_hope_the_texas_police_understand_why_i_didnt/
+https://teddit.net/r/funny/comments/10rwu8o/i_wouldnt_want_cootie_juices_on_my_dishes_either/
+https://teddit.net/r/funny/comments/10s97l6/you_had_one_job/
+https://teddit.net/r/funny/comments/10s9ic7/please_hold_my_hand_while_i_fart/
+https://teddit.net/r/funny/comments/10rvtxc/went_for_my_morning_movement_at_work_some_new_art/
+https://teddit.net/r/funny/comments/10sag0t/was_watching_groundhog_day_when/
+https://teddit.net/r/funny/comments/10rvzee/ais_attempt_at_a_sign_language_guide/
+https://teddit.net/r/funny/comments/10ry3n1/i_was_on_a_game_show_last_year_but_couldnt_tell/
+https://teddit.net/r/funny/comments/10rp5en/everyone_will_hear_her_coming/
+https://teddit.net/r/funny/comments/10s3vzl/pastor_falls_for_prank_callers_trap_live_on_air/
+https://teddit.net/r/funny/comments/10rzy9y/the_all_seeing_cats/
+https://teddit.net/r/funny/comments/10rz513/good_boi/
+https://teddit.net/r/funny/comments/10rqtkd/dont_drink_and/
+https://teddit.net/r/funny/comments/10shs45/china_rn/
+https://teddit.net/r/funny/comments/10sdwq7/out_of_hundreds_of_fake_tattoos_my_daughter_could/
+https://teddit.net/r/funny/comments/10rwshn/turn_back_to_me_playing_a_gig_in_2007_didnt_have/
+https://teddit.net/r/funny/comments/10sap3s/these_ubereats_ads_are_getting_out_of_hand/
+https://teddit.net/r/funny/comments/10sixxl/this_made_me_laugh_out_loud/
+https://teddit.net/r/funny/comments/10s1oeo/found_this_gem_on_facebook_marketplace/
+https://teddit.net/r/funny/comments/10s3ges/cannot_unsee/
+https://teddit.net/r/funny/comments/10rtrkp/safety_first_no_bringing_your_batarangs_on_the/
+https://teddit.net/r/funny/comments/10s8yl9/i_rate_this_hinge_video_prompt_300/
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		UCPdzbBejsLqZRl4vDclPCnw
		UCXuqSBlHAE6Xw-yeJA0Tunw
Original file line number	Diff line number	Diff line change
		@@ -1,3 +1,2 @@

		https://proxitok.pabloferreiro.es/@0x6a75/video/6638307464917814533
		https://proxitok.pabloferreiro.es/@0x6a75/video/6606359306138815750