Merge branch 'main' of https://github.com/palewire/news-homepages int…

…o asg017-adstxt-start
palewire · Oct 22, 2023 · adca639 · adca639
2 parents 7422b21 + b388b49
commit adca639
Show file tree

Hide file tree

Showing 18 changed files with 1,740 additions and 6,507 deletions.
diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml
@@ -18,7 +18,7 @@ jobs:
         name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pipenv'
 
       - id: install-pipenv
@@ -47,7 +47,7 @@ jobs:
         name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pipenv'
 
       - id: install-pipenv
@@ -93,7 +93,7 @@ jobs:
         name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pipenv'
 
       - id: install-pipenv
@@ -122,7 +122,7 @@ jobs:
         name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pipenv'
 
       - id: install-pipenv

diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml
@@ -22,7 +22,7 @@ jobs:
         name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           cache: 'pipenv'
 
       - id: install-pipenv

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: trailing-whitespace
         exclude: newshomepages/extensions/
@@ -22,7 +22,7 @@ repos:
         exclude: newshomepages/extensions/
 
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.10.0
     hooks:
     -   id: black
         exclude: newshomepages/extensions/
@@ -48,13 +48,13 @@ repos:
         - flake8-bugbear
 
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.15.0
     hooks:
     -   id: pyupgrade
         args: [--py37-plus]
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.5.1'  # Use the sha / tag you want to point at
+    rev: 'v1.6.1'  # Use the sha / tag you want to point at
     hooks:
     -   id: mypy
         exclude: newshomepages/extensions

diff --git a/Pipfile b/Pipfile
@@ -9,21 +9,17 @@ pytz = "*"
 internetarchive = "*"
 python-slugify = "*"
 python-telegram-bot = "*"
-"discord.py" = "==1.7.3"
-playwright = "==1.38.0"
-savepagenow = "==1.2.2"
+"discord.py" = "*"
+playwright = "*"
 bs4 = "*"
 html5lib = "*"
 jinja2 = "*"
 shot-scraper = "*"
 pillow = "*"
-aiohttp = ">=3.7.4"
 pandas = "*"
-selenium = "<4.3.0"
 rich = "*"
 iso3166 = "*"
 python-iso639 = "*"
-certifi = "*"
 requests = "*"
 retry = "*"
 storysniffer = "*"
@@ -61,4 +57,4 @@ pytest-vcr = "*"
 pytest-xdist = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3.10"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/_site/requirements.txt b/_site/requirements.txt
@@ -5,7 +5,6 @@ python-slugify
 python-telegram-bot
 discord.py
 playwright
-savepagenow
 bs4
 html5lib
 jinja2

diff --git a/newshomepages/extensions/singlefile/javascript/single-file-bootstrap.js b/newshomepages/extensions/singlefile/javascript/single-file-bootstrap.js
diff --git a/newshomepages/extensions/singlefile/javascript/single-file-frames.js b/newshomepages/extensions/singlefile/javascript/single-file-frames.js
diff --git a/newshomepages/extensions/singlefile/javascript/single-file-hooks-frames.js b/newshomepages/extensions/singlefile/javascript/single-file-hooks-frames.js
diff --git a/newshomepages/extensions/singlefile/javascript/single-file.js b/newshomepages/extensions/singlefile/javascript/single-file.js
diff --git a/newshomepages/sources/javascript/washingtonpost.js b/newshomepages/sources/javascript/washingtonpost.js
@@ -0,0 +1,3 @@
+document.querySelectorAll(
+  '.paywall-overlay'
+).forEach(el => el.remove())
diff --git a/newshomepages/sources/sites.csv b/newshomepages/sources/sites.csv
@@ -272,7 +272,7 @@ KCCINews,https://www.kcci.com/,KCCI,Des Moines,America/Chicago,US,en,iowa,
 KCDefender,https://www.kansascitydefender.com/,Kansas City Defender,Kansas City,America/Chicago,US,en,missouri,
 KCRG,https://www.kcrg.com/,KCRG,Cedar Rapids,America/Chicago,US,en,iowa,
 KETV,https://www.ketv.com/,KETV NewsWatch 7,Omaha,America/Chicago,US,en,nebraska,
-KHNews,https://khn.org/,Kaiser Health News,Washington,America/New_York,US,en,,
+KHNews,https://khn.org/,KFF Health News,Washington,America/New_York,US,en,,
 KHONnews,https://www.khon2.com/,KHON2 News,Honolulu,US/Hawaii,US,en,hawaii,
 KHOU,https://www.khou.com/,KHOU,Houston,America/Chicago,US,en,texas,
 KING5Seattle,https://www.king5.com/,KING5,Seattle,America/Los_Angeles,US,en,washington,
@@ -791,7 +791,6 @@ globeandmail,https://www.theglobeandmail.com/,The Globe and Mail,Toronto,America
 gosanangelo,https://www.gosanangelo.com/,The San Angelo Standard-Times,San Angelo,America/Chicago,US,en,texas|gannett,
 gpbnews,https://www.gpb.org/news,Georgia Public Broadcasting,Atlanta,America/New_York,US,en,georgia,
 gradynewsource,https://gradynewsource.uga.edu/,Grady Newsource,Athens,America/New_York,US,en,georgia|student,
-gridnews,https://www.grid.news/,Grid,Washington,America/New_York,US,en,capitol-hill,15000
 grist,https://grist.org/,Grist,Seattle,America/Los_Angeles,US,en,,
 guardian,https://theguardian.com/uk,The Guardian,London,Europe/London,GB,en,,
 hartfordcourant,https://www.courant.com/,Hartford Courant,Hartford,America/New_York,US,en,connecticut,

diff --git a/setup.py b/setup.py
@@ -87,14 +87,12 @@ def local_version(version):
         "python-telegram-bot",
         "discord.py",
         "playwright",
-        "savepagenow",
         "bs4",
         "html5lib",
         "jinja2",
         "shot-scraper",
         "pillow",
         "pandas",
-        "selenium<4.3.0",
         "rich",
         "iso3166",
         "python-iso639",
@@ -111,7 +109,6 @@ def local_version(version):
         "Development Status :: 5 - Production/Stable",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",