forked from 2factorauth/twofactorauth
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathvalidate-urls.rb
executable file
·51 lines (41 loc) · 1.68 KB
/
validate-urls.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
#!/usr/bin/env ruby
# frozen_string_literal: true
require 'json'
require 'net/http'
require 'uri'
require 'parallel'
# Fetch created/modified files in entries/**
diff = `git diff --name-only --diff-filter=AM origin/master...HEAD entries/`.split("\n")
@headers = {
'User-Agent' => "2factorauth/URLValidator (Ruby/#{RUBY_VERSION}; +https://2fa.directory/bot)",
'From' => 'https://2fa.directory/'
}
# Check if the supplied URL works
# rubocop:disable Metrics/AbcSize
# rubocop:disable Metrics/MethodLength
def check_url(path, url, res = nil)
depth = 0
loop do
depth += 1
raise('Too many redirections') if depth > 5
url = URI.parse(url)
res = Net::HTTP.get_response(url)
break unless res.is_a? Net::HTTPRedirection
url = URI(res['location']).is_a?(URI::HTTP) ? res['location'] : "#{url.scheme}://#{url.host}#{res['location']}"
end
puts "::warning file=#{path}:: Unexpected response from #{url} (#{res.code})" unless res.is_a? Net::HTTPSuccess
rescue StandardError => e
puts "::warning file=#{path}:: Unable to reach #{url}"
puts "::debug:: #{e.message}" unless e.instance_of?(TypeError)
end
# rubocop:enable Metrics/AbcSize
# rubocop:enable Metrics/MethodLength
Parallel.each(diff, progress: 'Validating URLs') do |path|
entry = JSON.parse(File.read(path)).values[0]
# Process the url,domain & additional-domains
check_url(path, (entry.key?('url') ? entry['url'] : "https://#{entry['domain']}/"))
entry['additional-domains']&.each { |domain| check_url(path, "https://#{domain}/") }
# Process documentation and recovery URLs
check_url(path, entry['documentation']) if entry.key? 'documentation'
check_url(path, entry['recovery']) if entry.key? 'recovery'
end