forked from lobsters/lobsters
-
Notifications
You must be signed in to change notification settings - Fork 0
/
send_new_webmentions
executable file
·102 lines (87 loc) · 3.23 KB
/
send_new_webmentions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
#!/usr/bin/env ruby
ENV["RAILS_ENV"] ||= "production"
APP_PATH = File.expand_path('../../config/application', __FILE__)
require File.expand_path('../../config/boot', __FILE__)
require APP_PATH
require 'webrick'
Rails.application.require_environment!
LAST_STORY_KEY = "webmentions:last_story_id".freeze
def endpoint_from_body(html)
doc = Nokogiri::HTML(html)
if !doc.css('[rel~="webmention"]').css('[href]').empty?
doc.css('[rel~="webmention"]').css('[href]').attribute('href').value
elsif !doc.css('[rel="http://webmention.org/"]').css('[href]').empty?
doc.css('[rel="http://webmention.org/"]').css('[href]').attribute('href').value
elsif !doc.css('[rel="http://webmention.org"]').css('[href]').empty?
doc.css('[rel="http://webmention.org"]').css('[href]').attribute('href').value
end
end
def endpoint_from_headers(header)
return unless header
if (matches = header.match(/<([^>]+)>; rel="[^"]*\s?webmention\s?[^"]*"/))
return matches[1]
elsif (matches = header.match(/<([^>]+)>; rel=webmention/))
return matches[1]
elsif (matches = header.match(/rel="[^"]*\s?webmention\s?[^"]*"; <([^>]+)>/))
return matches[1]
elsif (matches = header.match(/rel=webmention; <([^>]+)>/))
return matches[1]
elsif (matches = header.match(/<([^>]+)>; rel="http:\/\/webmention\.org\/?"/))
return matches[1]
elsif (matches = header.match(/rel="http:\/\/webmention\.org\/?"; <([^>]+)>/))
return matches[1]
end
end
# Some pages could return a relative link as their webmention endpoint.
# We need to translate this relative likn to an absolute one.
def uri_to_absolute(uri, req_uri)
abs_uri = URI.parse(uri)
if abs_uri.host
# Already absolute.
return uri
else
abs_uri.host = req_uri.host
abs_uri.scheme = req_uri.scheme
abs_uri.port = req_uri.port
return abs_uri
end
end
def send_webmention(source, target, endpoint)
sp = Sponge.new
sp.timeout = 10
# Don't check SSL certificate here for backward compatibility, security risk
# is minimal.
sp.ssl_verify = false
sp.fetch(endpoint.to_s, :post, {
"source" => URI.encode_www_form_component(source),
"target" => URI.encode_www_form_component(target),
}, nil, {}, 3)
end
if __FILE__ == $PROGRAM_NAME
last_story_id = (
Keystore.value_for(LAST_STORY_KEY) ||
Story.order('id desc').limit(1).offset(1).pluck(:id).try(:first)
).to_i
Story.where("id > ? AND is_expired = ?", last_story_id, false).order(:id).each do |s|
# mark it done so we don't hit them again if we or they crash
Keystore.put(LAST_STORY_KEY, s.id)
next unless s.url.present?
sp = Sponge.new
sp.timeout = 10
begin
response = sp.fetch(WEBrick::HTTPUtils.escape(s.url), :get, nil, nil, {
"User-agent" => "#{Rails.application.domain} webmention endpoint lookup",
}, 3)
rescue NoIPsError, DNSError
# other people's DNS issues (usually transient); just skip the webmention
next
end
next unless response
wm_endpoint_raw = endpoint_from_headers(response['link']) ||
endpoint_from_body(response.body.to_s)
next unless wm_endpoint_raw
wm_endpoint = uri_to_absolute(wm_endpoint_raw, URI.parse(s.url))
send_webmention(s.short_id_url, s.url, wm_endpoint)
last_story_id = s.id
end
end