Class: Fluent::GithubActivities::Crawler
- Inherits:
-
Object
- Object
- Fluent::GithubActivities::Crawler
show all
- Defined in:
- lib/fluent/plugin/github-activities/crawler.rb
Defined Under Namespace
Classes: EmptyRequestQueue
Constant Summary
collapse
- NO_INTERVAL =
0
- DEFAULT_INTERVAL =
1
- RELATED_USER_IMAGE_KEY =
"$github-activities-related-avatar"
- RELATED_ORGANIZATION_IMAGE_KEY =
"$github-activities-related-organization-logo"
- RELATED_EVENT =
"$github-activities-related-event"
- MERGE_COMMIT_MESSAGE_PATTERN =
/\AMerge pull request #\d+ from [^\/]+\/[^\/]+\n\n/
Instance Attribute Summary collapse
Instance Method Summary
collapse
Constructor Details
#initialize(options = {}) ⇒ Crawler
Returns a new instance of Crawler.
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 43
def initialize(options={})
@users_manager = UsersManager.new(:users => options[:watching_users],
:pos_file => options[:pos_file])
@access_token = options[:access_token]
@watching_users = options[:watching_users] || []
@include_commits_from_pull_request = options[:include_commits_from_pull_request]
@include_foreign_commits = options[:include_foreign_commits]
@request_queue = options[:request_queue] || []
@default_interval = options[:default_interval] || DEFAULT_INTERVAL
end
|
Instance Attribute Details
#interval_for_next_request ⇒ Object
Returns the value of attribute interval_for_next_request.
41
42
43
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 41
def interval_for_next_request
@interval_for_next_request
end
|
#on_emit=(value) ⇒ Object
Sets the attribute on_emit
40
41
42
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 40
def on_emit=(value)
@on_emit = value
end
|
#request_queue ⇒ Object
Returns the value of attribute request_queue.
41
42
43
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 41
def request_queue
@request_queue
end
|
Instance Method Details
127
128
129
130
131
132
133
134
135
136
137
138
139
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 127
def (request)
= {}
if request[:previous_entity_tag]
["If-None-Match"] = request[:previous_entity_tag]
elsif request[:type] == TYPE_EVENTS
position = @users_manager.position_for(request[:user])
if position
entity_tag = position["entity_tag"]
["If-None-Match"] = entity_tag if entity_tag
end
end
end
|
#process_commit(commit, push_event) ⇒ Object
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 208
def process_commit(commit, push_event)
$log.debug("GithubActivities::Crawler: processing commit #{commit["sha"]}") if $log
user = commit["author"]["login"]
if user and (@include_foreign_commits or watching_user?(user))
commit[RELATED_USER_IMAGE_KEY] = push_event["actor"]["avatar_url"]
if push_event["org"]
commit[RELATED_ORGANIZATION_IMAGE_KEY] = push_event["org"]["avatar_url"]
end
commit[RELATED_EVENT] = push_event
emit("commit", commit)
end
commit_refs = push_event["payload"]["commits"]
target_commit_ref = commit_refs.find do |commit_ref|
commit_ref["sha"] == commit["sha"]
end
target_commit_ref["commit"] = commit if target_commit_ref
completely_fetched = commit_refs.all? do |commit_ref|
commit_ref["commit"]
end
emit("push", push_event) if completely_fetched
end
|
#process_create_event(event) ⇒ Object
296
297
298
299
300
301
302
303
304
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 296
def process_create_event(event)
payload = event["payload"]
case payload["ref_type"]
when "branch"
emit("branch", event)
when "tag"
emit("tag", event)
end
end
|
#process_issue_event(event) ⇒ Object
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 237
def process_issue_event(event)
payload = event["payload"]
case payload["action"]
when "opened"
emit("issue-open", event)
when "closed"
emit("issue-close", event)
when "reopened"
emit("issue-reopen", event)
when "assigned"
emit("issue-assign", event)
when "unassigned"
emit("issue-unassign", event)
when "labeled"
emit("issue-label", event)
when "unlabeled"
emit("issue-unlabel", event)
end
end
|
286
287
288
289
290
291
292
293
294
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 286
def (event)
payload = event["payload"]
if payload["issue"]["pull_request"]
emit("pull-request-comment", event)
else
emit("issue-comment", event)
end
end
|
#process_pull_request_event(event) ⇒ Object
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 257
def process_pull_request_event(event)
payload = event["payload"]
case payload["action"]
when "opened"
emit("pull-request", event)
when "closed"
if payload["pull_request"]["merged"]
emit("pull-request-merged", event)
else
emit("pull-request-cancelled", event)
end
when "reopened"
emit("pull-request-reopen", event)
end
end
|
#process_push_event(event) ⇒ Object
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 192
def process_push_event(event)
payload = event["payload"]
commit_refs = payload["commits"]
if !@include_commits_from_pull_request and
push_event_from_merged_pull_request?(event)
return
end
commit_refs.reverse.each do |commit_ref|
@request_queue.push(:type => TYPE_COMMIT,
:uri => commit_ref["url"],
:sha => commit_ref["sha"],
:push => event)
end
end
|
#process_request ⇒ Object
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 59
def process_request
request = @request_queue.shift
$log.debug("GithubActivities::Crawler: processing request: #{request.inspect}") if $log
if request[:process_after] and
Time.now.to_i < request[:process_after]
@request_queue.push(request)
@interval_for_next_request = NO_INTERVAL
return false
end
uri = request_uri(request)
= (request)
$log.debug("GithubActivities::Crawler: requesting to #{uri.inspect}") if $log
response = http_get(uri, )
$log.debug("GithubActivities::Crawler: response: #{response.inspect}") if $log
case response
when Net::HTTPSuccess
$log.trace("GithubActivities::Crawler: Net::HTTPSuccess / request type: #{request[:type]}") if $log
body = JSON.parse(response.body)
case request[:type]
when TYPE_EVENTS
events = body
$log.trace("GithubActivities::Crawler: events size: #{events.size}") if $log
process_user_events(request[:user], events)
reserve_user_events(request[:user], :previous_response => response)
@users_manager.save_position_for(request[:user], :entity_tag => response["ETag"])
when TYPE_COMMIT
process_commit(body, request[:push])
end
when Net::HTTPNotModified
$log.trace("GithubActivities::Crawler: Net::HTTPNotModified / request type: #{request[:type]}") if $log
case request[:type]
when TYPE_EVENTS
reserve_user_events(request[:user],
:previous_response => response,
:previous_entity_tag => ["If-None-Match"])
end
@interval_for_next_request = @default_interval
return true
else
$log.trace("GithubActivities::Crawler: UnknownType / request type: #{request[:type]}") if $log
case request[:type]
when TYPE_COMMIT
fake_body = {
"sha" => request[:sha],
"author" => {},
}
process_commit(fake_body, request[:push])
end
end
@interval_for_next_request = @default_interval
return true
rescue StandardError => error
$log.error(error.inspect)
end
|
#process_user_event(user, event) ⇒ Object
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 164
def process_user_event(user, event)
event[RELATED_USER_IMAGE_KEY] = event["actor"]["avatar_url"]
if event["org"]
event[RELATED_ORGANIZATION_IMAGE_KEY] = event["org"]["avatar_url"]
end
case event["type"]
when "PushEvent"
process_push_event(event)
when "CommitCommentEvent"
emit("commit-comment", event)
when "IssuesEvent"
process_issue_event(event)
when "IssueCommentEvent"
(event)
when "ForkEvent"
emit("fork", event)
when "PullRequestEvent"
process_pull_request_event(event)
when "CreateEvent"
process_create_event(event)
else
emit(event["type"], event)
end
rescue StandardError => error
$log.exception(error)
end
|
#process_user_events(user, events) ⇒ Object
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 146
def process_user_events(user, events)
last_event_timestamp = UsersManager::DEFAULT_LAST_EVENT_TIMESTAMP
position = @users_manager.position_for(user)
if position and position["last_event_timestamp"]
last_event_timestamp = position["last_event_timestamp"]
end
events = events.sort do |a, b|
b["created_at"] <=> a["created_at"]
end
events.each do |event|
timestamp = Time.parse(event["created_at"]).to_i
next if timestamp <= last_event_timestamp
process_user_event(user, event)
@users_manager.save_position_for(user, :last_event_timestamp => timestamp)
end
end
|
#push_event_from_merged_pull_request?(event) ⇒ Boolean
275
276
277
278
279
280
281
282
283
284
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 275
def push_event_from_merged_pull_request?(event)
payload = event["payload"]
inserted_requests = []
commit_refs = payload["commits"]
if MERGE_COMMIT_MESSAGE_PATTERN =~ commit_refs.last["message"]
true
else
false
end
end
|
#request_uri(request) ⇒ Object
117
118
119
120
121
122
123
124
125
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 117
def request_uri(request)
uri = nil
case request[:type]
when TYPE_EVENTS
uri = user_activities(request[:user])
else
uri = request[:uri]
end
end
|
#reserve_user_events(user, options = {}) ⇒ Object
141
142
143
144
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 141
def reserve_user_events(user, options={})
request = @users_manager.new_events_request(user, options)
@request_queue.push(request)
end
|
#watching_user?(user) ⇒ Boolean
233
234
235
|
# File 'lib/fluent/plugin/github-activities/crawler.rb', line 233
def watching_user?(user)
@watching_users.include?(user)
end
|