Module: Webhookdb::Replicator::GithubRepoV1Mixin

Included in:
GithubIssueCommentV1, GithubIssueV1, GithubPullV1, GithubReleaseV1, GithubRepositoryEventV1
Defined in:
lib/webhookdb/replicator/github_repo_v1_mixin.rb

Overview

Mixin for repo-specific resources like issues and pull requests.

Constant Summary collapse

API_VERSION = 
"2022-11-28"
JSON_CONTENT_TYPE = 
"application/vnd.github+json"

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#service_integration ⇒ Webhookdb::ServiceIntegration

Returns:



17
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 17

def _mixin_backfill_url = raise NotImplementedError("/issues, /pulls, etc")

Class Method Details

._api_docs_url(tail) ⇒ Object 



10
11
12
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 10

def self._api_docs_url(tail)
  return "https://docs.github.com/en/rest#{tail}?apiVersion=#{API_VERSION}"
end

Instance Method Details

#_fetch_backfill_page(pagination_token, last_backfilled:) ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 183

def _fetch_backfill_page(pagination_token, last_backfilled:)
  if pagination_token.present?
    url = pagination_token
    query = {}
  else
    url = "https://api.github.com/repos/#{self.service_integration.api_url}#{self._mixin_backfill_url}"
    query = {per_page: 100}
    query.merge!(self._mixin_query_params(last_backfilled:))
  end
  response, data = self._http_get(url, query)
  next_link = nil
  if response.headers.key?("link")
    links = Webhookdb::Github.parse_link_header(response.headers["link"])
    next_link = links[:next] if links.key?(:next)
  end
  return data, next_link
end

#_fetch_enrichment(resource, _event, _request) ⇒ Object 



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 219

def _fetch_enrichment(resource, _event, _request)
  # If we're not set up to backfill, we cannot make an API call.
  return nil if self.service_integration.backfill_secret.nil?
  # We should fetch the full resource if the replicator needs it,
  # and the resource does not have the key we require.
  sentinel_key = self._mixin_fetch_resource_if_field_missing
  return nil if sentinel_key.nil? || resource.key?(sentinel_key)
  resource_url = resource.fetch("url")
  begin
    _response, data = self._http_get(resource_url, {})
  rescue Webhookdb::Http::Error => e
    # If the HTTP call fails due to an auth issue (or a deleted item),
    # we should still upsert what we have.
    # Tokens expire or can be revoked, but we don't want the webhook to stop inserting.
    ignore_error = [401, 403, 404].include?(e.response.code)
    return nil if ignore_error
    raise e
  end
  return data
end

#_fullreponame ⇒ Object 



30
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 30

def _fullreponame = self.service_integration.api_url

#_handle_repo_name_state_machine(step, tfield) ⇒ Object

If api_url isn’t set, prompt for it (via repo_name or api_url field).



110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 110

def _handle_repo_name_state_machine(step, tfield)
  if self.service_integration.api_url.blank?
    step.output = %(You are about to start replicating #{self.resource_name_plural} for a repository into WebhookDB.

First we need the full repository name, like 'webhookdb/webhookdb-cli'.)
    step.set_prompt("Repository name:").transition_field(self.service_integration, tfield)
    return true
  end
  return false if self._valid_repo_name?(self.service_integration.api_url)
  step.output = %(That repository is not valid. Include both the owner and name, like 'webhookdb/webhookdb-cli'.)
  step.set_prompt("Repository name:").transition_field(self.service_integration, tfield)
  return true
end

#_http_get(url, query) ⇒ Object 



201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 201

def _http_get(url, query)
  response = Webhookdb::Http.get(
    url,
    query,
    headers: {
      "Accept" => JSON_CONTENT_TYPE,
      "Authorization" => "Bearer #{self.service_integration.backfill_secret}",
      "X-GitHub-Api-Version" => API_VERSION,
    },
    logger: self.logger,
    timeout: Webhookdb::Github.http_timeout,
  )
  # Handle the GH-specific vnd JSON or general application/json
  parsed = response.parsed_response
  (parsed = Oj.load(parsed)) if response.headers["Content-Type"] == JSON_CONTENT_TYPE
  return response, parsed
end

#_is_repo_public? ⇒ Boolean

If we can make an unauthed request and find the repo, it is public.

Returns:

  • (Boolean) 


125
126
127
128
129
130
131
132
133
134
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 125

def _is_repo_public?
  resp = Webhookdb::Http.post(
    "https://github.com/#{self.service_integration.api_url}",
    method: :head,
    check: false,
    timeout: 5,
    logger: nil,
  )
  return resp.code == 200
end

#_mixin_backfill_url ⇒ Webhookdb::ServiceIntegration

Returns:



17
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 17

def _mixin_backfill_url = raise NotImplementedError("/issues, /pulls, etc")

#_mixin_fetch_resource_if_field_missing ⇒ Object

Some resources, like issues and pull requests, have a ‘simple’ representation in the list, and a full representation when fetched individually. Return the field that can be used to determine if the full resource needs to be fetched.



28
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 28

def _mixin_fetch_resource_if_field_missing = nil

#_mixin_fine_grained_permission ⇒ Object

github.com/settings/personal-access-tokens/new



22
23
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 22

def _mixin_fine_grained_permission = raise NotImplementedError("Issues", etc)
# Query params to use in the list call. Should include sorting when available.

#_mixin_query_params(last_backfilled:) ⇒ Object

Query params to use in the list call. Should include sorting when available.

Raises:

  • (NotImplementedError) 


24
25
26
27
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 24

def _mixin_query_params(last_backfilled:) = raise NotImplementedError
# Some resources, like issues and pull requests, have a 'simple' representation
# in the list, and a full representation when fetched individually.
# Return the field that can be used to determine if the full resource needs to be fetched.

#_mixin_webhook_events ⇒ Object 



18
19
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 18

def _mixin_webhook_events = raise NotImplementedError("Issues, Pulls, Issue comments, etc")
# https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=demilestoned#issues

#_mixin_webhook_key ⇒ Object

docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=demilestoned#issues



20
21
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 20

def _mixin_webhook_key = raise NotImplementedError("issue, etc")
# https://github.com/settings/personal-access-tokens/new

#_prepare_for_insert(resource, event, request, enrichment) ⇒ Object 



256
257
258
259
260
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 256

def _prepare_for_insert(resource, event, request, enrichment)
  # if enrichment is not nil, it's the detailed resource.
  # See _mixin_fetch_resource_if_field_missing
  return super(enrichment || resource, event, request, nil)
end

#_reponame ⇒ Object 



32
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 32

def _reponame = self._fullreponame.split("/").last

#_repoowner ⇒ Object 



31
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 31

def _repoowner = self._fullreponame.split("/").first

#_resource_and_event(request) ⇒ Object

Extract the resource from the request. The resource can be a normal resource, or a webhook, with X-GitHub-Hook-ID key as per docs.github.com/en/webhooks/webhook-events-and-payloads The headers are the only things that identify a webhook payload consistently.

Note that webhooks to a given integration can be for events we do not expect, such as someone sending events we aren’t handling (ie, if they don’t uncheck Pushes, we may get push events sent to the github_issue_v1 integration), and also for automated events like ‘ping’.



44
45
46
47
48
49
50
51
52
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 44

def _resource_and_event(request)
  # Note the canonical casing on the header name. GitHub sends X-GitHub-Hook-ID
  # but it's normalized here.
  is_webhook = (request.headers || {})["X-Github-Hook-Id"]
  return request.body, nil unless is_webhook
  resource = request.body.fetch(self._mixin_webhook_key, nil)
  return nil, nil if resource.nil?
  return resource, request.body
end

#_resource_to_data(resource, _event, _request, enrichment) ⇒ Object 



262
263
264
265
266
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 262

def _resource_to_data(resource, _event, _request, enrichment)
  # if enrichment is not nil, it's the detailed resource.
  # See _mixin_fetch_resource_if_field_missing
  return enrichment || resource
end

#_update_where_expr ⇒ Object 



54
55
56
57
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 54

def _update_where_expr
  ts = self._timestamp_column_name
  return self.qualified_table_sequel_identifier[ts] < Sequel[:excluded][ts]
end

#_valid_repo_name?(s) ⇒ Boolean

Returns:

  • (Boolean) 


33
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 33

def _valid_repo_name?(s) = %r{^[\w\-.]+/[\w\-.]+$} =~ s

#_verify_backfill_err_msg ⇒ Object 



177
178
179
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 177

def _verify_backfill_err_msg
  return "That access token didn't seem to work. Please look over the instructions and try again."
end

#_webhook_response(request) ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 59

def _webhook_response(request)
  hash = request.env["HTTP_X_HUB_SIGNATURE_256"]
  return Webhookdb::WebhookResponse.error("missing sha256") if hash.nil?
  secret = self.service_integration.webhook_secret
  return Webhookdb::WebhookResponse.error("no secret set, run `webhookdb integration setup`", status: 409) if
    secret.nil?
  request.body.rewind
  request_data = request.body.read
  verified = Webhookdb::Github.verify_webhook(request_data, hash, secret)
  return Webhookdb::WebhookResponse.ok if verified
  return Webhookdb::WebhookResponse.error("invalid sha256")
end

#_webhook_state_change_fields ⇒ Object 



72
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 72

def _webhook_state_change_fields = super + ["repo_name"]

#calculate_backfill_state_machine ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 136

def calculate_backfill_state_machine
  step = Webhookdb::Replicator::StateMachineStep.new
  return step if self._handle_repo_name_state_machine(step, "api_url")
  unless self.service_integration.backfill_secret.present?
    repo_public = self._is_repo_public?
    step.output = %(In order to backfill #{self.resource_name_plural},
WebhookDB requires an access token to authenticate.

You should go to https://github.com/settings/personal-access-tokens/new and create a new Personal Access Token.

For 'Expiration', give a custom date far in the future.

For 'Resource owner', choose the '#{self._repoowner}' organization.
**If it does not appear**, Fine-grained tokens are not enabled.
See instructions below.

For 'Repository access', choose 'Only select repositories', and the '#{self._fullreponame}' repository.

For 'Repository permissions', go to '#{self._mixin_fine_grained_permission}' and choose 'Read-only access'.

If you didn't see the needed owner under 'Resource owner,' it's because fine-grained tokens are not enabled.
Instead, create a new Classic personal access token from https://github.com/settings/tokens/new.
In the 'Note', mention this token is for WebhookDB,
give it an expiration, and under 'Scopes', ensure #{repo_public ? 'repo->public_repo' : 'repo'} is checked,
since #{self._fullreponame} is #{repo_public ? 'public' : 'private'}.

Then click 'Generate token'.)
    return step.secret_prompt("Personal access token").backfill_secret(self.service_integration)
  end

  unless (result = self.verify_backfill_credentials).verified
    self.service_integration.replicator.clear_backfill_information
    step.output = result.message
    return step.secret_prompt("Personal access token").backfill_secret(self.service_integration)
  end

  step.output = %(Great! We are going to start backfilling your #{self.resource_name_plural}.
#{self._query_help_output})
  return step.completed
end

#calculate_webhook_state_machine ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 79

def calculate_webhook_state_machine
  step = Webhookdb::Replicator::StateMachineStep.new
  return step if self._handle_repo_name_state_machine(step, "repo_name")
  if self.service_integration.webhook_secret.blank?
    step.output = %(Now, head to this route to create a webhook:

https://github.com/#{self.service_integration.api_url}/settings/hooks/new

For 'Payload URL', use this endpoint that is now available:

#{self._webhook_endpoint}

For 'Content type', choose 'application/json'. Form encoding works but loses some detail in events.

For 'Secret', choose your own secure secret, or use this one: '#{Webhookdb::Id.rand_enc(16)}'

For 'Which events would you like to trigger this webhook',
choose 'Let me select individual events',
uncheck 'Pushes', and select the following:

#{self._mixin_webhook_events.join("\n  ")}

Make sure 'Active' is checked, and press 'Add webhook'.)
    return step.secret_prompt("Webhook Secret").webhook_secret(self.service_integration)
  end
  step.output = %(Great! WebhookDB is now listening for #{self.resource_name_singular} webhooks.
#{self._query_help_output})
  return step.completed
end

#on_backfill_error(be) ⇒ Object 



240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 240

def on_backfill_error(be)
  e = Webhookdb::Errors.find_cause(be) do |ex|
    next true if ex.is_a?(Webhookdb::Http::Error) && ex.status == 401
  end
  return unless e
  message = Webhookdb::Messages::ErrorGenericBackfill.new(
    self.service_integration,
    response_status: e.status,
    response_body: e.body,
    request_url: e.uri.to_s,
    request_method: e.http_method,
  )
  self.service_integration.organization.alerting.dispatch_alert(message)
  return true
end

#process_state_change(field, value) ⇒ Object 



74
75
76
77
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 74

def process_state_change(field, value)
  attr = field == "repo_name" ? "api_url" : field
  return super(field, value, attr:)
end