Method: Webhookdb::Backfiller#backfill

Defined in:
lib/webhookdb/backfiller.rb

#backfill(last_backfilled) ⇒ Object

Use nil last_backfilled for a full sync, pass it for an incremental. Should be service integration last_backfilled_at, the timestamp of the latest resource, etc.



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/webhookdb/backfiller.rb', line 17

def backfill(last_backfilled)
  pagination_token = nil
  loop do
    page, next_pagination_token = self._fetch_backfill_page_with_retry(
      pagination_token, last_backfilled:,
    )
    if page.nil?
      msg = "Fetching a page should return an empty array, not nil. The service response probably is missing a key?"
      raise TypeError, msg
    end
    pagination_token = next_pagination_token
    page.each do |item|
      self.handle_item(item)
    end
    Amigo::DurableJob.heartbeat
    break if pagination_token.blank?
    if Webhookdb.regression_mode?
      Webhookdb.logger.warn("regression_mode_backfill_termination", backfiller: self.to_s, pagination_token:)
      break
    end
  end
  self.flush_pending_inserts if self.respond_to?(:flush_pending_inserts)
end