Class: Webhookdb::Replicator::AtomSingleFeedV1

Inherits:
Base
  • Object
show all
Includes:
Appydays::Loggable
Defined in:
lib/webhookdb/replicator/atom_single_feed_v1.rb

Defined Under Namespace

Classes: Backfiller

Constant Summary collapse

CONV_GEO_LAT = 
Webhookdb::Replicator::Column.converter_array_element(index: 0, sep: " ", cls: DECIMAL)
CONV_GEO_LNG = 
Webhookdb::Replicator::Column.converter_array_element(index: 1, sep: " ", cls: DECIMAL)

Constants inherited from Base

Base::MAX_INDEX_NAME_LENGTH

Constants included from DBAdapter::ColumnTypes

DBAdapter::ColumnTypes::BIGINT, DBAdapter::ColumnTypes::BIGINT_ARRAY, DBAdapter::ColumnTypes::BOOLEAN, DBAdapter::ColumnTypes::COLUMN_TYPES, DBAdapter::ColumnTypes::DATE, DBAdapter::ColumnTypes::DECIMAL, DBAdapter::ColumnTypes::DOUBLE, DBAdapter::ColumnTypes::FLOAT, DBAdapter::ColumnTypes::INTEGER, DBAdapter::ColumnTypes::INTEGER_ARRAY, DBAdapter::ColumnTypes::OBJECT, DBAdapter::ColumnTypes::TEXT, DBAdapter::ColumnTypes::TEXT_ARRAY, DBAdapter::ColumnTypes::TIMESTAMP, DBAdapter::ColumnTypes::UUID

Instance Attribute Summary

Attributes inherited from Base

#service_integration

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Base

#_any_subscriptions_to_notify?, #_backfill_state_change_fields, #_clear_backfill_information, #_clear_webook_information, #_coalesce_excluded_on_update, #_enqueue_backfill_jobs, #_extra_index_specs, #_fetch_enrichment, #_find_dependency_candidate, #_notify_dependents, #_parallel_backfill, #_prepare_for_insert, #_publish_rowupsert, #_resource_to_data, #_store_enrichment_body?, #_to_json, #_upsert_webhook, #_webhook_state_change_fields, #admin_dataset, #backfill, #backfill_not_supported_message, #calculate_and_backfill_state_machine, #calculate_dependency_state_machine_step, #calculate_preferred_create_state_machine, #calculate_webhook_state_machine, chunked_row_update_bounds, #clear_backfill_information, #clear_webhook_information, #create_table, #create_table_modification, #data_column, #dbadapter_table, #denormalized_columns, #descriptor, #dispatch_request_to, #documentation_url, #enqueue_sync_targets, #enrichment_column, #ensure_all_columns, #ensure_all_columns_modification, #find_dependent, #find_dependent!, #indices, #initialize, #on_dependency_webhook_upsert, #preferred_create_state_machine_method, #preprocess_headers_for_logging, #primary_key_column, #process_state_change, #process_webhooks_synchronously?, #qualified_table_sequel_identifier, #readonly_dataset, #remote_key_column, #requires_sequence?, #resource_name_plural, #resource_name_singular, #schema_and_table_symbols, #storable_columns, #synchronous_processing_response_body, #timestamp_column, #upsert_has_deps?, #upsert_webhook, #upsert_webhook_body, #verify_backfill_credentials, #webhook_endpoint, #webhook_response

Constructor Details

This class inherits a constructor from Webhookdb::Replicator::Base

Class Method Details

.descriptor ⇒ Webhookdb::Replicator::Descriptor

Returns:



11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 11

def self.descriptor
  return Webhookdb::Replicator::Descriptor.new(
    name: "atom_single_feed_v1",
    ctor: ->(sint) { Webhookdb::Replicator::AtomSingleFeedV1.new(sint) },
    feature_roles: [],
    resource_name_singular: "Atom Single Feed",
    supports_backfill: true,
    description: "Convert any Atom XML feed into a database table for querying and persistent archiving.",
    api_docs_url: "https://en.wikipedia.org/wiki/Atom_(web_standard)",
  )
end

Instance Method Details

#_backfillers ⇒ Object 



71
72
73
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 71

def _backfillers
  return [Backfiller.new(self)]
end

#_denormalized_columns ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 30

def _denormalized_columns
  col = Webhookdb::Replicator::Column
  return [
    col.new(:row_created_at, TIMESTAMP, index: true, optional: true, defaulter: :now),
    col.new(:updated, TIMESTAMP, index: true),
    col.new(:title, TEXT),
    col.new(:published, TIMESTAMP, index: true, optional: true),
    col.new(:geo_lat, DECIMAL, data_key: "georss:point", optional: true, converter: CONV_GEO_LAT),
    col.new(:geo_lng, DECIMAL, data_key: "georss:point", optional: true, converter: CONV_GEO_LNG),
  ]
end

#_fetch_backfill_page ⇒ Object 



61
62
63
64
65
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 61

def _fetch_backfill_page(*)
  io = Webhookdb::Http.get(self.service_integration.api_url, logger: self.logger, timeout: 30)
  feed_obj = Webhookdb::Xml::Atom.parse(io.body)
  return feed_obj.fetch("entries"), nil
end

#_remote_key_column ⇒ Object 



23
24
25
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 23

def _remote_key_column
  return Webhookdb::Replicator::Column.new(:entry_id, TEXT, data_key: "id")
end

#_resource_and_event(request) ⇒ Object 



46
47
48
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 46

def _resource_and_event(request)
  return request.body, nil
end

#_timestamp_column_name ⇒ Object 



42
43
44
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 42

def _timestamp_column_name
  return :updated
end

#_update_where_expr ⇒ Object 



50
51
52
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 50

def _update_where_expr
  return self.qualified_table_sequel_identifier[:updated] < Sequel[:excluded][:updated]
end

#_upsert_update_expr(inserting, **_kwargs) ⇒ Object 



54
55
56
57
58
59
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 54

def _upsert_update_expr(inserting, **_kwargs)
  update = super
  # Only set created_at if it's not set so the initial insert isn't modified.
  self._coalesce_excluded_on_update(update, [:row_created_at])
  return update
end

#_verify_backfill_err_msg ⇒ Object 



67
68
69
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 67

def _verify_backfill_err_msg
  return "Sorry, we can't reach that URL. Please double check it and try again."
end

#_webhook_response(_request) ⇒ Object 



89
90
91
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 89

def _webhook_response(_request)
  return Webhookdb::WebhookResponse.ok
end

#calculate_backfill_state_machine ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/webhookdb/replicator/atom_single_feed_v1.rb', line 93

def calculate_backfill_state_machine
  step = Webhookdb::Replicator::StateMachineStep.new
  # Must set this to fake out having credentials.
  self.service_integration.update(backfill_key: "placeholder") if self.service_integration.backfill_key.blank?
  if self.service_integration.api_url.blank?
    step.output = %(You're about to sync entries from an Atom URL into WebhookDB.
This will create a row for each 'entry' in the given feed,
and insert/update new rows periodically.

Paste in the URL to sync, and press Enter.)
    return step.prompting("URL").api_url(self.service_integration)
  end
  unless (result = self.verify_backfill_credentials).verified
    self.service_integration.update(api_url: "")
    step.output = result.message
    return step.prompting("URL").api_url(self.service_integration)
  end
  step.output = %(
All set! Your feed will be synced momentarily and then every few hours after that.

#{self._query_help_output})
  return step.completed
end