Class: Webhookdb::Replicator::IcalendarEventV1

Inherits:
Base
  • Object
show all
Includes:
Appydays::Loggable
Defined in:
lib/webhookdb/replicator/icalendar_event_v1.rb

Constant Summary collapse

CONV_REMOTE_KEY = 
Webhookdb::Replicator::Column::IsomorphicProc.new(
  ruby: lambda do |_, resource:, **_|
    "#{resource.fetch('calendar_external_id')}-#{resource.fetch('UID').fetch('v')}"
  end,
  # Because this is a non-nullable key, we never need this in SQL
  sql: ->(_) { Sequel.lit("'do not use'") },
)
CONV_DATE = 
Webhookdb::Replicator::Column::IsomorphicProc.new(
  ruby: lambda do |entry, **|
    self.entry_to_date(entry) if entry.is_a?(Hash) && self.entry_is_date_str?(entry)
  end,
  sql: ->(_) { raise NotImplementedError },
)
CONV_DATETIME = 
Webhookdb::Replicator::Column::IsomorphicProc.new(
  ruby: lambda do |entry, **|
    if entry.is_a?(Hash)
      if self.entry_is_date_str?(entry)
        nil
      else
        self.entry_to_datetime(entry).first
      end
    else
      # Entry may be a time if this was from the defaulter
      entry
    end
  end,
  sql: ->(_) { raise NotImplementedError },
)
CONV_MISSING_TZ = 
Webhookdb::Replicator::Column::IsomorphicProc.new(
  ruby: lambda do |entry, **|
    may_have_missing_tz = entry.is_a?(Hash) && !self.entry_is_date_str?(entry)
    if may_have_missing_tz
      tzparsed = self.entry_to_datetime(entry)[1]
      !tzparsed
    else
      false
    end
  end,
  sql: ->(_) { Sequel[false] },
)
CONV_GEO_LAT = 
Webhookdb::Replicator::Column.converter_array_element(index: 0, sep: ";", cls: DECIMAL)
CONV_GEO_LNG = 
Webhookdb::Replicator::Column.converter_array_element(index: 1, sep: ";", cls: DECIMAL)
CONV_COMMA_SEP_ARRAY = 
Webhookdb::Replicator::Column::IsomorphicProc.new(
  ruby: lambda do |entry, **|
    next [] if entry.nil?
    entries = []
    entry.each do |e|
      entries.concat(e.fetch("v").split(",").map(&:strip))
    end
    entries
  end,
  sql: ->(_) { raise NotImplementedError },
)
ARRAY_KEYS =

datatracker.ietf.org/doc/html/rfc5545#section-3.6.1 The following are OPTIONAL, and MAY occur more than once.

[
  "ATTACH",
  "ATTENDEE",
  "CATEGORIES",
  "COMMENT",
  "CONTACT",
  "EXDATE",
  "RSTATUS",
  "RELATED",
  "RESOURCES",
  "RDATE",
  "X-PROP",
  "IANA-PROP",
].freeze
NAME = 
"[-a-zA-Z0-9]+"
QSTR = 
'"[^"]*"'
PTEXT = 
'[^";:,]*'
PVALUE = 
"(?:#{QSTR}|#{PTEXT})".freeze
PARAM = 
"(#{NAME})=(#{PVALUE}(?:,#{PVALUE})*)".freeze
VALUE = 
".*"
LINE = 
"(?<name>#{NAME})(?<params>(?:;#{PARAM})*):(?<value>#{VALUE})".freeze

Constants inherited from Base

Base::MAX_INDEX_NAME_LENGTH

Constants included from DBAdapter::ColumnTypes

DBAdapter::ColumnTypes::BIGINT, DBAdapter::ColumnTypes::BIGINT_ARRAY, DBAdapter::ColumnTypes::BOOLEAN, DBAdapter::ColumnTypes::COLUMN_TYPES, DBAdapter::ColumnTypes::DATE, DBAdapter::ColumnTypes::DECIMAL, DBAdapter::ColumnTypes::DOUBLE, DBAdapter::ColumnTypes::FLOAT, DBAdapter::ColumnTypes::INTEGER, DBAdapter::ColumnTypes::INTEGER_ARRAY, DBAdapter::ColumnTypes::OBJECT, DBAdapter::ColumnTypes::TEXT, DBAdapter::ColumnTypes::TEXT_ARRAY, DBAdapter::ColumnTypes::TIMESTAMP, DBAdapter::ColumnTypes::UUID

Instance Attribute Summary

Attributes inherited from Base

#service_integration

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Base

#_any_subscriptions_to_notify?, #_backfill_state_change_fields, #_backfillers, #_clear_backfill_information, #_clear_webook_information, #_coalesce_excluded_on_update, #_enqueue_backfill_jobs, #_fetch_enrichment, #_find_dependency_candidate, #_notify_dependents, #_parallel_backfill, #_prepare_for_insert, #_publish_rowupsert, #_store_enrichment_body?, #_timestamp_column_name, #_to_json, #_upsert_update_expr, #_upsert_webhook, #_verify_backfill_err_msg, #_webhook_response, #_webhook_state_change_fields, #admin_dataset, #backfill, #calculate_and_backfill_state_machine, #calculate_backfill_state_machine, #calculate_dependency_state_machine_step, #calculate_preferred_create_state_machine, chunked_row_update_bounds, #clear_backfill_information, #clear_webhook_information, #create_table, #create_table_modification, #data_column, #dbadapter_table, #denormalized_columns, #descriptor, #dispatch_request_to, #enqueue_sync_targets, #enrichment_column, #ensure_all_columns, #ensure_all_columns_modification, #find_dependent, #find_dependent!, #indices, #initialize, #preferred_create_state_machine_method, #preprocess_headers_for_logging, #primary_key_column, #process_state_change, #process_webhooks_synchronously?, #qualified_table_sequel_identifier, #readonly_dataset, #remote_key_column, #requires_sequence?, #resource_name_plural, #resource_name_singular, #schema_and_table_symbols, #storable_columns, #synchronous_processing_response_body, #timestamp_column, #upsert_has_deps?, #upsert_webhook, #upsert_webhook_body, #verify_backfill_credentials, #webhook_endpoint, #webhook_response

Constructor Details

This class inherits a constructor from Webhookdb::Replicator::Base

Class Method Details

._compact_vevent_lines(lines) ⇒ String

Parameters:

  • lines (Array<String>)

Returns:

  • (String) 


188
189
190
191
192
193
194
195
196
197
198
199
200
201
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 188

def self._compact_vevent_lines(lines)
  # Walk backwards from the end, joining continuation lines.
  # Very hard to reason about this if it's written using normal array stuff.
  (1..(lines.length - 1)).reverse_each do |idx|
    line = lines[idx]
    prevline = lines[idx - 1]
    next unless line.start_with?(/\s+/)
    line.lstrip!
    prevline << line
    lines.delete_at(idx)
  end
  s = lines.join("\n")
  return s
end

._parse_line(input) ⇒ Object

Parameters:

  • input (String) 


262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 262

def self._parse_line(input)
  parts = /#{LINE}/o.match(input)
  return input, nil, {} if parts.nil?
  params = {}
  parts[:params].scan(/#{PARAM}/o) do |match|
    param_name = match[0]
    # params[param_name] ||= []
    match[1].scan(/#{PVALUE}/o) do |param_value|
      if param_value.size.positive?
        param_value = param_value.gsub(/\A"|"\z/, "")
        params[param_name] = param_value
        # params["x-tz-info"] = timezone_store.retrieve param_value if param_name == "tzid"
      end
    end
  end
  return parts[:name], parts[:value], params
end

._parse_time_with_tzid(value, tzid) ⇒ Object

Given a tzid and value for a timestamp, return a Time (with a timezone). While there’s no formal naming scheme, we see the following forms:

  • valid names like America/Los_Angeles, US/Eastern

  • dashes, like America-Los_Angeles, US-Eastern

  • Offsets, like GMT-0700’

In theory this can be any value, and must be given in the calendar feed (VTIMEZONE). However that is extremely difficult; even the icalendar gem doesn’t seem to do it 100% right. We can solve for this if needed; in the meantime, log it in Sentry and use UTC.

If the zone cannot be parsed, assume UTC.

Return a tuple of [Time, true if the zone could be parsed]. If the zone cannot be parsed, you usually want to log or store it.



294
295
296
297
298
299
300
301
302
303
304
305
306
307
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 294

def self._parse_time_with_tzid(value, tzid)
  if (zone = Time.find_zone(tzid.tr("-", "/")))
    return [zone.parse(value), true]
  end
  if /^(GMT|UTC)[+-]\d\d\d\d$/.match?(tzid)
    offset = tzid[3..]
    return [Time.parse(value + offset), true]
  end
  if (zone = Webhookdb::WindowsTZ.windows_name_to_tz[tzid])
    return [zone.parse(value), true]
  end
  zone = Time.find_zone!("UTC")
  return [zone.parse(value), false]
end

.descriptor ⇒ Webhookdb::Replicator::Descriptor

Returns:



12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 12

def self.descriptor
  return Webhookdb::Replicator::Descriptor.new(
    name: "icalendar_event_v1",
    ctor: ->(sint) { Webhookdb::Replicator::IcalendarEventV1.new(sint) },
    dependency_descriptor: Webhookdb::Replicator::IcalendarCalendarV1.descriptor,
    feature_roles: [],
    resource_name_singular: "iCalendar Event",
    supports_webhooks: true,
    description: "Individual events in an icalendar. See icalendar_calendar_v1.",
    api_docs_url: "https://icalendar.org/",
  )
end

.entry_is_date_str?(e) ⇒ Boolean

Returns:

  • (Boolean) 


41
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 41

def self.entry_is_date_str?(e) = self.value_is_date_str?(e.fetch("v"))

.entry_to_date(entry) ⇒ Date?

Returns:

  • (Date, nil)

Raises:

  • (ArgumentError) 


58
59
60
61
62
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 58

def self.entry_to_date(entry)
  value = entry.fetch("v")
  raise ArgumentError, "must pass a date string" unless self.value_is_date_str?(value)
  return Date.strptime(value, "%Y%m%d")
end

.entry_to_date_or_datetime(entry) ⇒ Array<Time,Date,true,false,nil>

Return tuple of parsed date or datetime, and a boolean of whether the timezone could be parsed (true if date was parsed).

Returns:

  • (Array<Time,Date,true,false,nil>) 


36
37
38
39
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 36

def self.entry_to_date_or_datetime(entry)
  return [self.entry_to_date(entry), true] if self.value_is_date_str?(entry.fetch("v"))
  return self.entry_to_datetime(entry)
end

.entry_to_datetime(entry) ⇒ Array<Time,true,false,nil>

Return tuple of parsed datetime, and a boolean of whether the timezone could be parsed.

Returns:

  • (Array<Time,true,false,nil>)

Raises:

  • (ArgumentError) 


47
48
49
50
51
52
53
54
55
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 47

def self.entry_to_datetime(entry)
  value = entry.fetch("v")
  raise ArgumentError, "do not pass a date string" if self.value_is_date_str?(value)
  return [Time.strptime(value, "%Y%m%dT%H%M%S%Z"), true] if value.end_with?("Z")
  if (tzid = entry["TZID"])
    return self._parse_time_with_tzid(value, tzid)
  end
  return [Time.find_zone!("UTC").parse(value), false]
end

.value_is_date_str?(v) ⇒ Boolean

Returns:

  • (Boolean) 


42
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 42

def self.value_is_date_str?(v) = v.length === 8

.vevent_to_hash(vevent_lines) ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 203

def self.vevent_to_hash(vevent_lines)
  result = {}
  vevent_str = self._compact_vevent_lines(vevent_lines)
  nest_depth = 0
  vevent_str.lines.each do |line|
    if line.start_with?("BEGIN")
      nest_depth += 1
      next
    elsif line.start_with?("END")
      nest_depth -= 1
      next
    end
    next if nest_depth > 1
    line.strip!
    next if line.empty?
    keyname, value, params = self._parse_line(line)
    unless value.nil?
      value.gsub!("\\r\\n", "\r\n")
      value.gsub!("\\n", "\n")
      value.gsub!("\\t", "\t")
    end
    entry = {"v" => value}
    entry.merge!(params)
    if ARRAY_KEYS.include?(keyname)
      result[keyname] ||= []
      result[keyname] << entry
    else
      result[keyname] = entry
    end
  end
  return result
end

Instance Method Details

#_denormalized_columns ⇒ Object 



122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 122

def _denormalized_columns
  col = Webhookdb::Replicator::Column
  tsconv = {converter: CONV_DATETIME}
  dateconv = {converter: CONV_DATE}
  return [
    col.new(:calendar_external_id, TEXT, index: true),
    col.new(:uid, TEXT, data_key: ["UID", "v"], index: true),
    col.new(:row_updated_at, TIMESTAMP, index: true, defaulter: :now, optional: true),
    col.new(:last_modified_at,
            TIMESTAMP,
            index: true,
            data_key: "LAST-MODIFIED",
            defaulter: :now,
            optional: true,
            **tsconv,),
    col.new(:created_at, TIMESTAMP, optional: true, data_key: "CREATED", **tsconv),
    col.new(:start_at, TIMESTAMP, index: true, index_not_null: true, data_key: "DTSTART", **tsconv),
    # This is True when start/end at fields are missing timezones in the underlying feed.
    # Their timestamps are in UTC.
    col.new(:missing_timezone, BOOLEAN, data_key: "DTSTART", converter: CONV_MISSING_TZ),
    col.new(:end_at, TIMESTAMP, index: true, index_not_null: true, data_key: "DTEND", optional: true, **tsconv),
    col.new(:start_date, DATE, index: true, index_not_null: true, data_key: "DTSTART", **dateconv),
    col.new(:end_date, DATE, index: true, index_not_null: true, data_key: "DTEND", optional: true, **dateconv),
    col.new(:status, TEXT, data_key: ["STATUS", "v"], optional: true),
    col.new(:categories, TEXT_ARRAY, data_key: ["CATEGORIES"], optional: true, converter: CONV_COMMA_SEP_ARRAY),
    col.new(:priority, INTEGER, data_key: ["PRIORITY", "v"], optional: true, converter: col::CONV_TO_I),
    col.new(:geo_lat, DECIMAL, data_key: ["GEO", "v"], optional: true, converter: CONV_GEO_LAT),
    col.new(:geo_lng, DECIMAL, data_key: ["GEO", "v"], optional: true, converter: CONV_GEO_LNG),
    col.new(:classification, TEXT, data_key: ["CLASS", "v"], optional: true),
    col.new(:recurring_event_id, TEXT, optional: true, index: true, index_not_null: true),
    col.new(:recurring_event_sequence, INTEGER, optional: true),
  ]
end

#_extra_index_specs ⇒ Array<Webhookdb::Replicator::IndexSpec>

Returns:



169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 169

def _extra_index_specs
  return [
    Webhookdb::Replicator::IndexSpec.new(
      columns: [:calendar_external_id, :start_at, :end_at],
      where: Sequel[:status].is_distinct_from("CANCELLED") & (Sequel[:start_at] !~ nil),
    ),
    Webhookdb::Replicator::IndexSpec.new(
      columns: [:calendar_external_id, :start_date, :end_date],
      where: Sequel[:status].is_distinct_from("CANCELLED") & (Sequel[:start_date] !~ nil),
    ),
  ]
end

#_remote_key_column ⇒ Object 



111
112
113
114
115
116
117
118
119
120
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 111

def _remote_key_column
  return Webhookdb::Replicator::Column.new(
    :compound_identity,
    TEXT,
    data_key: "<compound key, see converter>",
    index: true,
    converter: CONV_REMOTE_KEY,
    optional: true, # This is done via the converter, data_key never exists
  )
end

#_resource_and_event(request) ⇒ Object 



156
157
158
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 156

def _resource_and_event(request)
  return request.body, nil
end

#_resource_to_data(r) ⇒ Object 



160
161
162
163
164
165
166
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 160

def _resource_to_data(r, *)
  data = r.dup
  data.delete("calendar_external_id")
  data.delete("recurring_event_id")
  data.delete("recurring_event_sequence")
  return data
end

#_update_where_expr ⇒ Object 



182
183
184
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 182

def _update_where_expr
  return self.qualified_table_sequel_identifier[:last_modified_at] < Sequel[:excluded][:last_modified_at]
end

#backfill_not_supported_message ⇒ Object 



327
328
329
330
331
332
333
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 327

def backfill_not_supported_message
  return %(#{self.resource_name_singular} does not support backfilling.
See https://docs.webhookdb.com/guides/icalendar/ for instructions on setting up your integration.

You can POST 'SYNC' messages to WebhookDB to force-sync a user's feed,
though keep in mind calendar providers only refresh feeds periodically.)
end

#calculate_webhook_state_machine ⇒ Object 



314
315
316
317
318
319
320
321
322
323
324
325
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 314

def calculate_webhook_state_machine
  if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
    return step
  end
  step = Webhookdb::Replicator::StateMachineStep.new
  step.output = %(Great! You are all set.
Refer to https://docs.webhookdb.com/guides/icalendar/ for detailed instructions
on syncing data from iCalendar/ics feeds.

#{self._query_help_output(prefix: "Once data is available, you can query #{self.resource_name_plural}")})
  return step.completed
end

#documentation_url ⇒ Object 



9
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 9

def documentation_url = Webhookdb::Icalendar::DOCUMENTATION_URL

#on_dependency_webhook_upsert(_ical_svc, _ical_row) ⇒ Object 



309
310
311
312
 
# File 'lib/webhookdb/replicator/icalendar_event_v1.rb', line 309

def on_dependency_webhook_upsert(_ical_svc, _ical_row, **)
  # We use an async job to sync when the dependency syncs
  return
end