Class: AffiliateWindow::ETL::Extracter

Inherits:
Object
  • Object
show all
Defined in:
lib/affiliate_window/etl/extracter.rb

Overview

rubocop:disable Metrics/ClassLength

Constant Summary collapse

CHUNK_SIZE = 
100

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(client:, logger: nil) ⇒ Extracter

Returns a new instance of Extracter.



8
9
10
11
 
# File 'lib/affiliate_window/etl/extracter.rb', line 8

def initialize(client:, logger: nil)
  self.client = client
  self.logger = logger
end

Instance Attribute Details

#client ⇒ Object

Returns the value of attribute client.



6
7
8
 
# File 'lib/affiliate_window/etl/extracter.rb', line 6

def client
  @client
end

#logger ⇒ Object

Returns the value of attribute logger.



6
7
8
 
# File 'lib/affiliate_window/etl/extracter.rb', line 6

def logger
  @logger
end

Instance Method Details

#catch_invalid_relationship_error(&block) ⇒ Object

If the current account is not affiliated with the merchant, the API does not let you retrieve commission groups for that merchant.



176
177
178
179
180
181
 
# File 'lib/affiliate_window/etl/extracter.rb', line 176

def catch_invalid_relationship_error(&block)
  block.call
rescue AffiliateWindow::Error => e
  raise unless e.message.match(/Invalid merchant \/ affiliate relationship/)
  nil
end

#check_all_records_received!(pagination) ⇒ Object 



167
168
169
170
171
172
 
# File 'lib/affiliate_window/etl/extracter.rb', line 167

def check_all_records_received!(pagination)
  retrieved = pagination.fetch(:i_rows_returned)
  total = pagination.fetch(:i_rows_available)

  fail "Did not receive all records: #{retrieved} retrieved out of #{total}" unless total == retrieved
end

#extract(type, params = {}) ⇒ Object 



13
14
15
16
17
 
# File 'lib/affiliate_window/etl/extracter.rb', line 13

def extract(type, params = {})
  Enumerator.new do |yielder|
    public_send("extract_#{type}", yielder, params)
  end
end

#extract_commission_groups(yielder, merchant_ids:) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/affiliate_window/etl/extracter.rb', line 40

def extract_commission_groups(yielder, merchant_ids:)
  merchant_ids.each.with_index do |id, index|
    maybe_response = catch_invalid_relationship_error do
      client.get_commission_group_list(merchant_id: id)
    end

    next unless maybe_response
    response = maybe_response

    commission_groups = [response.fetch(:commission_group)].flatten

    commission_groups.each do |record|
      yielder.yield(record.merge(
        record_type: :commission_group,
        merchant_id: id,
      ))
    end

    write "Extracted commission groups for #{index + 1} / #{merchant_ids.count} merchants"
  end
end

#extract_daily_clicks(yielder, date:) ⇒ Object 



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'lib/affiliate_window/etl/extracter.rb', line 123

def extract_daily_clicks(yielder, date:)
  response = client.get_click_stats(
    start_date: "#{date}T00:00:00",
    end_date: "#{date}T23:59:59",
    date_type: "transaction",
  )
  results = response.fetch(:results)
  pagination = response.fetch(:pagination)

  check_all_records_received!(pagination)

  click_stats = results.fetch(:click_stats)
  click_stats.each do |record|
    yielder.yield(record.merge(
      record_type: :click_stat,
      date: date,
    ))
  end

  write "Extracted #{click_stats.count} click stats for #{date}"
end

#extract_daily_impressions(yielder, date:) ⇒ Object 



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/affiliate_window/etl/extracter.rb', line 145

def extract_daily_impressions(yielder, date:)
  response = client.get_impression_stats(
    start_date: "#{date}T00:00:00",
    end_date: "#{date}T23:59:59",
    date_type: "transaction",
  )
  results = response.fetch(:results)
  pagination = response.fetch(:pagination)

  check_all_records_received!(pagination)

  impression_stats = results.fetch(:impression_stats)
  impression_stats.each do |record|
    yielder.yield(record.merge(
      record_type: :impression_stat,
      date: date,
    ))
  end

  write "Extracted #{impression_stats.count} impression stats for #{date}"
end

#extract_daily_transactions(yielder, date:) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/affiliate_window/etl/extracter.rb', line 62

def extract_daily_transactions(yielder, date:)
  response = client.get_transaction_list(
    start_date: "#{date}T00:00:00",
    end_date: "#{date}T23:59:59",
    date_type: "transaction",
  )
  results = response.fetch(:results)
  pagination = response.fetch(:pagination)

  check_all_records_received!(pagination)

  transactions = results.fetch(:transaction)
  transactions.each do |record|
    yielder.yield(record.merge(record_type: :transaction))
  end

  write "Extracted #{transactions.count} transactions for #{date}"

  transaction_ids = transactions.map { |t| t.fetch(:i_id) }
  extract_transaction_products(yielder, transaction_ids: transaction_ids)
end

#extract_merchants(yielder, _params) ⇒ Object

rubocop:disable Metrics/AbcSize



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/affiliate_window/etl/extracter.rb', line 19

def extract_merchants(yielder, _params) # rubocop:disable Metrics/AbcSize
  response = client.get_merchant_list
  merchants = response.fetch(:merchant)
  merchant_ids = merchants.map { |m| m.fetch(:i_id) }

  count = 0
  merchant_ids.each_slice(CHUNK_SIZE) do |ids|
    response = client.get_merchant(merchant_ids: ids)
    merchants = response.fetch(:merchant)

    merchants.each do |record|
      yielder.yield(record.merge(record_type: :merchant))
    end

    count += [CHUNK_SIZE, ids.count].min
    write "Extracted #{count} / #{merchant_ids.count} merchants"
  end

  extract_commission_groups(yielder, merchant_ids: merchant_ids)
end

#extract_transaction_products(yielder, transaction_ids:) ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/affiliate_window/etl/extracter.rb', line 101

def extract_transaction_products(yielder, transaction_ids:)
  count = 0
  transaction_ids.each_slice(CHUNK_SIZE) do |ids|
    response = client.get_transaction_product(transaction_ids: ids)

    # Occasionally, Affiliate Window fails to record purchases so they
    # artificially create some by extrapolating historical data. The
    # transactions they create don't have any associated product data so
    # skip over empty transaction_product responses.
    next unless response

    transaction_products = [response.fetch(:transaction_product)].flatten

    transaction_products.each do |record|
      yielder.yield(record.merge(record_type: :transaction_product))
    end

    count += [CHUNK_SIZE, ids.count].min
    write "Extracted #{transaction_products.count} products for #{count} / #{transaction_ids.count} transactions"
  end
end

#extract_transactions(yielder, transaction_ids:) ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/affiliate_window/etl/extracter.rb', line 84

def extract_transactions(yielder, transaction_ids:)
  count = 0
  transaction_ids.each_slice(CHUNK_SIZE) do |ids|
    response = client.get_transaction(transaction_ids: ids)

    transactions = response.fetch(:transaction)
    transactions.each do |record|
      yielder.yield(record.merge(record_type: :transaction))
    end

    count += [CHUNK_SIZE, ids.count].min
    write "Extracted #{count} / #{transaction_ids.count} transactions"
  end

  extract_transaction_products(yielder, transaction_ids: transaction_ids)
end

#write(message) ⇒ Object 



183
184
185
186
 
# File 'lib/affiliate_window/etl/extracter.rb', line 183

def write(message)
  return unless logger
  logger.debug { "[quota:#{client.remaining_quota}] #{message}" }
end