Class: Remi::DataSource::Salesforce

Inherits:
Remi::DataSource show all
Includes:
Remi::DataSubject::Salesforce
Defined in:
lib/remi/data_subject/salesforce.rb

Instance Attribute Summary

Attributes inherited from Remi::DataSubject

#fields

Instance Method Summary collapse

Methods included from Remi::DataSubject::Salesforce

#field_symbolizer, #restforce_client

Methods inherited from Remi::DataSource

#df, #extract

Methods included from Testing::DataStub

#empty_stub_df, #stub_boolean, #stub_date, #stub_datetime, #stub_decimal, #stub_df, #stub_float, #stub_integer, #stub_json, #stub_row_array, #stub_string, #stub_values

Methods inherited from Remi::DataSubject

#df, #df=, #enforce_types, #field_symbolizer

Constructor Details

#initialize(*args, **kargs, &block) ⇒ Salesforce

Returns a new instance of Salesforce.



27
28
29
30
 
# File 'lib/remi/data_subject/salesforce.rb', line 27

def initialize(*args, **kargs, &block)
  super
  init_salesforce(*args, **kargs, &block)
end

Instance Method Details

#extract! ⇒ Object

Public: Called to extract data from the source.

Returns data in a format that can be used to create a dataframe.



35
36
37
38
39
40
 
# File 'lib/remi/data_subject/salesforce.rb', line 35

def extract!
  @extract = sf_bulk.query(@sfo, @query, 10000)

  check_for_errors(@extract)
  @extract
end

#sf_bulk ⇒ Object 



42
43
44
 
# File 'lib/remi/data_subject/salesforce.rb', line 42

def sf_bulk
  @sf_bulk ||= SalesforceBulkApi::Api.new(restforce_client).tap { |o| o.connection.set_status_throttle(5) }
end

#to_dataframe ⇒ Object

Public: Converts extracted data to a dataframe. Currently only supports Daru DataFrames.

Returns a Remi::DataFrame



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/remi/data_subject/salesforce.rb', line 50

def to_dataframe
  @logger.info "Converting salesforce query results to a dataframe"

  hash_array = {}
  extract['batches'].each do |batch|
    next unless batch['response']

    batch['response'].each do |record|
      record.each do |field, value|
        next if ['xsi:type','type'].include? field
        (hash_array[field.to_sym] ||= []) << case value.first
          when Hash
            value.first["xsi:nil"] == "true" ? nil : value.first
          else
            value.first
          end
      end
    end

    # delete raw result at end of processing to free memory
    batch['response'] = nil
  end

  Remi::DataFrame.create(@remi_df_type, hash_array, order: hash_array.keys)
end