Module: Beam::Upload

Defined in:
lib/beam/upload.rb

Instance Method Summary collapse

Instance Method Details

#add_to_batch(batch, record) ⇒ Object 



135
136
137
138
139
140
141
142
 
# File 'lib/beam/upload.rb', line 135

def add_to_batch(batch, record)
  batch << record if record
  if batch.size >= @batch_size
    import_from_batch(batch, true)
    batch = []
  end
  batch
end

#delete_csv_if_present ⇒ Object

deletes csv file upfront before we unzip the file



62
63
64
 
# File 'lib/beam/upload.rb', line 62

def delete_csv_if_present
  File.delete "#{@file_path}/#{@csv_file_name}" if File.exists? "#{@file_path}/#{@csv_file_name}"
end

#error_in_upload(rows) ⇒ Object

Creates error file with error rows, e.g. for users.csv file, it creates errors_users.csv



68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/beam/upload.rb', line 68

def error_in_upload(rows)
  csv_file_path = "#{@file_path}/errors_#{@csv_file_name}"
  begin
    CSV.open(csv_file_path, "wb") do |csv|
      rows.each do |row|
        csv << row
      end
    end
  rescue Exception => e
    Rails.logger.error e.formatted_exception("Building error file for #{@file_name} failed!")
  end
end

#import_from_batch(batch) ⇒ Object 



144
145
146
 
# File 'lib/beam/upload.rb', line 144

def import_from_batch(batch)
  import(batch, :validate => false) if @batch_process
end

#invalid_file_message ⇒ Object 



159
160
161
 
# File 'lib/beam/upload.rb', line 159

def invalid_file_message
  "Please upload the right template and verify data before upload"
end

#log_and_return_error(row_hash, e, index) ⇒ Object 



154
155
156
157
 
# File 'lib/beam/upload.rb', line 154

def log_and_return_error(row_hash, e, index)
  Rails.logger.error e.formatted_exception("Error on #{index}: \n #{row_hash.values}")
  row_hash.values + [invalid_file_message]
end

#log_and_return_validation_error(record, row_hash, index) ⇒ Object 



148
149
150
151
152
 
# File 'lib/beam/upload.rb', line 148

def log_and_return_validation_error(record, row_hash, index)
  error_msg = record.errors.messages.values.join(', ')
  Rails.logger.error("Error on #{index}: \n #{row_hash.values + [error_msg]}")
  row_hash.values + [error_msg]
end

#parse ⇒ Object

parses the csv file, creates record for the model and returns response hash,e.g. :status=>200, :total_rows=>4, :error_rows=>[[“Test1”, nil, “is invalid”]] also, it creates error file to consume



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/beam/upload.rb', line 95

def parse
  response  = { errors: 0, status: 200, total_rows: 0, error_rows: []}
  index = 0
  batch = []

  begin
    CSV.foreach("#{@file_path}/#{@csv_file_name}", :encoding => 'iso-8859-1:UTF-8', headers: true) do |row|
      index += 1
      row_hash = row.to_hash
      response[:total_rows] += 1
      begin
        record, response[:errors], error_row = validate_record(response[:errors], row_hash, index)
        response[:error_rows] << error_row if error_row
          
        if @batch_process
          batch = add_to_batch(batch, record)
        else
          record.save!
        end
      rescue Exception => e
        response[:errors] +=1
        response[:error_rows] << log_and_return_error(row_hash, e, index)
      end
    end
    import_from_batch(batch)
  rescue Exception => e
    response[:errors] = 1
    response[:error_rows] = log_and_return_error({}, e, '')
    response[:status] = 500
  end

  if response[:error_rows].blank?
    post_parse_success_calls 
  elsif @error_file_needed
    error_in_upload(response[:error_rows])
  end

  response
end

#post_parse_success_calls ⇒ Object 



163
164
165
 
# File 'lib/beam/upload.rb', line 163

def post_parse_success_calls
  # abstract method
end

#unzip_file ⇒ Object

unzips a zipped-csv-file in a given directory



57
58
59
 
# File 'lib/beam/upload.rb', line 57

def unzip_file
  `unzip #{@file_path}/#{@file_name} -d #{@file_path}`
end

#upload_config ⇒ Object 



20
21
22
23
24
25
 
# File 'lib/beam/upload.rb', line 20

def upload_config
  @error_file_needed  = Beam.config[:error_file_needed]
  @batch_process      = Beam.config[:batch_process]
  @batch_size         = Beam.config[:batch_size]
  @zipped             = Beam.config[:zipped]
end

#upload_file(file_name, file_path, callback_method = 'parse') ⇒ Object

params:

file_name:        zip file name, e.g. users.csv.zip
file_path:        path/to/zip-file, e.g. Rails.root/tmp
callback_method:  method which does the job to parse CSV and load records, default is parse method



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/beam/upload.rb', line 31

def upload_file(file_name, file_path, callback_method='parse')
  status = {}
  upload_config
  @file_name          = file_name
  @file_path          = file_path      
  @original_zip_file  = "#{@file_path}/#{@file_name}"
  @csv_file_name      = @file_name.gsub(/\.zip/,'')

  begin
    if @zipped
      delete_csv_if_present
      unzip_file
    end
    
    status = self.send(callback_method)
  rescue Exception =>  e
    error_in_upload([[invalid_file_message]]) if @error_file_needed
    Rails.logger.error e.formatted_exception("Uploading file #{@file_name} failed!")
    status = { errors: 1, status: 500}
  ensure
    File.delete @original_zip_file if File.exists?(@original_zip_file)
  end
  status
end

#validate_record(errors_count, row_hash, index) ⇒ Object

Validates each record and returns error count if record is invalid



82
83
84
85
86
87
88
89
90
 
# File 'lib/beam/upload.rb', line 82

def validate_record(errors_count, row_hash, index)
  record = new(row_hash)
  unless record.valid?
    errors_count += 1
    [nil, errors_count, log_and_return_validation_error(record, row_hash, index)]
  else
    [record, errors_count, nil]
  end
end