Class: Fech::Table

Inherits:
Object
  • Object
show all
Defined in:
lib/fech-ftp/table.rb,
lib/fech-ftp/table_methods.rb

Instance Method Summary collapse

Constructor Details

#initialize(cycle, opts = {}) ⇒ Table

Returns a new instance of Table.



3
4
5
6
7
8
9
10
11
 
# File 'lib/fech-ftp/table.rb', line 3

def initialize(cycle, opts={})
  @cycle    = cycle
  @headers  = opts[:headers]
  @file     = opts[:file]
  @format   = opts[:format]
  @location = opts[:location]
  @receiver = opts[:connection] || receiver
  @parser   = parser
end

Instance Method Details

#create_table(row) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/fech-ftp/table.rb', line 44

def create_table(row)
  db, table = @receiver
  table = table.to_s.pluralize.to_sym
  db.create_table(table) { primary_key :id }

  row.each do |k,v|
    v = v.nil? ? String : v.class
    db.alter_table table do
      add_column k, v
    end
  end

  @receiver = db[table]
  @receiver << row
end

#enter_row(row) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
 
# File 'lib/fech-ftp/table.rb', line 26

def enter_row(row)
  case @format
  when :db
    table_exist? ? @receiver << row : create_table(row)
  when :csv
    @receiver << row.values
  else
    @receiver << row
  end
end

#fetch_file(&blk) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/fech-ftp/table.rb', line 60

def fetch_file(&blk)
  zip_file = "#{@file}#{@cycle.to_s[2..3]}.zip"
  Net::FTP.open("ftp.fec.gov") do |ftp|
    ftp.
    ftp.chdir("./FEC/#{@cycle}")
    begin
      ftp.get(zip_file, "./#{zip_file}")
    rescue Net::FTPPermError
      raise 'File not found - please try the other methods'
    end
  end

  unzip(zip_file, &blk)
end

#format_row(line) ⇒ Object 



89
90
91
92
93
94
95
96
 
# File 'lib/fech-ftp/table.rb', line 89

def format_row(line)
  hash = {}
  line = line.encode('UTF-8', invalid: :replace, replace: ' ').chomp.split("|")

  @parser.each { |k,blk| hash[k] = blk.call(line) }

  return hash
end

#parse_date(date) ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/fech-ftp/table.rb', line 98

def parse_date(date)
  if date == '' && table_exist?
    if table_exist?
      return Date.new(@cycle, 1,1)
    else
      return ''
    end
  end

  if date.length == 8
    Date.strptime(date, "%m%d%Y")
  else
    Date.parse(date)
  end
end

#parser ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/fech-ftp/table.rb', line 75

def parser
  @headers.map.with_index do |h,i|
    if h.to_s =~ /cash|amount|contributions|total|loan|transfer|debts|refund|expenditure/
      [h, ->(line) { line[i].to_f }]
    elsif h == :filing_id
      [h, ->(line) { line[i].to_i }]
    elsif h.to_s =~ /_date/
      [h, ->(line) { parse_date(line[i]) }]
    else
      [h, ->(line) { line[i] }]
    end
  end
end

#receiver ⇒ Object 



13
14
15
16
17
18
19
 
# File 'lib/fech-ftp/table.rb', line 13

def receiver
  if @format == :csv
    CSV.open("#{@file}#{@cycle.to_s[2..3]}.csv", 'a+', headers: @headers, write_headers: true)
  else
    []
  end
end

#retrieve_data ⇒ Object 



21
22
23
24
 
# File 'lib/fech-ftp/table.rb', line 21

def retrieve_data
  fetch_file { |row| enter_row(row) }
  return @receiver
end

#table_exist? ⇒ Boolean

the @receiver obj is the database itself. This assumes the table needs to be created.

Returns:

  • (Boolean) 


40
41
42
 
# File 'lib/fech-ftp/table.rb', line 40

def table_exist?
  @receiver.respond_to? :columns
end

#unzip(zip_file, &blk) ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/fech-ftp/table.rb', line 114

def unzip(zip_file, &blk)
  Zip::File.open(zip_file) do |zip|
    zip.each do |entry|
      path = @location.nil? ? entry.name : @location + entry.name
      entry.extract(path) if !File.file?(path)
      File.delete(zip_file)
      File.foreach(path) do |row|
        blk.call(format_row(row))
      end
      File.delete(path)
    end
  end
end