Class: HeyDan::Script

Inherits:
Object
  • Object
show all
Defined in:
lib/heydan/script.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts) ⇒ Script

Returns a new instance of Script.



13
14
15
16
17
18
19
20
21
 
# File 'lib/heydan/script.rb', line 13

def initialize(opts)
  @folder = opts[:folder]
  @source = opts[:source]
  @variable  = opts[:variable]
  @jurisdiction_type = HeyDan.options[:type]
  @fromsource = HeyDan.options[:fromsource]
  @identifiers = {}
  @source_file = HeyDan::SourceFile.new(@folder, @source, @variable)
end

Instance Attribute Details

#data ⇒ Object

Returns the value of attribute data.



5
6
7
 
# File 'lib/heydan/script.rb', line 5

def data
  @data
end

#folder ⇒ Object

Returns the value of attribute folder.



6
7
8
 
# File 'lib/heydan/script.rb', line 6

def folder
  @folder
end

#fromsource ⇒ Object

Returns the value of attribute fromsource.



4
5
6
 
# File 'lib/heydan/script.rb', line 4

def fromsource
  @fromsource
end

#id ⇒ Object

Returns the value of attribute id.



9
10
11
 
# File 'lib/heydan/script.rb', line 9

def id
  @id
end

#identifiers ⇒ Object

builds a hash with new_identifier => file_name



11
12
13
 
# File 'lib/heydan/script.rb', line 11

def identifiers
  @identifiers
end

#jurisdiction_type ⇒ Object

Returns the value of attribute jurisdiction_type.



3
4
5
 
# File 'lib/heydan/script.rb', line 3

def jurisdiction_type
  @jurisdiction_type
end

#source ⇒ Object

Returns the value of attribute source.



7
8
9
 
# File 'lib/heydan/script.rb', line 7

def source
  @source
end

#source_file ⇒ Object

Returns the value of attribute source_file.



10
11
12
 
# File 'lib/heydan/script.rb', line 10

def source_file
  @source_file
end

#variable ⇒ Object

Returns the value of attribute variable.



8
9
10
 
# File 'lib/heydan/script.rb', line 8

def variable
  @variable
end

Instance Method Details

#add_attributes ⇒ Object 



157
158
159
160
161
162
163
164
165
166
 
# File 'lib/heydan/script.rb', line 157

def add_attributes
  @data[1..-1].each_index do |i| 
      row = @data[i+1]
      jf = get_jurisdiction_filename(row[0])
      next if row[0].nil? || !jf.exists?
      jf.add_attribute(@data[0][1], row[1])
      jf.save
      @progress.progress = i  if HeyDan.help?
    end
end

#add_datasets ⇒ Object 



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/heydan/script.rb', line 116

def add_datasets
  metadata = @source_file.variable
  metadata.keep_if { |k| ['id', 'name', 'short_description', 'tags'].include?(k)}
  metadata["years"] = @data[0][1..-1]
  id = metadata['id']
  @data[1..-1].each_index do |i| 
    row = @data[i+1]
    next if row[0].nil? || @identifiers[row[0]].nil?
    jf = get_jurisdiction_filename(@identifiers[row[0]])
    next if row[0].nil? || !jf.exists?
    ds = jf.get_dataset(id)
    if !ds.nil?
      if ds['version'] >= version
        @progress.progress = i if HeyDan.help?
        next
      end
    end
    jf.datasets.delete(ds)
    metadata["version"] = version
    metadata["data"] = row[1..-1]
    jf.add_dataset(metadata)
    jf.save
    @progress.progress = i if HeyDan.help?
  end
end

#add_identifiers ⇒ Object 



146
147
148
149
150
151
152
153
154
155
 
# File 'lib/heydan/script.rb', line 146

def add_identifiers
  @data[1..-1].each_index do |i| 
    row = @data[i+1]
    jf = get_jurisdiction_filename(row[0])
    next if row[0].nil? || !jf.exists?
    jf.add_identifier(@data[0][1], row[1])
    jf.save
    @progress.progress = i  if HeyDan.help?
  end
end

#build ⇒ Object

overwritten by the developer



24
25
 
# File 'lib/heydan/script.rb', line 24

def build
end

#build_identifier_hash(identifier) ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/heydan/script.rb', line 84

def build_identifier_hash(identifier)
  identifier_file = File.join(HeyDan.folders[:downloads], "identifiers_file_#{identifier}.json")
  if File.exist?(identifier_file)
    @identifiers = JSON.parse(File.read(identifier_file))
    return @identifiers
  end
  HeyDan::HelpText.build_identifier(identifier)
  get_identifiers_from_files(identifier)
  File.open(identifier_file, 'w') do |file|
    file.write(@identifiers.to_json)
  end
  @identifiers
end

#dataset_file_name ⇒ Object 



41
42
43
 
# File 'lib/heydan/script.rb', line 41

def dataset_file_name
  "#{@folder}_#{@source}_#{@variable}.csv"
end

#dataset_file_path ⇒ Object 



49
50
51
 
# File 'lib/heydan/script.rb', line 49

def dataset_file_path
  File.join(HeyDan.folders[:datasets], dataset_file_name)
end

#download ⇒ Object

downloads from the cdn



54
55
56
 
# File 'lib/heydan/script.rb', line 54

def download
  @data = HeyDan::Helper.get_data_from_url(HeyDan.cdn + '/' + dataset_file_name)
end

#get_data ⇒ Object 



173
174
175
176
177
 
# File 'lib/heydan/script.rb', line 173

def get_data
  if @data.nil?
    @data = HeyDan::Helper.get_data(dataset_file_name)
  end
end

#get_identifiers ⇒ Object 



168
169
170
171
 
# File 'lib/heydan/script.rb', line 168

def get_identifiers
  @id = @data[0][0]
  @identifiers = build_identifier_hash(@id)
end

#get_identifiers_from_files(identifier) ⇒ Object 



98
99
100
101
102
103
104
105
106
 
# File 'lib/heydan/script.rb', line 98

def get_identifiers_from_files(identifier)
  @identifiers = {} 
  Dir.glob(File.join(HeyDan.folders[:jurisdictions], '*.json')).each do |j|
    jf = HeyDan::JurisdictionFile.new(name: j.gsub(HeyDan.folders[:jurisdictions] + '/', ''))
    return if !jf.match_type?(@jurisdiction_type)
    @identifiers["#{jf.get_identifier(identifier)}"] = j.gsub(HeyDan.folders[:jurisdictions] + '/', '')
  end
  @identifiers
end

#get_jurisdiction_filename(id) ⇒ Object 



142
143
144
 
# File 'lib/heydan/script.rb', line 142

def get_jurisdiction_filename(id)
  HeyDan::JurisdictionFile.new(name: @identifiers[id] || id)
end

#name ⇒ Object 



45
46
47
 
# File 'lib/heydan/script.rb', line 45

def name
  "#{@folder}_#{@source}_#{@variable}.csv"
end

#process ⇒ Object

runs through download, build and validate



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/heydan/script.rb', line 65

def process
  if @fromsource
    puts "From Source is specified, processing from source for #{name}" if HeyDan.help?
    process_from_source 
  end

  begin
    if HeyDan::Helper.dataset_exists?(dataset_file_name)
      puts "Dataset for #{name} exists" if HeyDan.help?
    else
      download 
    end
  rescue 
      puts "Had trouble downloading #{name}, processing from source instead" if HeyDan.help?
      process_from_source        
  end      
  update_jurisdiction_files
end

#process_from_source ⇒ Object 



58
59
60
61
62
 
# File 'lib/heydan/script.rb', line 58

def process_from_source
  build
  validate_build
  HeyDan::Helper.save_data(dataset_file_name, @data)
end

#type ⇒ Object

overwritten by the developer, can be dataset, attribute, or identifer



32
33
34
 
# File 'lib/heydan/script.rb', line 32

def type
  'dataset'
end

#update_jurisdiction_files ⇒ Object 



108
109
110
111
112
113
114
 
# File 'lib/heydan/script.rb', line 108

def update_jurisdiction_files
  get_data
  get_identifiers
  @progress = ProgressBar.create(:title => "Updating Files for #{@source} #{@variable} from #{@folder} for jurisdictions #{(' matching ' + @jurisdiction_type) if @jurisdiction_type}", :starting_at => 0, :total => @data[1..-1].size) if HeyDan.help?
  self.send("add_#{type}s")
  @progress.finish if HeyDan.help?
end

#validate_build ⇒ Object

overwritten by the developer



28
29
 
# File 'lib/heydan/script.rb', line 28

def validate_build
end

#version ⇒ Object

overwritten by developer



37
38
39
 
# File 'lib/heydan/script.rb', line 37

def version
  1
end