Class: DataValidation::DataAccess

Inherits:
Object
  • Object
show all
Defined in:
lib/data_validation/data_access.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(html_url, api_url, api_body) ⇒ DataAccess

Returns a new instance of DataAccess.



10
11
12
13
14
 
# File 'lib/data_validation/data_access.rb', line 10

def initialize(html_url, api_url, api_body)
  @html_url = html_url
  @api_url = api_url
  @api_body = api_body
end

Instance Attribute Details

#api_body ⇒ Object (readonly)

Returns the value of attribute api_body.



8
9
10
 
# File 'lib/data_validation/data_access.rb', line 8

def api_body
  @api_body
end

#api_url ⇒ Object (readonly)

Returns the value of attribute api_url.



8
9
10
 
# File 'lib/data_validation/data_access.rb', line 8

def api_url
  @api_url
end

#conf_standing_data ⇒ Object (readonly)

Returns the value of attribute conf_standing_data.



8
9
10
 
# File 'lib/data_validation/data_access.rb', line 8

def conf_standing_data
  @conf_standing_data
end

#html_url ⇒ Object (readonly)

Returns the value of attribute html_url.



8
9
10
 
# File 'lib/data_validation/data_access.rb', line 8

def html_url
  @html_url
end

Class Method Details

.get_high_school_div_stat_urls(season = '17', category = 'boys') ⇒ Object 



164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/data_validation/data_access.rb', line 164

def self.get_high_school_div_stat_urls(season='17', category='boys')
  is_for_hs = category == 'boys' || category == 'girls'
  high_school_div_stat_main_url = if is_for_hs
                                    "#{DataValidation.web_request_host}/common/hs_#{category}.php"
                                  else
                                    "#{DataValidation.web_request_host}/common/college_#{category}.php"
                                  end
  doc = Nokogiri::HTML(open(high_school_div_stat_main_url))
  urls = doc.search('div#content_well a[href*=rating]').map do |item|
    href = item.attr('href')
    if href.match /rating\d\d/
      href.sub!('..', DataValidation.web_request_host)
      href.sub!('update17', "update#{season}") if season!= '17'
      href
    end
  end.compact

  final_urls = [] + urls
  ## append other metrics e.g. sos, rpi
  if is_for_hs
    %w|sos rpi wl qwf|.each { |stat_name| final_urls += replace_with_stat_name(urls, stat_name) }
  else
    %w|tsi poll rpi sos qwf trend|.each { |stat_name| final_urls += replace_with_stat_name(urls, stat_name) }
  end
  puts "totally #{final_urls.length} urls for #{season}, #{category}"
  final_urls
end

.get_pr_for_pro(season, category, div_id) ⇒ Object 



157
158
159
160
161
162
 
# File 'lib/data_validation/data_access.rb', line 157

def self.get_pr_for_pro(season, category, div_id)
  puts "get PR ranking for #{season}, #{category}, #{div_id} from " + DataValidation.api_request_host + '/rest/LaxPower/rankingTableProPR'
  HTTParty.post(DataValidation.api_request_host + '/rest/LaxPower/rankingTableProPR',
                headers: {'Content-Type' => 'application/json'},
                body: %Q|{"season": "#{season}","category": "#{category}", "divisionId": #{div_id}, "pageSize": 1000, "currPage": 1}|)['results']['results']['teams']
end

.get_pro_pr_table(html_url) ⇒ Object

for pro only



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/data_validation/data_access.rb', line 139

def self.get_pro_pr_table html_url
  table = []
  doc = Nokogiri::HTML(open(html_url))
  doc.search('div#content_well td:nth-of-type(3) > div:nth-of-type(2) tr:not(:first-of-type)').map do |team|
    team_data = []
    team.children.each_with_index do |item, index|
      value = item.text
      if index == 3 # number of win and loss
        value.split('-').each { |n| team_data << n }
      else
        team_data << value
      end
    end
    table << team_data
  end
  table
end

.replace_with_stat_name(urls, stat_name) ⇒ Object 



192
193
194
195
196
 
# File 'lib/data_validation/data_access.rb', line 192

def self.replace_with_stat_name urls, stat_name
  urls.map do |url|
    url.sub 'rating', stat_name
  end
end

Instance Method Details

#extract_conf_standing_data(doc) ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/data_validation/data_access.rb', line 68

def extract_conf_standing_data(doc)
  category = if html_url.match /bingrl/
               'GIRLS'
             elsif html_url.match /binboy/
               'BOYS'
             elsif html_url.match /binmen/
               'MEN'
             elsif html_url.match /binwom/
               'WOMEN'
             end

  season = "20#{html_url.match(/update(..)/)[1]}"

  div_id = html_url.match(/(\d+)\.php/)[1].to_i

  conf_data = get_conference(season, category, div_id)['results']['results']['conferences']

  # is_for_mw_pr = html_url.include?('binmen') || html_url.include?('binwomen')

  ## only rating page has conf standing tables
  doc.search('div.laxcss a').remove ## remove team names from conf standing tables
  @conf_standing_data = []
  conf_standing_record = []
  lines = doc.search('div.laxcs1, div.laxcss').text.lines

  lines.each_with_index do |line, index|
    if /^\s(?<conf_full_name>[A-Z].*)Conference/ =~ line || /^\s(?<conf_full_name>[A-Z].*)Overall/ =~ line
      if conf_standing_record.length > 0 ## it's a new conf standing table
        @conf_standing_data << conf_standing_record
        conf_standing_record = []
      end

      conf = get_conf(conf_full_name, conf_data)
      if conf.nil? ## last standing table has no record, the response of "getConferences" api will not include this conference
        puts 'No independent conf data'
        @conf_standing_data << conf_standing_record
        break
      end

      conf_standing_record << [conf['state'], conf['teamClass'], conf['conference']]
    elsif /^\s*\d+\s+/.match line
      conf_standing_record << line.split(' ')
    end

    if (index == lines.length - 1)
      @conf_standing_data << conf_standing_record
    end
  end
end

#get_conf(conf_full_name, conf_data) ⇒ Object 



118
119
120
121
 
# File 'lib/data_validation/data_access.rb', line 118

def get_conf(conf_full_name, conf_data)
  selected = conf_data.select { |conf| conf_full_name.include?("#{conf['state']}") && conf_full_name.include?("#{conf['teamClass']}") && conf_full_name.include?("#{conf['conference']}")}
  selected.first if selected && selected.size > 0
end

#get_conference(season, category, div_id) ⇒ Object 



131
132
133
134
135
136
 
# File 'lib/data_validation/data_access.rb', line 131

def get_conference(season, category, div_id)
  puts "get conference data for #{season}, #{category}, #{div_id} from " + DataValidation.api_request_host + '/rest/LaxPower/getConferences'
  HTTParty.post(DataValidation.api_request_host + '/rest/LaxPower/getConferences',
                headers: {'Content-Type' => 'application/json'},
                body: %Q|{"season": "#{season}","category": "#{category}", "divisionId": #{div_id}}|)
end

#get_main_table_data ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/data_validation/data_access.rb', line 16

def get_main_table_data
  doc = Nokogiri::HTML(open(html_url))

  if html_url.match /rating\d+/
    extract_conf_standing_data(doc)
  end

  team_names = doc.search('div.cs1 div.cs1 a').map { |node| node.text if /^[A-Z\d]+.PHP$/.match(node.attr('href')) }.compact ## extract team names

  doc.search('div.cs1 div.cs1 a').remove ## then remove all links from the doc

  index = 0
  doc.search('div.cs1 div.cs1').text.lines.map do |line|
    if /^\s*(?<rank>\d+)\s+(?<region>[a-zA-Z][a-zA-Z\/\d-]*\s([a-zA-Z\/\d-]+\s)*)?(?<rest>.*)/ =~ line
      data_array = ([rank] << team_names[index])
      data_array << region.strip unless region.nil? || region.strip.empty?
      rest_parts = rest.split(' ')
      rest_parts.each do |item|
        if item.match /^\d+-/ # win-loss-tie value
          data_array += item.split('-')
        else
          data_array << item
        end
      end
      index += 1
      data_array
    end
  end.compact
end

#get_regional_rating_table_data ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/data_validation/data_access.rb', line 46

def get_regional_rating_table_data
  doc = Nokogiri::HTML(open(html_url))
  index = 0

  doc.search('div.cs1').text.lines.map do |line|
    if /^\s*(?<rank>\d+)\s+(?<name>[a-zA-Z][a-zA-Z'\/\d-]*\s([a-zA-Z'\/\d-]+\s)*)?(?<rest>.*)/ =~ line
      data_array = [rank]
      data_array << name.strip unless name.nil? || name.strip.empty?
      rest_parts = rest.split(' ')
      rest_parts.each do |item|
        if item.match /^\d+-/ # win-loss-tie value
          data_array += item.split('-')
        else
          data_array << item
        end
      end
      index += 1
      data_array
    end
  end.compact
end

#get_response_from_mobile_api ⇒ Object 



123
124
125
 
# File 'lib/data_validation/data_access.rb', line 123

def get_response_from_mobile_api
  HTTParty.post(api_url.strip, headers: {'Content-Type' => 'application/json'}, body: api_body)
end

#get_response_from_mobile_api_with_params(url, body) ⇒ Object 



127
128
129
 
# File 'lib/data_validation/data_access.rb', line 127

def get_response_from_mobile_api_with_params(url, body)
  HTTParty.post(url, headers: {'Content-Type' => 'application/json'}, body: body)
end