Class: HltvScrapping::Call

Inherits:
Object
  • Object
show all
Defined in:
lib/hltv_scrapping.rb

Class Method Summary collapse

Class Method Details

.generate_models ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/hltv_scrapping.rb', line 5

def self.generate_models
  puts "Génération des team"
  # team generator
  system("rails g model team name maps:integer kd_diff:integer kd:float rating:float link")

  puts "Génération des matchs"
  # matches generator
  system("rails g model match event date:datetime map key link")

  puts "Génération des participations aux matchs"
  # matches generator
  system("rails g model participation team:references match:references score:integer win:boolean")

  puts "Génération des map stats"
  # team map stats generator
  system("rails g model map_stat map team:references win draw losses win_rate:integer total_rounds:integer round_win_after_first_kill_percent:integer round_win_after_first_death_percent:integer")

  puts "Génération de player"
  # players generator
  system("rails g model player link team:references maps_played:integer name actif:boolean")

  puts "Génération de player global stat"
  #player globals stats generator
  system("rails g model player_global_stat total_kills:integer headshot_percent:float total_deaths:integer kd:float damage_per_round:float grenade_dmg_per_round:float maps_played:integer rounds_played:integer kills_per_round:float assist_per_round:float death_per_round:float saved_by_teammates_per_round:float saved_teammates_per_round:float player:references")

  puts "Génération de participation de joueurs"
  #Play participation match generator
  system("rails g model player_participation_match player:references kills:integer head_shots:integer assists:integer deaths:integer kast_percent:float kd_diff:integer average_damage_per_rounds:float fk_diff:integer participation:references")

  #migrate migrations
  system("rails db:migrate")
end

.get_all_matches ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/hltv_scrapping.rb', line 57

def self.get_all_matches
  teams = Team.all
  teams.each do |team|
    link = "https://www.hltv.org#{team.link}"
    html_file = open(link)
    html_doc = Nokogiri::HTML(html_file)

    html_doc.search('table.stats-table > tbody > tr').each do |element|
      match_date = Date.parse(element.search('td.time > a').first.text)
      link = element.search('td.time > a').map { |link| link['href'] }.first
      event = element.search('td.gtSmartphone-only > a').first.text
      map = element.search('td.statsMapPlayed > span').first.text.downcase
      score = element.search('td.gtSmartphone-only > span').last.text

      team2 = Team.find_by_name(element.search('td:nth-child(4) > a').last.text)
      score1 = score.split("-").first.to_i
      score2 = score.split("-").last.to_i
      if team2.present?
        team_uniq_id = team.id > team2.id ? "#{team2.id}#{team.id}" : "#{team.id}#{team2.id}"
        uniq_key = "#{team_uniq_id}#{match_date}#{map}"
        if Match.where(key: uniq_key).empty?
        #match part
          team_match = Match.find_or_create_by(event: event, date: match_date, map: map, key: uniq_key, link: link)
          #participation part
          p1 = Participation.find_or_create_by(team: team, match: team_match, score: score1, win: score1 > score2 )
          p2 = Participation.find_or_create_by(team: team2, match: team_match, score: score2, win: score2 > score1 )

          puts "Match saved"
        else
          puts "#{element.search('td:nth-child(4) > a').last.text} doesn't exist in db"
        end
      end
    end
  end
end

.get_all_teams ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/hltv_scrapping.rb', line 39

def self.get_all_teams
  link = "https://www.hltv.org/stats/teams"
  html_file = open(link)
  html_doc = Nokogiri::HTML(html_file)

  html_doc.search('table.stats-table > tbody > tr').each do |element|
    team_link = element.search('td > a').map { |link| link['href'] }.first
    match_link = team_link.split("/").insert(3, "matches").join("/")
    team_name = element.search('td.teamCol > a').map { |link| link.text }.first
    team_maps_played = element.search('td.statsDetail').map { |link| link.text }.first
    kd_diff = element.search('td.kdDiffCol').map { |link| link.text }.first.to_i
    kd = element.search('td.statsDetail').map { |link| link.text }.last.to_f
    rating = element.search('td.ratingCol').map { |link| link.text }.first.to_f
    new_team = Team.find_or_create_by(name: team_name, link: match_link, maps: team_maps_played, kd_diff: kd_diff, kd: kd, rating: rating)

  end
end

.get_player_participation ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/hltv_scrapping.rb', line 199

def self.get_player_participation
  Match.all.each do |match|
    match_link = "https://www.hltv.org#{match.link}"
    puts match_link
    html_file = open(match_link)
    # html_file = open(match_link)
    html_doc = Nokogiri::HTML(html_file)

    html_doc.search('div.colCon > div.contentCol > div > table:nth-of-type(1) > tbody > tr').each do |element|
      player1 = Player.select{|player| player.link == element.search('td:nth-child(1) > a').map { |link| link['href'] }.first && (match.participations.map(&:team_id).include? player.team_id) }.first
      kills1 = element.search('td:nth-child(2)').text.split("(").first.to_i
      head_shots1 = element.search('td:nth-child(2)').text.split("(").last.gsub(")","").to_i
      assists1 = element.search('td:nth-child(3)').text.to_i
      deaths1 = element.search('td:nth-child(4)').text.to_i
      kast1 = element.search('td:nth-child(5)').text.to_f
      kd_diff1 = element.search('td:nth-child(6)').text.to_f
      average_damage_per_round1 = element.search('td:nth-child(7)').text.to_f
      fk_diff1 = element.search('td:nth-child(8)').text.to_f
      if player1
        participation1 = match.participations.where(team_id: player1.team.id).first
        # byebug
        pp1 = PlayerParticipationMatch.find_or_create_by(
          player: player1,
          kills: kills1,
          head_shots: head_shots1,
          assists: assists1,
          deaths: deaths1,
          kast_percent: kast1,
          kd_diff: kd_diff1,
          average_damage_per_rounds: average_damage_per_round1,
          fk_diff: fk_diff1,
          participation: participation1
        )
        puts "--------------"
        puts match.id
        puts pp1.errors.inspect
        puts "--------------"
      end
    end

    html_doc.search('div.colCon > div.contentCol > div > table:nth-of-type(2)  > tbody > tr').each do |element|
      player = Player.all.select{|player| player.link == element.search('td:nth-child(1) > a').map { |link| link['href']  }.first && (match.participations.map(&:team_id).include? player.team_id) }.first
      kills = element.search('td:nth-child(2)').text.split("(").first.to_i
      head_shots = element.search('td:nth-child(2)').text.split("(").last.gsub(")","").to_i
      assists = element.search('td:nth-child(3)').text.to_i
      deaths = element.search('td:nth-child(4)').text.to_i
      kast = element.search('td:nth-child(5)').text.to_f
      kd_diff = element.search('td:nth-child(6)').text.to_f
      average_damage_per_round = element.search('td:nth-child(7)').text.to_f
      fk_diff = element.search('td:nth-child(8)').text.to_f
      if player
        participation = match.participations.where(team_id: player.team.id).first
        pp2 = PlayerParticipationMatch.find_or_create_by(
          player: player,
          kills: kills,
          head_shots: head_shots,
          assists: assists,
          deaths: deaths,
          kast_percent: kast,
          kd_diff: kd_diff,
          average_damage_per_rounds: average_damage_per_round,
          fk_diff: fk_diff,
          participation: participation
        )
        puts pp2.errors.inspect
      end
    end
  end
end

.get_players ⇒ Object 



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/hltv_scrapping.rb', line 121

def self.get_players
  Team.all.each do |team|
    new_link = team.link.split("/")
    new_link.reject!{ |el| el == "matches"}
    players_link = "https://www.hltv.org#{new_link.join("/")}"
    puts players_link
    html_file = open(players_link)
    html_doc = Nokogiri::HTML(html_file)

    html_doc.search('div:nth-child(9) > div > .teammate-info').each do |element|
      player_name = element.search("a > div.text-ellipsis").text
      player_maps = element.search("span").text.split(" ").first.to_i
      player_link = element.search("a").map { |link| link['href'] }.first
      Player.find_or_create_by(name: player_name, maps_played: player_maps.to_i, link: player_link, team: team, actif: true)
    end

    html_doc.search('div:nth-child(12) > div > .teammate-info').each do |element|
      player_name = element.search("a > div.text-ellipsis").text
      player_maps = element.search("span").text.split(" ").first.to_i
      player_link = element.search("a").map { |link| link['href'] }.first
      Player.find_or_create_by(name: player_name, maps_played: player_maps.to_i, link: player_link, team: team, actif: false)
    end

    html_doc.search('div:nth-child(15) > div > .teammate-info').each do |element|
      player_name = element.search("a > div.text-ellipsis").text
      player_maps = element.search("span").text.split(" ").first.to_i
      player_link = element.search("a").map { |link| link['href'] }.first
      Player.find_or_create_by(name: player_name, maps_played: player_maps.to_i, link: player_link, team: team, actif: false)
    end

  end
end

.get_players_global_stats ⇒ Object 



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/hltv_scrapping.rb', line 154

def self.get_players_global_stats
  Player.all.each do |player|
    player_link = "https://www.hltv.org#{player.link}"
    html_file = open(player_link)
    html_doc = Nokogiri::HTML(html_file)

    html_doc.search('div.contentCol > div > div.statistics > div > div:nth-child(1)').each do |element|
      @total_kills = element.search('div:nth-child(1) > span:nth-child(2)').text.to_i
      @headshot_percent = element.search('div:nth-child(2) > span:nth-child(2)').text.to_f
      @total_deaths = element.search('div:nth-child(3) > span:nth-child(2)').text.to_i
      @kd = element.search('div:nth-child(4) > span:nth-child(2)').text.to_f
      @damage_per_round = element.search('div:nth-child(5) > span:nth-child(2)').text.to_f
      @grenade_dmg_per_round = element.search('div:nth-child(6) > span:nth-child(2)').text.to_f
      @maps_played = element.search('div:nth-child(7) > span:nth-child(2)').text.to_i
    end

    html_doc.search('div.contentCol > div > div.statistics > div > div:nth-child(2)').each do |element|
      @rounds_played = element.search('div:nth-child(1) > span:nth-child(2)').text.to_i
      @kills_per_round = element.search('div:nth-child(2) > span:nth-child(2)').text.to_f
      @assist_per_round = element.search('div:nth-child(3) > span:nth-child(2)').text.to_f
      @death_per_round = element.search('div:nth-child(4) > span:nth-child(2)').text.to_f
      @saved_by_teammates_per_round = element.search('div:nth-child(5) > span:nth-child(2)').text.to_f
      @saved_teammates_per_round = element.search('div:nth-child(6) > span:nth-child(2)').text.to_f
    end

    hsh_player_stats = {
      total_kills: @total_kills,
      headshot_percent: @headshot_percent,
      total_deaths: @total_deaths,
      kd: @kd,
      damage_per_round: @damage_per_round,
      grenade_dmg_per_round: @grenade_dmg_per_round,
      maps_played: @maps_played,
      rounds_played: @rounds_played,
      kills_per_round: @kills_per_round,
      assist_per_round: @assist_per_round,
      death_per_round: @death_per_round,
      saved_by_teammates_per_round: @saved_by_teammates_per_round,
      saved_teammates_per_round: @saved_teammates_per_round,
      player: player
    }
    PlayerGlobalStat.find_or_create_by(hsh_player_stats)
  end
end

.update_stats ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/hltv_scrapping.rb', line 93

def self.update_stats
  Team.all.each do |team|
    maps_link = "https://www.hltv.org#{team.link.gsub("matches", "maps")}"
    html_file = open(maps_link)
    html_doc = Nokogiri::HTML(html_file)

    html_doc.search('div.two-grid > div.col').each do |element|
      map_name = element.search('div.map-pool > a > div > div').text
      wdl = element.search('div.stats-rows > div > span')[1].text unless element.search('div.stats-rows > div > span')[1].nil?
      win = wdl.split("/").first.to_i unless wdl.nil?
      draw = wdl.split("/")[1].to_i unless wdl.nil?
      losses = wdl.split("/").last.to_i unless wdl.nil?

      win_rate = element.search('div.stats-rows > div > span')[3].text.to_i unless element.search('div.stats-rows > div > span')[3].nil?
      total_rounds = element.search('div.stats-rows > div > span')[5].text.to_i unless element.search('div.stats-rows > div > span')[5].nil?
      round_win_after_first_kill = element.search('div.stats-rows > div > span')[7].text.to_i unless element.search('div.stats-rows > div > span')[7].nil?
      round_win_after_first_death = element.search('div.stats-rows > div > span')[9].text.to_i unless element.search('div.stats-rows > div > span')[9].nil?
      existant_stats = MapStat.where(map: map_name, team: team)

      if existant_stats.any?
        existant_stats.first.update(win: win, draw: draw, losses: losses, win_rate: win_rate, total_rounds: total_rounds, round_win_after_first_kill_percent: round_win_after_first_kill, round_win_after_first_death_percent: round_win_after_first_death)
      else
        MapStat.find_or_create_by(map: map_name, team: team ,win: win, draw: draw, losses: losses, win_rate: win_rate, total_rounds: total_rounds, round_win_after_first_kill_percent: round_win_after_first_kill, round_win_after_first_death_percent: round_win_after_first_death)
      end
    end
  end
end