Class: AtCoderFriends::ScrapingAgent

Inherits:
Object
  • Object
show all
Includes:
PathUtil
Defined in:
lib/at_coder_friends/scraping_agent.rb

Overview

scrapes AtCoder contest site and

  • fetches problems

  • submits sources

Constant Summary collapse

BASE_URL = 
'https://atcoder.jp/'
XPATH_SECTION = 
'//h3[.="%<title>s"]/following-sibling::section'
LANG_TBL = 
{
  'cxx'  => '3003',
  'cs'   => '3006',
  'java' => '3016',
  'rb'   => '3024'
}.freeze

Constants included from PathUtil

PathUtil::CASES_DIR, PathUtil::SMP_DIR

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from PathUtil

cases_dir, contest_name, smp_dir, split_prg_path

Constructor Details

#initialize(contest, config) ⇒ ScrapingAgent

Returns a new instance of ScrapingAgent.



26
27
28
29
30
31
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 26

def initialize(contest, config)
  @contest = contest
  @config = config
  @agent = Mechanize.new
  # @agent.log = Logger.new(STDERR)
end

Instance Attribute Details

#agent ⇒ Object (readonly)

Returns the value of attribute agent.



24
25
26
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 24

def agent
  @agent
end

#config ⇒ Object (readonly)

Returns the value of attribute config.



24
25
26
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 24

def config
  @config
end

#contest ⇒ Object (readonly)

Returns the value of attribute contest.



24
25
26
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 24

def contest
  @contest
end

Instance Method Details

#common_url(path) ⇒ Object 



33
34
35
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 33

def common_url(path)
  File.join(BASE_URL, path)
end

#contest_url(path) ⇒ Object 



37
38
39
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 37

def contest_url(path)
  File.join(BASE_URL, 'contests', contest, path)
end

#fetch_all ⇒ Object 



41
42
43
44
45
46
47
48
49
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 41

def fetch_all
  puts "***** fetch_all #{@contest} *****"
  
  fetch_assignments.map do |q, url|
    pbm = fetch_problem(q, url)
    yield pbm if block_given?
    pbm
  end
end

#fetch_assignments ⇒ Object 



69
70
71
72
73
74
75
76
77
78
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 69

def fetch_assignments
  url = contest_url('tasks')
  puts "fetch list from #{url} ..."
  sleep 0.1
  page = agent.get(url)
  ('A'..'Z').each_with_object({}) do |q, h|
    link = page.link_with(text: q)
    link && h[q] = link.href
  end
end

#fetch_problem(q, url) ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 80

def fetch_problem(q, url)
  puts "fetch problem from #{url} ..."
  sleep 0.1
  page = agent.get(url)
  Problem.new(q) do |pbm|
    pbm.html = page.body
    if contest == 'arc001'
      page.search('//h3').each do |h3|
        query = format(XPATH_SECTION, title: h3.content)
        sections = page.search(query)
        sections[0] && parse_section(pbm, h3, sections[0])
      end
    else
      page.search('//*[./h3]').each do |section|
        h3 = section.search('h3')[0]
        parse_section(pbm, h3, section)
      end
    end
  end
end

#login ⇒ Object 



59
60
61
62
63
64
65
66
67
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 59

def 
  sleep 0.1
  page = agent.get(common_url('login'))
  form = page.forms[1]
  form.field_with(name: 'username').value = config['user']
  form.field_with(name: 'password').value = config['password']
  sleep 0.1
  form.submit
end

#parse_section(pbm, h3, section) ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 101

def parse_section(pbm, h3, section)
  title = h3.content.strip
  title.delete!("\u008f\u0090") # agc002
  text = section.content
  code = section.search('pre')[0]&.content || ''
  case title
  when /^制約$/
    pbm.desc += text
  when /^入出?力$/
    pbm.desc += text
    pbm.fmt = code
  when /^入力例\s*(?<no>[\d０-９]+)$/
    pbm.add_smp($LAST_MATCH_INFO[:no], :in, code)
  when /^出力例\s*(?<no>[\d０-９]+)$/
    pbm.add_smp($LAST_MATCH_INFO[:no], :exp, code)
  end
end

#post_src(q, ext, src) ⇒ Object

Raises:



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 119

def post_src(q, ext, src)
  lang_id = LANG_TBL[ext.downcase]
  raise AppError, ".#{ext} is not available." unless lang_id
  sleep 0.1
  page = agent.get(contest_url('submit'))
  form = page.forms[1]
  form.field_with(name: 'data.TaskScreenName') do |sel|
    option = sel.options.find { |op| op.text.start_with?(q) }
    option&.select || (raise AppError, "unknown problem:#{q}.")
  end
  form.add_field!('data.LanguageId', lang_id)
  form.field_with(name: 'sourceCode').value = src
  sleep 0.1
  form.submit
end

#submit(path) ⇒ Object 



51
52
53
54
55
56
57
 
# File 'lib/at_coder_friends/scraping_agent.rb', line 51

def submit(path)
  path, _dir, prg, _base, ext, q = split_prg_path(path)
  puts "***** submit #{prg} *****"
  src = File.read(path, encoding: Encoding::UTF_8)
  
  post_src(q, ext, src)
end