Class: StudentProgress::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/student_progress/scraper.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Attribute Details

#current_lab ⇒ Object

Returns the value of attribute current_lab.



2
3
4
 
# File 'lib/student_progress/scraper.rb', line 2

def current_lab
  @current_lab
end

#first_name ⇒ Object

Returns the value of attribute first_name.



2
3
4
 
# File 'lib/student_progress/scraper.rb', line 2

def first_name
  @first_name
end

#last_name ⇒ Object

Returns the value of attribute last_name.



2
3
4
 
# File 'lib/student_progress/scraper.rb', line 2

def last_name
  @last_name
end

#profile_url ⇒ Object

Returns the value of attribute profile_url.



2
3
4
 
# File 'lib/student_progress/scraper.rb', line 2

def profile_url
  @profile_url
end

#total_labs_complete ⇒ Object

Returns the value of attribute total_labs_complete.



2
3
4
 
# File 'lib/student_progress/scraper.rb', line 2

def total_labs_complete
  @total_labs_complete
end

#total_lessons_complete ⇒ Object

Returns the value of attribute total_lessons_complete.



2
3
4
 
# File 'lib/student_progress/scraper.rb', line 2

def total_lessons_complete
  @total_lessons_complete
end

Class Method Details

.login_user(email, password) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
 
# File 'lib/student_progress/scraper.rb', line 31

def self.(email, password)
  @@session = Capybara::Session.new(:poltergeist)

  @@session.visit('https://learn.co')
  @@session.fill_in "user-email", with: email
  @@session.fill_in "user-password", with: password
  puts "Signing in now..."
  sleep 0.5
  @@session.click_button "Sign in"
end

.scrape_lessons ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/student_progress/scraper.rb', line 42

def self.scrape_lessons
  doc = Nokogiri::HTML(@@session.body)
  script = doc.css('script')[8].text
  data = script.gsub("\n//<![CDATA[",'').gsub("//]]>",'')
  match_data = /track_nav_data=(.+);/.match(data)
  string = match_data.captures.first.split(';').first
  hash = JSON.parse(string)
  hash['topics'].each.with_index(1) do |topic_hash, index|
    
    topic = StudentProgress::Topic.create_from_hash({
      title: topic_hash['title'],
      id: topic_hash['id']
    })
    topic_hash['units'].each do |unit_hash|
      unit = StudentProgress::Unit.create_from_hash({
        title: unit_hash['title'],
        id: unit_hash['id'],
        topic: topic
      })
      unit_hash['lessons'].each do |lesson_hash|
        lesson = StudentProgress::Lesson.create_from_hash({
          id: lesson_hash['id'],
          title: lesson_hash['title'],
          content_type: lesson_hash['content_type'],
          repo: "https://github.com/#{lesson_hash['github_repo_name']}",
          unit: unit
        })
      end
    end
    
  end
end

.scrape_progress(cohort) ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/student_progress/scraper.rb', line 4

def self.scrape_progress(cohort)
  print "#students loaded: "
  report = StudentProgress::ProgressReport.create(created_at: DateTime.now, cohort_id: cohort.id)
  cohort.students.collect do |s|
    s.github_username
  end.each.with_index(1) do |username, index|
    @@session.visit("https://learn.co/#{username}")
    student = StudentProgress::Student.find_or_create(
      github_username: username
    )
    student.first_name = @@session.first('.media-block__content--fill').text.split("\n").first.split(' ').first
    student.last_name = @@session.first('.media-block__content--fill').text.split("\n").first.split(' ').slice(1)
    student.save
    StudentProgress::StudentReport.create(
      current_lab: @@session.find('h4 a').text,
      lessons_complete: @@session.all('.flex-grid__item__stretcher h3.heading').map{|n| n.text}[1].split(' / ').first.to_i,
      created_at: DateTime.now,
      labs_complete: @@session.all('.flex-grid__item__stretcher h3.heading').map{|n| n.text}[3].split(' / ').first.to_i,
      student_id: student.id,
      progress_report_id: report.id
    )
        
    print "#{index}.."
  end
  puts "Done!"
end