Class: DevTo::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/dev_to/scraper.rb

Instance Method Summary collapse

Instance Method Details

#get_page ⇒ Object 



3
4
5
 
# File 'lib/dev_to/scraper.rb', line 3

def get_page
  Nokogiri::HTML(open("https://dev.to/"))
end

#get_posts ⇒ Object 



7
8
9
 
# File 'lib/dev_to/scraper.rb', line 7

def get_posts
  self.get_page.css(".articles-list .substories .single-article")
end

#make_content(post) ⇒ Object 



24
25
26
27
28
 
# File 'lib/dev_to/scraper.rb', line 24

def make_content(post)
  post_page = Nokogiri::HTML(open("#{post.url}"))
  post.content = post_page.search("body #article-body").text.strip
  post.date = post_page.search("#main-title .published-at").text.strip
end

#make_posts ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/dev_to/scraper.rb', line 11

def make_posts
  self.get_posts.each do |data|
    post = DevTo::Post.new
    post.title = data.search(".content h3 span").remove
    post.title = data.search(".content h3").text.strip.gsub(/[[:space:]]+/," ")
    post.author = data.search("h4").text.strip
    post.tags = data.search(".tags .tag").collect { |tag| tag.text.strip}
    post.url = "https://dev.to" + data.search("a").last.attr("href")
    post.comments = data.search(".engagement-count-number").first.text.strip
    post.likes = data.search(".engagement-count-number").last.text.strip
  end
end