Class: Ietf::Data::Importer::Scrapers::IetfScraper

Inherits:
BaseScraper
  • Object
show all
Defined in:
lib/ietf/data/importer/scrapers/ietf_scraper.rb

Overview

Scraper for IETF groups from datatracker.ietf.org

Constant Summary collapse

BASE_URL =

Base URL for IETF datatracker

"https://datatracker.ietf.org/group/"

Instance Method Summary collapse

Methods inherited from BaseScraper

#fetch_html, #log

Instance Method Details

#fetch ⇒ Array<Ietf::Data::Importer::Group>

Fetch all IETF groups

Returns:



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/ietf/data/importer/scrapers/ietf_scraper.rb', line 17

def fetch
  groups = []
  log "Fetching IETF groups..."

  # Fetch all group types
  group_types = fetch_group_types

  # For each group type, fetch its groups
  group_types.each do |type|
    log "Fetching #{type[:name]} groups...", 1

    # Skip if URL is empty
    next if type[:url].nil? || type[:url].empty?

    # Construct the full URL
    type_url = if type[:url].start_with?('/')
      "https://datatracker.ietf.org#{type[:url]}"
    else
      "https://datatracker.ietf.org/#{type[:url]}"
    end
    type_doc = fetch_html(type_url)
    next unless type_doc

    # Extract groups from the table
    extract_groups_from_table(type_doc, type, groups)
  end

  groups
end