Class: Dubhe::Sources::Feeds

Inherits:
Object
  • Object
show all
Includes:
Dubhe::Source
Defined in:
lib/dubhe/sources/feeds.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(subscriptions: nil, store: Store.new, fetcher: Fetch::Client.new, policy: Policy.new(profile: :feed), interval: 3600, clock: -> { Time.now }) ⇒ Feeds

Returns a new instance of Feeds.



18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/dubhe/sources/feeds.rb', line 18

def initialize(subscriptions: nil, store: Store.new, fetcher: Fetch::Client.new,
               policy: Policy.new(profile: :feed), interval: 3600, clock: -> { Time.now })
  @store = store
  @fetcher = fetcher
  @policy = policy
  @interval = interval
  @clock = clock
  @subscriptions_path = File.join(@store.root, "sources.jsonc")
  values = subscriptions.nil? ? load_subscriptions : subscriptions
  @subscriptions = Array(values).map { |value| normalize_subscription(value) }
  @raw_entries = {}
  @last_errors = {}
end

Instance Attribute Details

#last_errors ⇒ Object (readonly)

Returns the value of attribute last_errors.



16
17
18
 
# File 'lib/dubhe/sources/feeds.rb', line 16

def last_errors
  @last_errors
end

Class Method Details

.discover(html, base_url: nil) ⇒ Object 



134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/dubhe/sources/feeds.rb', line 134

def self.discover(html, base_url: nil)
  document = html.is_a?(Jabbah::Document) ? html : Jabbah.parse(html)
  document.search("link").filter_map do |link|
    rel = link["rel"].to_s.split.map(&:downcase)
    type = link["type"].to_s.downcase
    next unless rel.include?("alternate") && %w[application/rss+xml application/atom+xml application/feed+json application/json].include?(type)

    href = link["href"]
    next unless href && !href.empty?
    base_url ? URI.join(base_url, href).to_s : href
  rescue URI::InvalidURIError
    nil
  end.uniq
end

Instance Method Details

#add(url, title: nil, site_url: nil, group: nil) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/dubhe/sources/feeds.rb', line 36

def add(url, title: nil, site_url: nil, group: nil)
  normalized = url.to_s.strip
  existing = @subscriptions.find { |subscription| subscription.url == normalized }
  return existing if existing

  subscription = Subscription.new(id: Digest::SHA256.hexdigest(normalized), title: title || normalized,
    url: normalized, site_url: site_url, group: group, etag: nil, last_modified: nil,
    interval: @interval, next_refresh_at: nil, failures: 0)
  @subscriptions << subscription
  persist_subscriptions
  subscription
end

#entries(filter: nil) ⇒ Object 



96
97
98
99
100
101
 
# File 'lib/dubhe/sources/feeds.rb', line 96

def entries(filter: nil)
  result = @store.entries(source_id: id)
  return result unless filter

  result.select { |entry| filter.call(entry) }
end

#export_opml(title: "Dubhe subscriptions") ⇒ Object 



129
130
131
132
 
# File 'lib/dubhe/sources/feeds.rb', line 129

def export_opml(title: "Dubhe subscriptions")
  Iklil.render_opml(@subscriptions.map { |subscription| {title: subscription.title, xml_url: subscription.url,
    html_url: subscription.site_url, category: subscription.group} }, title: title)
end

#fetch_body(entry) ⇒ Object 



103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/dubhe/sources/feeds.rb', line 103

def fetch_body(entry)
  source = @raw_entries[entry.id]
  if source.nil?
    stored = @store.body(entry.id)
    return stored if stored
    return Body.new(kind: :text, content: entry.summary.to_s, base_url: entry.url, attachments: [], headers: {})
  end
  content = source.content.to_s
  if source.html?
    result = @policy.apply(content, base_url: entry.url)
    extracted = Jabbah::Extract.article(result.document)
    content = extracted ? extracted[:content].to_html : result.html
    Body.new(kind: :html, content: content, base_url: entry.url, attachments: source.enclosures, headers: {"blocked_count" => result.blocked_count})
  else
    Body.new(kind: :text, content: content, base_url: entry.url, attachments: source.enclosures, headers: {})
  end
end

#icon ⇒ Object 



34
 
# File 'lib/dubhe/sources/feeds.rb', line 34

def icon = "rss"

#id ⇒ Object 



32
 
# File 'lib/dubhe/sources/feeds.rb', line 32

def id = "feeds"

#import_opml(bytes) ⇒ Object 



125
126
127
 
# File 'lib/dubhe/sources/feeds.rb', line 125

def import_opml(bytes)
  Iklil.parse_opml(bytes).each { |subscription| add(subscription.xml_url, title: subscription.title, site_url: subscription.html_url) }
end

#refresh(force: false) ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/dubhe/sources/feeds.rb', line 59

def refresh(force: false)
  refreshed = []
  now = @clock.call
  @subscriptions = @subscriptions.map do |subscription|
    next subscription if !force && subscription.next_refresh_at && subscription.next_refresh_at > now

    begin
      headers = {"If-None-Match" => subscription.etag, "If-Modified-Since" => subscription.last_modified}
      response = @fetcher.get(subscription.url, headers: headers)
      if response.status == 304
        refreshed.concat(@store.entries(source_id: id).select { |entry| entry.updated_at && entry.updated_at >= now - subscription.interval })
        next schedule(subscription, now: now, failures: 0)
      end
      raise Error, "feed returned HTTP #{response.status}" unless response.status.between?(200, 299)

      feed = Iklil.parse(response.body, base_url: subscription.url)
      entries = feed.entries.map do |raw_entry|
        entry = convert_entry(subscription, raw_entry)
        @raw_entries[entry.id] = raw_entry
        entry
      end
      entries.each do |entry|
        @store.save_entry(entry)
        @store.save_body(entry.id, fetch_body(entry))
      end
      refreshed.concat(entries)
      schedule(subscription, now: now, failures: 0, title: feed.title, site_url: feed.site_url,
        etag: response.headers["etag"], last_modified: response.headers["last-modified"])
    rescue StandardError => error
      @last_errors[subscription.id] = error
      schedule(subscription, now: now, failures: subscription.failures + 1)
    end
  end
  persist_subscriptions
  refreshed
end

#remove(id_or_url) ⇒ Object 



49
50
51
52
53
54
55
 
# File 'lib/dubhe/sources/feeds.rb', line 49

def remove(id_or_url)
  before = @subscriptions.length
  @subscriptions.reject! { |subscription| subscription.id == id_or_url.to_s || subscription.url == id_or_url.to_s }
  changed = before != @subscriptions.length
  persist_subscriptions if changed
  changed
end

#subscriptions ⇒ Object 



57
 
# File 'lib/dubhe/sources/feeds.rb', line 57

def subscriptions = @subscriptions.dup

#supports?(capability) ⇒ Boolean

Returns:

  • (Boolean) 


121
122
123
 
# File 'lib/dubhe/sources/feeds.rb', line 121

def supports?(capability)
  %i[refresh search unread].include?(capability)
end

#title ⇒ Object 



33
 
# File 'lib/dubhe/sources/feeds.rb', line 33

def title = "Feeds"