Module: WebArchive

Defined in:

lib/webarchive.rb,
lib/webarchive/version.rb

Overview

classes and functions of webarchive package

Defined Under Namespace

Classes: ArchiveQueue, Client, Completer, NoAlternativeURIError, Req, UnexpectedResponseError

Constant Summary

HISTORY_FILE =

'~/.webarchive_history'

VERSION =

'0.1.5'

Class Method Summary

• .debug_output(source, uri, content) ⇒ void

  Write log to a file.

• .encode_non_ascii(str) ⇒ String
• .launch(wait_secs: 1, max_retry: 3, read_timeout_secs: 15, redirect: false, canonical_uri: true, history: true, debug: false, verbose: false) ⇒ Concurrent::Promises::Future

  Launch the CLI.

• .prepend_http(uri) ⇒ String
• .to_ascii_uri(str) ⇒ Addressable::URI

  Encode non-ASCII components in the given string and make a URI instance from.

Class Method Details

.debug_output(source, uri, content) ⇒ void

This method returns an undefined value.

Write log to a file

Parameters:

• source (String)
• content (String)

# File 'lib/webarchive.rb', line 268

def self.debug_output(source, uri, content)
  ts = Time.now.strftime('%Y%m%d%H%M%S')
  filename = "#{self}-#{source}-#{uri.gsub(/\W+/, '_')[0..30]}-"
  filename += Digest::SHA256.hexdigest(uri + ts)[0..8]
  Tempfile.open(filename) do |f|
    f.puts content
  end
end

.encode_non_ascii(str) ⇒ String

Parameters:

• str (String)

Returns:

• (String)

# File 'lib/webarchive.rb', line 233

def self.encode_non_ascii(str)
  if str =~ /[^[:ascii:]]/
    Addressable::URI.encode(str)
  else
    str
  end
end

.launch(wait_secs: 1, max_retry: 3, read_timeout_secs: 15, redirect: false, canonical_uri: true, history: true, debug: false, verbose: false) ⇒ Concurrent::Promises::Future

Launch the CLI

Returns:

• (Concurrent::Promises::Future)

# File 'lib/webarchive.rb', line 324

def self.launch(wait_secs: 1, max_retry: 3,
                read_timeout_secs: 15,
                redirect: false, canonical_uri: true,
                history: true, debug: false, verbose: false)

  args = method(__method__).parameters.map{ |k, v| [v, binding.local_variable_get(v)] }
  puts "Arguments: #{args.inspect}" if verbose

  wait_secs = 0 if wait_secs.negative?
  max_retry = 0 if max_retry.negative?

  Thread.abort_on_exception = true
  completer = nil
  if history
    completer = Completer.new(File.expand_path(HISTORY_FILE))
    Readline.completion_proc = completer.to_proc
    Readline.completion_append_character = ''
  end

  client = Client.new(wait_secs: wait_secs, max_retry: max_retry,
                      redirect: redirect, canonical_uri: canonical_uri)

  # prepare queues
  client.add_queue(
    ArchiveQueue.new('archive.org (logged out)', wait_secs) do |uri|
      u = URI.parse('https://web.archive.org/save/' + uri)
      u.open(read_timeout: read_timeout_secs) do |f|
        if f.meta['content-location'] && verbose
          puts "<https://web.archive.org#{f.meta['content-location']}>"
        elsif verbose
          puts f.meta.inspect
        end
      end
    end
  )

  client.add_queue(
    ArchiveQueue.new('megalodon.jp', wait_secs) do |uri|
      agent = Mechanize.new
      agent.read_timeout = read_timeout_secs
      page = agent.get('https://megalodon.jp/pc/?' +
                       Addressable::URI.form_encode(url: uri))
      form = page.forms.first
      raise UnexpectedResponseError, page.inspect unless form

      res = agent.submit(form)
      if debug
        debug_output('megalodonjp', uri, res.body)
      end
      og = res.at('meta[property="og:url"]')
      uri = if og
              og[:content]
            else
              res.links.map(&:href).find(-> { res.uri.to_s }) do |x|
                x =~ %r{megalodon\.jp/[\d-]+/}
              end
            end
      puts "<#{uri}>" if verbose
      agent.shutdown
    end
  )

  client.add_queue(
    ArchiveQueue.new('archive.today', wait_secs) do |uri|
      agent = Mechanize.new
      agent.read_timeout = read_timeout_secs
      agent.follow_meta_refresh = true

      page = agent.get('https://archive.is/')
      form = page.form_with(id: 'submiturl')
      if debug
        debug_output('archivetoday', uri, page.inspect)
      end
      raise UnexpectedResponseError, page.inspect unless form

      form['anyway'] = '1'
      form.field_with(name: 'url').value = uri
      sleep 5.0               # not submit too fast
      page = agent.submit(form)
      puts "<#{page.uri}>" if verbose
      agent.shutdown
    end
  )

  # main loop

  uri_regexp = URI::DEFAULT_PARSER.make_regexp
  all = Concurrent::Promises.future{}
  while line = Readline.readline("Q(#{client.queued_uris})> ", add_hist: true)
    uri = ''
    begin
      uri = to_ascii_uri(line).to_s
    rescue Addressable::URI::InvalidURIError => e
      warn_archive_fail(line.strip, '<>', e.message)
    end
    next if uri == ''

    puts uri if verbose

    if uri !~ uri_regexp
      warn "invalid; skipping '#{uri}'"
      next
    end

    f = client.send_uri(uri).then {
      completer&.append_to_history(uri)
    }.on_rejection { |reason, _|
      warn "skipping canonical/redirect for #{uri}: #{reason}" if
        !x.is_a?(NoAlternativeURIError)
    }
    all = all.zip(f)
  end

  all.wait
  client.wait_for_queues
  # TODO: trap INT and ask for confirmation
  all
end

.prepend_http(uri) ⇒ String

Parameters:

• str (String)

Returns:

• (String)

# File 'lib/webarchive.rb', line 243

def self.prepend_http(uri)
  if %r/\.[a-z]{2,4}(\/|$)/.match(uri) && %r{(^http|://)}.match(uri).nil?
    'http://' + uri
  else
    uri
  end
end

.to_ascii_uri(str) ⇒ Addressable::URI

Encode non-ASCII components in the given string and make a URI instance from

Parameters:

• str (String)

Returns:

• (Addressable::URI)

# File 'lib/webarchive.rb', line 254

def self.to_ascii_uri(str)
  uri = prepend_http(str.strip)
  u = Addressable::URI.parse(uri)
  u.host = SimpleIDN.to_ascii(u.host)
  u.path, u.query, u.fragment = [
    u.path, u.query, u.fragment
  ].map(&method(:encode_non_ascii))
  u
end