Class: CrawlKit::S3

Inherits:
Object
  • Object
show all
Defined in:
lib/crawl_kit/s3.rb

Class Method Summary collapse

Class Method Details

.buckets ⇒ Object 



8
9
10
 
# File 'lib/crawl_kit/s3.rb', line 8

def buckets
  connection.buckets
end

.connection ⇒ Object 



4
5
6
 
# File 'lib/crawl_kit/s3.rb', line 4

def connection
  AWS::S3.new
end

.crawl_bucket(bucket_name = nil) ⇒ Object 



12
13
14
15
16
 
# File 'lib/crawl_kit/s3.rb', line 12

def crawl_bucket(bucket_name=nil)
  bucket_name ||= CrawlKit.config.crawl_bucket
  raise "A crawl bucket is required" unless bucket_name
  buckets[bucket_name]
end

.file_exists?(key) ⇒ Boolean

Returns:

  • (Boolean) 


22
23
24
 
# File 'lib/crawl_kit/s3.rb', line 22

def file_exists?(key)
  get_object(key).exists?
end

.get_file(key) ⇒ Object 



26
27
28
29
30
 
# File 'lib/crawl_kit/s3.rb', line 26

def get_file(key)
  if file = get_object(key)
    decompress(file.read)
  end
end

.get_object(key) ⇒ Object 



18
19
20
 
# File 'lib/crawl_kit/s3.rb', line 18

def get_object(key)
  crawl_bucket.objects[key]
end

.get_previous_version(hashed_uri, step = 1) ⇒ Object

A step of 0 will give the current version.



44
45
46
47
 
# File 'lib/crawl_kit/s3.rb', line 44

def get_previous_version(hashed_uri, step=1)
  previous_version = get_versions(hashed_uri).sort_by{|version| version.head.last_modified}.reverse[step]
  decompress(previous_version.read) if previous_version
end

.get_versions(hashed_uri) ⇒ Object 



38
39
40
41
 
# File 'lib/crawl_kit/s3.rb', line 38

def get_versions(hashed_uri)
  s3_object = get_object(hashed_uri)
  versions = s3_object.versions.to_a
end

.save_page(key, body, options) ⇒ Object 



32
33
34
35
36
 
# File 'lib/crawl_kit/s3.rb', line 32

def save_page(key, body, options)
  options.merge!(content_type: 'text/html', content_encoding: 'deflate', storage_class: :reduced_redundancy)
  object = get_object(key)
  object.write(Zlib::Deflate.deflate(body), options)
end