Module: MogileFS::Bigfile

Includes:
Util
Included in:
MogileFS
Defined in:
lib/mogilefs/bigfile.rb

Constant Summary collapse

GZIP_HEADER =

mogtool(1) has this

"\x1f\x8b".freeze

Constants included from Util

Util::CHUNK_SIZE

Instance Method Summary collapse

Methods included from Util

#sysrwloop, #syswrite_full

Instance Method Details

#bigfile_stat(key) ⇒ Object

returns a big_info hash if successful



11
12
13
 
# File 'lib/mogilefs/bigfile.rb', line 11

def bigfile_stat(key)
  parse_info(get_file_data(key))
end

#bigfile_write(key, wr_io, opts = { :verify => false }) ⇒ Object

returns total bytes written and the big_info hash if successful, raises an exception if not wr_io is expected to be an IO-like object capable of receiving the syswrite method.



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/mogilefs/bigfile.rb', line 18

def bigfile_write(key, wr_io, opts = { :verify => false })
  info = bigfile_stat(key)
  zi = nil
  md5 = opts[:verify] ? Digest::MD5.new : nil
  total = 0

  # we only decode raw zlib deflated streams that mogtool (unfortunately)
  # generates.  tarballs and gzip(1) are up to to the application to decrypt.
  filter = Proc.new do |buf|
    if zi == nil
      if info[:compressed] && info[:type] == 'file' &&
           buf.length >= 2 && buf[0,2] != GZIP_HEADER
        zi = Zlib::Inflate.new

        # mogtool(1) seems to have a bug that causes it to generate bogus
        # MD5s if zlib deflate is used.  Don't trust those MD5s for now...
        md5 = nil
      else
        zi = false
      end
    end
    buf ||= ''
    if zi
      zi.inflate(buf)
    else
      md5 << buf
      buf
    end
  end if (info[:compressed] || md5)

  info[:parts].each_with_index do |part,part_nr|
    next if part_nr == 0 # info[:parts][0] is always empty
    uris = verify_uris(part[:paths].map { |path| URI.parse(path) })
    if uris.empty?
      # part[:paths] may not be valid anymore due to rebalancing, however we
      # can get_keys on key,<part_nr> and retry paths if all paths fail
      part[:paths] = get_paths("#{key.gsub(/^big_info:/, '')},#{part_nr}")
      uris = verify_uris(part[:paths].map { |path| URI.parse(path) })
      raise MogileFS::Backend::NoDevices if uris.empty?
    end

    sock = http_get_sock(uris[0])
    md5.reset if md5
    w = sysrwloop(sock, wr_io, filter)

    if md5 && md5.hexdigest != part[:md5]
      raise MogileFS::ChecksumMismatchError, "#{md5} != #{part[:md5]}"
    end
    total += w
  end

  syswrite_full(wr_io, zi.finish) if zi

  [ total, info ]
end