Class: MogileFS::MogileFS

Inherits:
Client
  • Object
show all
Includes:
Bigfile, Util
Defined in:
lib/mogilefs/mogilefs.rb

Overview

MogileFS File manipulation client.

Constant Summary collapse

HTTP_200_OK =

internal Regexp for matching an “HTTP 200 OK” head response

%r{\AHTTP/\d+\.\d+\s+200\s+}.freeze

Constants included from Bigfile

Bigfile::GZIP_HEADER

Constants included from Util

Util::CHUNK_SIZE

Instance Attribute Summary collapse

Attributes inherited from Client

#backend, #hosts

Instance Method Summary collapse

Methods included from Bigfile

#bigfile_stat, #bigfile_write

Methods included from Util

#sysrwloop, #syswrite_full

Methods inherited from Client

#err, #errstr, #readonly?, #reload

Constructor Details

#initialize(args = {}) ⇒ MogileFS

Creates a new MogileFS::MogileFS instance. args must include a key :domain specifying the domain of this client.

Raises:

  • (ArgumentError) 


30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/mogilefs/mogilefs.rb', line 30

def initialize(args = {})
  @domain = args[:domain]

  @get_file_data_timeout = 5

  raise ArgumentError, "you must specify a domain" unless @domain

  if @backend = args[:db_backend]
    @readonly = true
  else
    super
  end
end

Instance Attribute Details

#domain ⇒ Object (readonly)

The domain of keys for this MogileFS client.



15
16
17
 
# File 'lib/mogilefs/mogilefs.rb', line 15

def domain
  @domain
end

#get_file_data_timeout ⇒ Object

The timeout for get_file_data. Defaults to five seconds.



20
21
22
 
# File 'lib/mogilefs/mogilefs.rb', line 20

def get_file_data_timeout
  @get_file_data_timeout
end

Instance Method Details

#delete(key) ⇒ Object

Removes key.

Raises:



177
178
179
180
181
 
# File 'lib/mogilefs/mogilefs.rb', line 177

def delete(key)
  raise MogileFS::ReadOnlyError if readonly?

  @backend.delete :domain => @domain, :key => key
end

#each_key(prefix) ⇒ Object

Enumerates keys starting with key.



47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/mogilefs/mogilefs.rb', line 47

def each_key(prefix)
  after = nil

  keys, after = list_keys prefix

  until keys.nil? or keys.empty? do
    keys.each { |k| yield k }
    keys, after = list_keys prefix, after
  end

  nil
end

#get_file_data(key, &block) ⇒ Object

Retrieves the contents of key.



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/mogilefs/mogilefs.rb', line 63

def get_file_data(key, &block)
  paths = get_paths key

  return nil unless paths

  paths.each do |path|
    next unless path
    case path
    when /^http:\/\// then
      begin
        sock = http_get_sock(URI.parse(path))
        return block_given? ? yield(sock) : sock.read
      rescue MogileFS::Timeout, Errno::ECONNREFUSED,
             EOFError, SystemCallError, MogileFS::InvalidResponseError
        next
      end
    else
      next unless File.exist? path
      return File.read(path)
    end
  end

  nil
end

#get_paths(key, noverify = true, zone = nil) ⇒ Object

Get the paths for key.



91
92
93
94
95
96
97
 
# File 'lib/mogilefs/mogilefs.rb', line 91

def get_paths(key, noverify = true, zone = nil)
  opts = { :domain => @domain, :key => key,
           :noverify => noverify ? 1 : 0, :zone => zone }
  @backend.respond_to?(:_get_paths) and return @backend._get_paths(opts)
  res = @backend.get_paths(opts)
  (1..res['paths'].to_i).map { |i| res["path#{i}"] }
end

#list_keys(prefix, after = nil, limit = 1000, &block) ⇒ Object

Lists keys starting with prefix follwing after up to limit. If after is nil the list starts at the beginning.



245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
 
# File 'lib/mogilefs/mogilefs.rb', line 245

def list_keys(prefix, after = nil, limit = 1000, &block)
  if @backend.respond_to?(:_list_keys)
    return @backend._list_keys(domain, prefix, after, limit, &block)
  end

  res = begin
    @backend.list_keys(:domain => domain, :prefix => prefix,
                       :after => after, :limit => limit)
  rescue MogileFS::Backend::NoneMatchError
    return nil
  end

  keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
  if block_given?
    # emulate the MogileFS::Mysql interface, slowly...
    keys.each do |key|
      paths = get_paths(key) or next
      length = paths_size(paths) or next
      yield key, length, paths.size
    end
  end

  [ keys, res['next_after'] ]
end

#new_file(key, klass = nil, bytes = 0, &block) ⇒ Object

Creates a new file key in klass. bytes is currently unused.

The block operates like File.open.

Raises:



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/mogilefs/mogilefs.rb', line 104

def new_file(key, klass = nil, bytes = 0, &block) # :yields: file
  raise MogileFS::ReadOnlyError if readonly?
  opts = { :domain => @domain, :key => key, :multi_dest => 1 }
  opts[:class] = klass if klass
  res = @backend.create_open(opts)

  dests = if dev_count = res['dev_count'] # multi_dest succeeded
    (1..dev_count.to_i).map do |i|
      [res["devid_#{i}"], res["path_#{i}"]]
    end
  else # single destination returned
    # 0x0040:  d0e4 4f4b 2064 6576 6964 3d31 2666 6964  ..OK.devid=1&fid
    # 0x0050:  3d33 2670 6174 683d 6874 7470 3a2f 2f31  =3&path=http://1
    # 0x0060:  3932 2e31 3638 2e31 2e37 323a 3735 3030  92.168.1.72:7500
    # 0x0070:  2f64 6576 312f 302f 3030 302f 3030 302f  /dev1/0/000/000/
    # 0x0080:  3030 3030 3030 3030 3033 2e66 6964 0d0a  0000000003.fid..

    [[res['devid'], res['path']]]
  end

  case (dests[0][1] rescue nil)
  when nil, '' then
    raise MogileFS::EmptyPathError
  when /^http:\/\// then
    MogileFS::HTTPFile.open(self, res['fid'], klass, key,
                            dests, bytes, &block)
  else
    raise MogileFS::UnsupportedPathError,
          "paths '#{dests.inspect}' returned by backend is not supported"
  end
end

#paths_size(paths) ⇒ Object 



208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
 
# File 'lib/mogilefs/mogilefs.rb', line 208

def paths_size(paths)
  paths.each do |path|
    next unless path
    case path
    when /^http:\/\// then
      begin
        url = URI.parse path
        s = Socket.mogilefs_new_request(url.host, url.port,
                                 "HEAD #{url.request_uri} HTTP/1.0\r\n\r\n",
                                 @get_file_data_timeout)
        res = s.recv(4096, 0)
        if res =~ HTTP_200_OK
          head, body = res.split(/\r\n\r\n/, 2)
          if head =~ /^Content-Length:\s*(\d+)/i
            return $1.to_i
          end
        end
        next
      rescue MogileFS::Timeout, Errno::ECONNREFUSED,
             EOFError, SystemCallError
        next
      ensure
        s.close rescue nil
      end
    else
      next unless File.exist? path
      return File.size(path)
    end
  end

  nil
end

#rename(from, to) ⇒ Object

Renames a key from to key to.

Raises:



193
194
195
196
197
198
 
# File 'lib/mogilefs/mogilefs.rb', line 193

def rename(from, to)
  raise MogileFS::ReadOnlyError if readonly?

  @backend.rename :domain => @domain, :from_key => from, :to_key => to
  nil
end

#size(key) ⇒ Object

Returns the size of key.



202
203
204
205
206
 
# File 'lib/mogilefs/mogilefs.rb', line 202

def size(key)
  @backend.respond_to?(:_size) and return @backend._size(domain, key)
  paths = get_paths(key) or return nil
  paths_size(paths)
end

#sleep(duration) ⇒ Object

Sleeps duration.



186
187
188
 
# File 'lib/mogilefs/mogilefs.rb', line 186

def sleep(duration)
  @backend.sleep :duration => duration
end

#store_content(key, klass, content) ⇒ Object

Stores content into key in class klass.

Raises:



160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/mogilefs/mogilefs.rb', line 160

def store_content(key, klass, content)
  raise MogileFS::ReadOnlyError if readonly?

  new_file key, klass do |mfp|
    if content.is_a?(MogileFS::Util::StoreContent)
      mfp.streaming_io = content
    else
      mfp << content
    end
  end

  content.length
end

#store_file(key, klass, file) ⇒ Object

Copies the contents of file into key in class klass. file can be either a file name or an object that responds to #read.

Raises:



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/mogilefs/mogilefs.rb', line 140

def store_file(key, klass, file)
  raise MogileFS::ReadOnlyError if readonly?

  new_file key, klass do |mfp|
    if file.respond_to? :sysread then
      return sysrwloop(file, mfp)
    else
	if File.size(file) > 0x10000 # Bigass file, handle differently
 mfp.big_io = file
 return
	else
        return File.open(file, "rb") { |fp| sysrwloop(fp, mfp) }
      end
    end
  end
end