Class: SdbDal::Repository

Inherits:
Object
  • Object
show all
Defined in:
lib/sdb_dal/repository.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(sdb_domain_prefix, clob_bucket, aws_key_id, aws_secret_key, memcache_servers = nil, a_storage = nil, append_table_to_domain = true, options = {}) ⇒ Repository

Returns a new instance of Repository.



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/sdb_dal/repository.rb', line 12

def initialize(
    sdb_domain_prefix,#MyDevPayApp
    clob_bucket,
    aws_key_id,
    aws_secret_key,
    memcache_servers = nil  ,
    a_storage=nil,
    append_table_to_domain=true,
    options={}
                
  )
            
  @use_cache=true
  @storage=a_storage
  @storage||=Storage.new(aws_key_id,aws_secret_key,memcache_servers)
  @sdb_domain_prefix = sdb_domain_prefix
  @clob_bucket = clob_bucket
  @memcache_servers = memcache_servers
  @aws_key_id = aws_key_id
  @aws_secret_key = aws_secret_key
  @append_table_to_domain=append_table_to_domain
  @logger = options[:logger]
  if !@logger
    @logger = Logger.new(STDOUT)
    @logger.level = Logger::ERROR
  end
  @sdb=AwsSdb::Service.new(:access_key_id=>aws_key_id,:secret_access_key=>aws_secret_key,:url=>"http://sdb.amazonaws.com",:logger=>@logger)
  @session_cache=MemoryRepository.new

  # #create_domain()
end

Instance Attribute Details

#storage ⇒ Object

Returns the value of attribute storage.



11
12
13
 
# File 'lib/sdb_dal/repository.rb', line 11

def storage
  @storage
end

#use_cache ⇒ Object

Returns the value of attribute use_cache.



10
11
12
 
# File 'lib/sdb_dal/repository.rb', line 10

def use_cache
  @use_cache
end

Instance Method Details

#build_query(table_name, attribute_descriptions, options = {}) ⇒ Object 



149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/sdb_dal/repository.rb', line 149

def build_query(table_name,attribute_descriptions,options={})


  query=""
  params=nil
  params=options[:params] if options.has_key?(:params)
  if options.has_key?(:map)
    params||={}
    keys=options[:map][:keys]
    values=options[:map][:values]
    (0..keys.length-1).each do |i|
      key=keys[i]
      value=values[i]
      params[key]=value
    end

  end
  if !@append_table_to_domain
    extend_query(query," ['metadata%%table_name' = '#{table_name}']")

  end
  if options
    if options.has_key?(:query)
      extend_query(query,"["+options[:query]+"]")
    end
    if params
      params.each do |key,value|
        got_something=false
        if attribute_descriptions.has_key?(key)
          if value==:NOT_NULL
            got_something=true
            extend_query(query," ['#{key}' starts-with '']")
          end
          if value==:NULL
            got_something=true
            extend_query(query," not ['#{key}' starts-with '']")
          end
          if value.respond_to?(:less_than) && value.less_than
            got_something=true
            extend_query(query," ['#{key}' < '#{escape_quotes(attribute_descriptions[key].format_for_sdb_single( value.less_than))}']")
          end
          if value.respond_to?(:greater_than) && value.greater_than
            got_something=true
            extend_query(query," ['#{key}' > '#{escape_quotes( attribute_descriptions[key].format_for_sdb_single( value.greater_than))}']")
          end
          if value.respond_to?(:less_than_or_equal_to) && value.less_than_or_equal_to
            got_something=true
            extend_query(query,"['#{key}' <= '#{escape_quotes( attribute_descriptions[key].format_for_sdb_single( value.less_than_or_equal_to))}']")
          end
          if value.respond_to?(:greater_than_or_equal_to) && value.greater_than_or_equal_to
            got_something=true
            extend_query(query," ['#{key}' >= '#{escape_quotes(attribute_descriptions[key].format_for_sdb_single( value.greater_than_or_equal_to))}']")
          end
          if !got_something
            extend_query(query," ['#{key}' = '#{escape_quotes( attribute_descriptions[key].format_for_sdb_single(value))}']")
          end
        else
          # #it must be formatted already.  likely an index
          extend_query(query,"['#{key}' = '#{escape_quotes value}']")
        end
      end
    end
    if options.has_key?(:order_by) && options[:order_by]
      clause=" ['#{options[:order_by]}' starts-with ''] sort '#{options[:order_by]}' "
      if options.has_key?(:order) and options[:order]==:descending
        clause<<" desc "
      end
      extend_query(query,clause)
    end
    if options.has_key?(:conditions)
      options[:conditions].each do |condition|

        extend_query(query," [ "+condition.to_sdb_query()+"]")
      end
    end
         
  end
  return query
   
end

#clear ⇒ Object 



410
411
412
 
# File 'lib/sdb_dal/repository.rb', line 410

def  clear
  @session_cache.clear
end

#clear_session_cache ⇒ Object 



44
45
46
 
# File 'lib/sdb_dal/repository.rb', line 44

def clear_session_cache
  @session_cache.clear
end

#create_domain ⇒ Object

def put_into_cache(table_name, primary_key, formatted_attributes)

    if @use_cache
    cacheItem=DomainObjectCacheItem.new(table_name, primary_key, formatted_attributes)

    yaml=cacheItem.ya2yaml(:syck_compatible => true)

    @storage.put(
      @clob_bucket,
      make_cache_key(table_name,primary_key),
      yaml ,
      {})
  end
end



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/sdb_dal/repository.rb', line 108

def create_domain
  20.times do |i|
    begin
      @sdb.create_domain(@sdb_domain_prefix)
      return
  
    rescue  => e
      s= "#{e.message}\n#{e.backtrace}"
      @logger.warn(s) if @logger
      sleep(i*i)
           
    end
  end
      
end

#destroy(table_name, primary_key) ⇒ Object 



397
398
399
400
401
402
403
 
# File 'lib/sdb_dal/repository.rb', line 397

def destroy(table_name, primary_key)
  @session_cache.destroy(table_name,primary_key)
  @sdb.delete_attributes(make_domain_name(table_name),make_cache_key(table_name, primary_key) )
  #    if @use_cache
  #      @storage.delete(@clob_bucket,make_cache_key(table_name,primary_key))
  #    end
end

#escape_quotes(value) ⇒ Object 



145
146
147
148
 
# File 'lib/sdb_dal/repository.rb', line 145

def escape_quotes(value)
  return nil unless value
  value.gsub( "'","\\'")
end

#extend_query(query, new_clause) ⇒ Object 



138
139
140
141
142
143
144
 
# File 'lib/sdb_dal/repository.rb', line 138

def extend_query(query,new_clause)
  if query.length>0
    query << " intersection "
  end

  query << new_clause
end

#find_one(table_name, primary_key, attribute_descriptions) ⇒ Object

, non_clob_attribute_names, clob_attribute_names)



295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
 
# File 'lib/sdb_dal/repository.rb', line 295

def find_one(table_name, primary_key,attribute_descriptions)#, non_clob_attribute_names, clob_attribute_names)
  session_cache_result=@session_cache.find_one(table_name, make_cache_key(table_name,primary_key),attribute_descriptions)
  return session_cache_result if session_cache_result
  #    if @use_cache
  #      [email protected](@clob_bucket,make_cache_key(table_name,primary_key))
  #      if yaml
  #        result=YAML::load( yaml)
  #        if result.respond_to?(:non_clob_attributes) && result.non_clob_attributes!=nil
  #          return parse_attributes(attribute_descriptions, result.non_clob_attributes)
  #        end
  #
  #      end
  #    end
  attributes=parse_attributes(attribute_descriptions,sdb_get_attributes(table_name,primary_key))
  if attributes
    # #attributes[:primary_key]=primary_key #put_into_cache(table_name,
    # primary_key, attributes)
  end
   
  attributes
end

#flatten_key(key) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/sdb_dal/repository.rb', line 47

def flatten_key(key)
  if key.is_a?( Array)
    flattened_key=""
    key.each do |key_part|
      flattened_key<<CGI.escape(key_part.to_s)+"/"
    end
    return flattened_key[0..-2]
  else
    return CGI.escape(key.to_s)
  end
end

#get_clob(table_name, primary_key, clob_name) ⇒ Object 



67
68
69
70
 
# File 'lib/sdb_dal/repository.rb', line 67

def get_clob(table_name,primary_key,clob_name)
  return @storage.get(@clob_bucket,make_clob_key(table_name,primary_key,clob_name))
    
end

#make_cache_key(table_name, primary_key) ⇒ Object 



58
59
60
61
62
63
 
# File 'lib/sdb_dal/repository.rb', line 58

def make_cache_key(table_name,primary_key)

  primary_key=flatten_key(primary_key)
  primary_key="#{table_name}/#{primary_key}" unless @append_table_to_domain
  return primary_key
end

#make_clob_key(table_name, primary_key, clob_name) ⇒ Object 



64
65
66
 
# File 'lib/sdb_dal/repository.rb', line 64

def make_clob_key(table_name,primary_key,clob_name)
  return "clobs/#{table_name}/#{flatten_key(primary_key)}/#{clob_name}"
end

#make_domain_name(table_name) ⇒ Object 



316
317
318
319
320
321
322
 
# File 'lib/sdb_dal/repository.rb', line 316

def make_domain_name(table_name)
  if @append_table_to_domain
    @sdb_domain_prefix+"_"+table_name
  else
    @sdb_domain_prefix
  end
end

#parse_attributes(attribute_descriptions, attributes) ⇒ Object 



379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
 
# File 'lib/sdb_dal/repository.rb', line 379

def parse_attributes(attribute_descriptions,attributes)
  if !attributes || attributes.length==0
    return nil
  end
  parsed_attributes={}
  attribute_descriptions.each do |attribute_name,attribute_description|
    value=attributes[attribute_name.to_sym]
    if !value
      value=attributes[attribute_name]
    end
    #sdb attributes are often array of one
    if !attribute_description.is_collection && value.respond_to?(:flatten) && value.length==1
      value=value[0]
    end
    parsed_attributes[attribute_name.to_sym]=attribute_description.parse_from_sdb(value)
  end
  parsed_attributes
end

#pause ⇒ Object

parse date in yyyy-mm-dd format



406
407
408
 
# File 'lib/sdb_dal/repository.rb', line 406

def pause
  sleep(2)
end

#put_attributes(table_name, primary_key, formatted_attributes, options = {}) ⇒ Object 



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/sdb_dal/repository.rb', line 123

def put_attributes(table_name,primary_key, formatted_attributes,options={})
  20.times do |i|
    begin
      @sdb.put_attributes(make_domain_name(table_name),make_cache_key(table_name,primary_key) , formatted_attributes, true )
      return

    rescue Exception => e
      s= "#{e.message}\n#{e.backtrace}"
      @logger.warn(s) if @logger
      puts e.to_yaml
      sleep(i*i)
    end
  end
   
end

#query(table_name, attribute_descriptions, options) ⇒ Object

#result will be an array of hashes. each hash is a set of attributes



230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/sdb_dal/repository.rb', line 230

def query(table_name,attribute_descriptions,options)
  if options.has_key?(:limit) and !options.has_key?(:order_by)
    session_cache_result=@session_cache.query(table_name,attribute_descriptions,options)
    if options[:limit]==session_cache_result.length
      return session_cache_result
    end
  end

  the_query=build_query(table_name, attribute_descriptions, options)
  max=250
  if options[:limit]
    max=options[:limit].to_i

  end
  page_size=max>250?250:max
  sdb_result,token=sdb_query_with_attributes(table_name,the_query,page_size)

  while !(token.nil? || token.empty? || sdb_result.length>=max)
    page_size=max- sdb_result.length
    page_size=page_size>250?250:page_size
    partial_results,token=sdb_query_with_attributes(table_name,the_query,page_size,token)

    sdb_result.merge!( partial_results)
  end
  result=[]
  sdb_result.each{|primary_key,sdb_row|
    attributes =parse_attributes(attribute_descriptions,sdb_row)
    if attributes
      # #attributes[:primary_key]=primary_key

      result<<attributes
    end
  }
  if options and options[:order_by]
    result.sort! do |a,b|
      a_value=a[options[:order_by]]
      b_value=b[options[:order_by]]
      if options[:order] && options[:order]!=:ascending
        if !a_value
          1
        else
          if b_value
            b_value <=> a_value
          else
            -1
          end
        end
      else
        if !b_value
          1
        else
          if a_value
            a_value <=> b_value
          else
            -1
          end
        end
      end
    end
  end
  if options[:limit] && result.length>options[:limit]
    result=result[0..(options[:limit]-1)]
  end
  return result
end

#save(table_name, primary_key, attributes, index_descriptions) ⇒ Object 



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/sdb_dal/repository.rb', line 71

def save(table_name, primary_key, attributes,index_descriptions)
  @session_cache.save(table_name,primary_key,attributes,index_descriptions)
  formatted_attributes={}
  attributes.each do |description,value|
    if value || description.value_type==:boolean
      if description.is_clob
        @storage.put(
          @clob_bucket,
          make_clob_key(table_name,primary_key,description.name),
          value.to_s,
          {})
      else
        formatted_attributes[description.name]=description.format_for_sdb(value)
      end
    end
  end
  if !@append_table_to_domain
    formatted_attributes['metadata%%table_name'] = table_name

  end
  put_attributes(table_name,primary_key, formatted_attributes )
  # put_into_cache(table_name, primary_key, formatted_attributes)
  
end

#sdb_get_attributes(table_name, primary_key) ⇒ Object 



323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
 
# File 'lib/sdb_dal/repository.rb', line 323

def sdb_get_attributes(table_name,primary_key)
  
  @logger.debug( "SDB get_attributes #{table_name} : #{primary_key}") if @logger
 
  20.times do |i|
    begin
      return @sdb.get_attributes(make_domain_name(table_name), make_cache_key(table_name,primary_key))
    rescue Exception => e
      s= "#{e.message}\n#{e.backtrace}"
      @logger.warn(s) if @logger
   
      sleep(i*i)
    ensure
     
    end
  end

end

#sdb_query(table_name, query, max, token = nil) ⇒ Object 



341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
 
# File 'lib/sdb_dal/repository.rb', line 341

def sdb_query(table_name,query,max,token=nil)
  
  @logger.debug( "SDB query:#{table_name}(#{max}) : #{query}   #{token}"  ) if @logger
  #      puts "#{table_name}  #{query}   (#{max}) #{token}"
  20.times do |i|
    begin
      return @sdb.query(make_domain_name(table_name),query,max,token)
    
    rescue Exception => e
      s= "#{e.message}\n#{e.backtrace}"
      @logger.error(s) if @logger
   
      sleep(i*i)
    ensure
     
    end
  end

end

#sdb_query_with_attributes(table_name, query, max, token = nil) ⇒ Object 



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
 
# File 'lib/sdb_dal/repository.rb', line 360

def sdb_query_with_attributes(table_name,query,max,token=nil)

  @logger.debug( "SDB query:#{table_name}(#{max}) : #{query}   #{token}"  ) if @logger
#      puts "#{table_name}  #{query}   (#{max}) #{token}"
  20.times do |i|
    begin
      return @sdb.query_with_attributes(make_domain_name(table_name),query,max,token)

    rescue Exception => e
      s= "#{e.message}\n#{e.backtrace}"
      @logger.error(s) if @logger

      sleep(i*i)
    ensure

    end
  end

end