Class: Spider::Model::Storage::BaseStorage

Inherits:
Object
  • Object
show all
Includes:
Logger
Defined in:
lib/spiderfw/model/storage/base_storage.rb

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Logger

add, close, close_all, datetime_format, datetime_format=, #debug, debug, debug?, #debug?, enquire_loggers, #error, error, #error?, error?, #fatal, fatal, #fatal?, fatal?, info, #info, info?, #info?, method_missing, open, reopen, send_to_loggers, unknown, #unknown, #warn, warn, warn?, #warn?

Constructor Details

#initialize(url) ⇒ BaseStorage

Returns a new instance of BaseStorage.



86
87
88
89
90
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 86

def initialize(url)
    @url = url
    @configuration = {}
    parse_url(url)
end

Class Attribute Details

.capabilities ⇒ Object (readonly)

An Hash of storage capabilities. The default for db storages is => false, :sequences => true, :transactions => true (The BaseStorage class provides file sequences in case the subclass does not support them.)



18
19
20
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 18

def capabilities
  @capabilities
end

Instance Attribute Details

#instance_name ⇒ Object

Returns the value of attribute instance_name.



8
9
10
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 8

def instance_name
  @instance_name
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



7
8
9
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 7

def url
  @url
end

Class Method Details

.base_types ⇒ Object 



28
29
30
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 28

def base_types
    Model.base_types
end

.connection_alive?(conn) ⇒ Boolean

Checks whether a connection is still alive. Must be implemented by subclasses.

Returns:

  • (Boolean) 


75
76
77
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 75

def connection_alive?(conn)
    raise "Virtual"
end

.connection_attributes ⇒ Object 



152
153
154
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 152

def self.connection_attributes
    @connection_attributes ||= {}
end

.connection_pools ⇒ Object 



46
47
48
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 46

def connection_pools
    @pools ||= {}
end

.disconnect(conn) ⇒ Object 



70
71
72
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 70

def disconnect(conn)
    raise "Virtual"
end

.get_connection(*args) ⇒ Object 



50
51
52
53
54
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 50

def get_connection(*args)
    @pools ||= {}
    @pools[args] ||= ConnectionPool.new(args, self)
    @pools[args].get_connection
end

.inherited(subclass) ⇒ Object 



79
80
81
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 79

def inherited(subclass)
    subclass.instance_variable_set("@capabilities", @capabilities)
end

.max_connections ⇒ Object 



42
43
44
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 42

def max_connections
    nil
end

.new_connection(*args) ⇒ Object

Returns a new connection. Must be implemented by the subclasses; args are implementation specific.



38
39
40
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 38

def new_connection(*args)
    raise "Unimplemented"
end

.release_connection(conn, conn_params) ⇒ Object

Frees a connection, relasing it to the pool



57
58
59
60
61
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 57

def release_connection(conn, conn_params)
    return unless conn
    return unless @pools && @pools[conn_params]
    @pools[conn_params].release(conn)
end

.remove_connection(conn, conn_params) ⇒ Object

Removes a connection from the pool.



64
65
66
67
68
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 64

def remove_connection(conn, conn_params)
    return unless conn
    return unless @pools && @pools[conn_params]
    @pools[conn_params].remove(conn)
end

.sequence_sync ⇒ Object 



24
25
26
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 24

def sequence_sync
    @sequence_sync ||= ::Sync.new
end

.storage_type ⇒ Object 



20
21
22
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 20

def storage_type
    :none
end

.supports?(capability) ⇒ Boolean

True if given named capability is supported by the Storage.

Returns:

  • (Boolean) 


33
34
35
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 33

def supports?(capability)
    @capabilities[capability]
end

Instance Method Details

#==(storage) ⇒ Object 



191
192
193
194
195
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 191

def ==(storage)
    return false unless self.class == storage.class
    return false unless self.url == storage.url
    return true
end

#commit ⇒ Object

Raises:



236
237
238
239
240
241
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 236

def commit
    return false unless transactions_enabled?
    raise StorageException, "Commit without a transaction" unless in_transaction?
    return curr[:savepoints].pop unless curr[:savepoints].empty?
    commit!
end

#commit! ⇒ Object 



255
256
257
258
259
260
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 255

def commit!
    Spider.logger.debug("#{self.class.name} commit connection #{curr[:conn].object_id}")
    curr[:transaction_nesting] = 0
    do_commit
    release
end

#commit_or_continue ⇒ Object

Raises:



243
244
245
246
247
248
249
250
251
252
253
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 243

def commit_or_continue
    return false unless transactions_enabled?
    raise StorageException, "Commit without a transaction" unless in_transaction?
    if curr[:transaction_nesting] == 1
        commit
        curr[:transaction_nesting] = 0
        return true
    else
        curr[:transaction_nesting] -= 1
    end
end

#configure(conf) ⇒ Object 



92
93
94
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 92

def configure(conf)
    @configuration.merge!(conf.to_hash)
end

#connect ⇒ Object

Instantiates a new connection with current connection params.



128
129
130
131
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 128

def connect
    return self.class.get_connection(*@connection_params)
    #Spider::Logger.debug("#{self.class.name} in thread #{Thread.current} acquired connection #{@conn}")
end

#connected? ⇒ Boolean

True if currently connected.

Returns:

  • (Boolean) 


134
135
136
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 134

def connected?
    curr[:conn] != nil
end

#connection ⇒ Object

Returns the current connection, or creates a new one. If a block is given, will release the connection after yielding.



141
142
143
144
145
146
147
148
149
150
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 141

def connection
    curr[:conn] = connect
    if block_given?
        yield curr[:conn]
        release # unless is_connected
        return true
    else
        return curr[:conn]
    end
end

#connection_attributes ⇒ Object 



156
157
158
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 156

def connection_attributes
    self.class.connection_attributes[connection] ||= {}
end

#connection_pool ⇒ Object 



123
124
125
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 123

def connection_pool
    self.class.connection_pools[@connection_params]
end

#create_sequence(name, start = 1, increment = 1) ⇒ Object 



308
309
310
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 308

def create_sequence(name, start=1, increment=1)
    sequence_next(name, start-1, increment)
end

#curr ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 108

def curr
    var = nil
    if Spider.conf.get('storage.shared_connection')
        $STORAGES ||= {}
        var = $STORAGES
    else
        var = Thread.current
    end
    var[:storages] ||= {}
    var[:storages][self.class.storage_type] ||= {}
    var[:storages][self.class.storage_type][@connection_params] ||= {
        :transaction_nesting => 0, :savepoints => []
    }
end

#do_commit ⇒ Object

Raises:



262
263
264
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 262

def do_commit
    raise StorageException, "The current storage does not support transactions" 
end

#do_rollback ⇒ Object

Raises:



280
281
282
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 280

def do_rollback
    raise StorageException, "The current storage does not support transactions" 
end

#do_start_transaction ⇒ Object

May be implemented by subclasses.

Raises:



217
218
219
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 217

def do_start_transaction
   raise StorageException, "The current storage does not support transactions" 
end

#generate_uuid ⇒ Object 



312
313
314
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 312

def generate_uuid
    Spider::DataTypes::UUID.generate
end

#get_mapper(model) ⇒ Object

Raises:



100
101
102
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 100

def get_mapper(model)
    raise StorageException, "Unimplemented"
end

#in_transaction ⇒ Object 



221
222
223
224
225
226
227
228
229
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 221

def in_transaction
    if in_transaction?
        curr[:transaction_nesting] += 1
        return true
    else
        start_transaction
        return false
    end
end

#in_transaction? ⇒ Boolean

Returns:

  • (Boolean) 


231
232
233
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 231

def in_transaction?
    return false
end

#parse_url(url) ⇒ Object

Raises:



96
97
98
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 96

def parse_url(url)
    raise StorageException, "Unimplemented"
end

#prepare_value(type, value) ⇒ Object 



187
188
189
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 187

def prepare_value(type, value)
    return value
end

#release ⇒ Object

Releases the current connection to the pool.



161
162
163
164
165
166
167
168
169
170
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 161

def release
    # The subclass should check if the connection is alive, and if it is not call remove_connection instead
    c = curr[:conn]
    #Spider.logger.debug("#{self} in thread #{Thread.current} releasing #{curr[:conn]}")
    curr[:conn] = nil
    self.class.release_connection(c, @connection_params)
    #Spider.logger.debug("#{self} in thread #{Thread.current} released #{curr[:conn]}")
    return nil
    #@conn = nil
end

#rollback ⇒ Object 



266
267
268
269
270
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 266

def rollback
    raise "Can't rollback in a nested transaction" if curr[:transaction_nesting] > 1
    return rollback_savepoint(curr[:savepoints].last) unless curr[:savepoints].empty?
    rollback!
end

#rollback! ⇒ Object 



272
273
274
275
276
277
278
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 272

def rollback!
    curr[:transaction_nesting] = 0
    Spider.logger.debug("#{self.class.name} rollback")
    do_rollback
    curr[:savepoints] = []
    release
end

#rollback_savepoint(name = nil) ⇒ Object 



288
289
290
291
292
293
294
295
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 288

def rollback_savepoint(name=nil)
    if name
        curr[:savepoints] = curr[:savepoints][0,(curr[:savepoints].index(name))]
        name
    else
        curr[:savepoints].pop
    end
end

#savepoint(name) ⇒ Object 



284
285
286
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 284

def savepoint(name)
    curr[:savepoints] << name
end

#sequence_exists?(name) ⇒ Boolean

Returns:

  • (Boolean) 


304
305
306
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 304

def sequence_exists?(name)
    File.exist?(sequence_file_path(name))
end

#sequence_file_path(name) ⇒ Object

Utility methods



299
300
301
302
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 299

def sequence_file_path(name)
    path = 'var/sequences/'+name
    return path
end

#sequence_next(name, newval = nil, increment = 1) ⇒ Object

Increments a named sequence and returns the new value



323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 323

def sequence_next(name, newval=nil, increment=1)
    path = sequence_file_path(name)
    FileUtils.mkpath(File.dirname(path))
    self.class.sequence_sync.lock(::Sync::EX)
    if newval
        seq = newval
    else
        seq = 0
        File.open(path, 'a+') do |f|
            f.rewind
            f.flock File::LOCK_EX
            cur = f.gets
            if (cur)
                seq, increment_str = cur.split('|')
            else
                seq, increment_str = 0, 1
            end
            seq = seq.to_i
            increment = increment_str.to_i if increment_str
            f.close
        end
        seq += increment
    end
    File.open(path, 'w+') do |f|
        f.print(seq)
        f.print("|#{increment}") if (increment != 1)
        f.flock File::LOCK_UN
        f.close
    end
    self.class.sequence_sync.lock(::Sync::UN)
    return seq
end

#start_transaction ⇒ Object 



206
207
208
209
210
211
212
213
214
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 206

def start_transaction
    return unless transactions_enabled?
    curr[:transaction_nesting] += 1
    return savepoint("point#{curr[:savepoints].length}") if in_transaction?

    Spider.logger.debug("#{self.class.name} starting transaction for connection #{connection.object_id}")
    do_start_transaction
    return true
end

#supports?(capability) ⇒ Boolean

Returns:

  • (Boolean) 


104
105
106
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 104

def supports?(capability)
    self.class.supports?(capability)
end

#supports_transactions? ⇒ Boolean

Returns:

  • (Boolean) 


198
199
200
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 198

def supports_transactions?
    return self.class.supports?(:transactions)
end

#transactions_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


202
203
204
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 202

def transactions_enabled?
    @configuration['enable_transactions'] && supports_transactions?
end

#update_sequence(name, val) ⇒ Object 



317
318
319
320
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 317

def update_sequence(name, val)
    # not an alias because the set value behaviour of next_sequence isn't expected in subclasses
    sequence_next(name, val)
end

#value_for_condition(type, value) ⇒ Object

Prepares a value that will be used in a condition.



178
179
180
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 178

def value_for_condition(type, value)
    return prepare_value(type, value)
end

#value_for_save(type, value, save_mode) ⇒ Object

Prepares a value for saving.



173
174
175
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 173

def value_for_save(type, value, save_mode)
    return prepare_value(type, value)
end

#value_to_mapper(type, value) ⇒ Object 



182
183
184
 
# File 'lib/spiderfw/model/storage/base_storage.rb', line 182

def value_to_mapper(type, value)
    value
end