Class: Spider::Model::Storage::Db::Mysql

Inherits:
DbStorage show all
Defined in:
lib/spiderfw/model/storage/db/adapters/mysql.rb

Defined Under Namespace

Modules: MapperExtension

Class Attribute Summary collapse

Attributes inherited from BaseStorage

#instance_name, #url

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from DbStorage

#alter_table, #column_name, #create_table, #drop_field, #drop_table, #dump, #foreign_key_name, #get_mapper, inherited, #initialize, #lock, #parse_db_column, #query, #query_finished, #query_start, #reflect_column, #safe_schema_conversion?, #schema_field_equal?, #sequence_name, #shorten_identifier, #sql_add_field, #sql_alter_field, #sql_alter_table, #sql_condition, #sql_condition_value, #sql_create_primary_key, #sql_delete, #sql_drop_field, #sql_drop_foreign_key, #sql_drop_primary_key, #sql_drop_table, #sql_insert, #sql_joins, #sql_keys, #sql_limit, #sql_max, #sql_order, #sql_tables, #sql_tables_join, #sql_truncate, #sql_update, #sql_update_values, storage_type

Methods inherited from BaseStorage

#==, #commit, #commit!, #commit_or_continue, #connect, #connected?, #connection, #connection_attributes, connection_attributes, #connection_pool, connection_pools, #create_sequence, #curr, #generate_uuid, get_connection, #get_mapper, #in_transaction, inherited, #initialize, max_connections, release_connection, remove_connection, #rollback, #rollback!, #sequence_exists?, #sequence_file_path, #sequence_next, sequence_sync, #start_transaction, storage_type, #supports?, supports?, #supports_transactions?, #transactions_enabled?, #update_sequence, #value_for_condition, #value_for_save

Methods included from Logger

add, close, close_all, datetime_format, datetime_format=, #debug, debug, debug?, #debug?, enquire_loggers, #error, error, #error?, error?, #fatal, fatal, #fatal?, fatal?, info, #info, info?, #info?, method_missing, open, reopen, send_to_loggers, unknown, #unknown, #warn, warn, warn?, #warn?

Constructor Details

This class inherits a constructor from Spider::Model::Storage::Db::DbStorage

Class Attribute Details

.field_flags ⇒ Object (readonly)

Returns the value of attribute field_flags.



74
75
76
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 74

def field_flags
  @field_flags
end

.field_types ⇒ Object (readonly)

Returns the value of attribute field_types.



74
75
76
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 74

def field_types
  @field_types
end

.reserved_kewords ⇒ Object (readonly)

Returns the value of attribute reserved_kewords.



74
75
76
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 74

def reserved_kewords
  @reserved_kewords
end

.safe_conversions ⇒ Object (readonly)

Returns the value of attribute safe_conversions.



74
75
76
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 74

def safe_conversions
  @safe_conversions
end

.type_synonyms ⇒ Object (readonly)

Returns the value of attribute type_synonyms.



74
75
76
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 74

def type_synonyms
  @type_synonyms
end

Class Method Details

.base_types ⇒ Object 



8
9
10
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 8

def self.base_types
    super << Spider::DataTypes::Binary
end

.connection_alive?(conn) ⇒ Boolean

Returns:

  • (Boolean) 


104
105
106
107
108
109
110
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 104

def self.connection_alive?(conn)
    begin
        return conn.ping
    rescue
        return false
    end 
end

.disconnect(conn) ⇒ Object 



83
84
85
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 83

def self.disconnect(conn)
    conn.close
end

.new_connection(host = nil, user = nil, passwd = nil, db = nil, port = nil, sock = nil, flag = nil) ⇒ Object 



76
77
78
79
80
81
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 76

def self.new_connection(host=nil, user=nil, passwd=nil, db=nil, port=nil, sock=nil, flag=nil)
    conn = ::Mysql.new(host, user, passwd, db, port, sock, flag)
    conn.autocommit(true)
    conn.query("SET NAMES 'utf8'")
    return conn
end

.parse_url(url) ⇒ Object 



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 112

def self.parse_url(url)
    # db:mysql://<username:password>@<host>:<port>/<database>
    if (url =~ /.+:\/\/(?:(.+):(.+)@)?(.+)?\/(.+)/)
        user = $1
        pass = $2
        location = $3
        db_name = $4
    else
        raise ArgumentError, "Mysql url '#{url}' is invalid"
    end
    if (location =~ /localhost:(\/.+)/)
        host = 'localhost'
        sock = $1
    else
        location =~ /(.+)(?::(\d+))/
        host = $1
        port = $2
    end
    return [host, user, pass, db_name, port, sock]
end

Instance Method Details

#column_attributes(type, attributes) ⇒ Object 



481
482
483
484
485
486
487
488
489
490
491
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 481

def column_attributes(type, attributes)
    db_attributes = super(type, attributes)
    case type.name
    when 'String'
        db_attributes[:length] = attributes[:length] || 255
    when 'Fixnum'
        db_attributes[:length] = 11
    end
    db_attributes[:autoincrement] = false if attributes[:autoincrement] && !attributes[:primary_key]
    return db_attributes
end

#column_type(type, attributes) ⇒ Object 



456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 456

def column_type(type, attributes)
    case type.name
    when 'String'
        'VARCHAR'
    when 'Spider::DataTypes::Text'
        'TEXT'
    when 'Fixnum'
        'INT'
    when 'Float'
        'FLOAT'
    when 'Date'
        'DATE'
    when 'DateTime'
        'DATETIME'
    when 'Time'
        'TIME'
    when 'Spider::DataTypes::Binary'
        'BLOB'
    when 'Spider::DataTypes::Bool'
        'TINYINT'
    when 'BigDecimal', 'Spider::DataTypes::Decimal'
        'DECIMAL'
    end
end

#configure(conf) ⇒ Object 



87
88
89
90
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 87

def configure(conf)
    super
    @configuration['default_engine'] ||= Spider.conf.get('db.mysql.default_engine')
end

#describe_table(table) ⇒ Object 



336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 336

def describe_table(table)
    columns = {}
    primary_keys = []
    foreign_keys = []
    order = []
    connection do |c|
        res = c.query("select * from #{table} where 1=0")
        fields = res.fetch_fields
        fields.each do |f|
            type =  self.class.field_types[f.type]
            length = f.length;
            length /= 3 if ['CHAR', 'VARCHAR'].include?(type)
            scale = nil
            precision = f.decimals
            # FIXME
            if (type == 'DECIMAL')
                scale = f.decimals
                precision = length - scale
                length = 0
            end
            col = {
                :type => type,
                :length => length,
                :precision => precision,
                :scale => scale
            }
            flags = f.flags
            self.class.field_flags.each do |flag_name, flag_val|
                col[flag_name] = (flags & flag_val == 0) ? false : true
            end
            columns[f.name] = col
            order << f.name
            primary_keys << f.name if f.is_pri_key?
        end                 
        res = c.query("select * from INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE constraint_schema = '#{@db_name}' and table_name = '#{table}'")
        while h = res.fetch_hash
            fk_table = h['REFERENCED_TABLE_NAME']
            if fk_table
                fk_fields1 = h['COLUMN_NAME'].split(',')
                fk_fields2 = h['REFERENCED_COLUMN_NAME'].split(',')
                fk_name = h['CONSTRAINT_NAME']
                fk_fields = {}
                fk_fields1.each_index{ |i| fk_fields[fk_fields1[i]] = fk_fields2[i] }
                foreign_keys << ForeignKeyConstraint.new(fk_name, fk_table, fk_fields)
            end
        end
        
    end
    return {:columns => columns, :order => order, :primary_keys => primary_keys, :foreign_key_constraints => foreign_keys}
end

#do_commit ⇒ Object 



153
154
155
156
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 153

def do_commit
    curr[:conn].commit if curr[:conn]
    curr[:in_transaction] = false
end

#do_rollback ⇒ Object 



158
159
160
161
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 158

def do_rollback
    curr[:conn].rollback if curr[:conn]
    curr[:in_transaction] = false
end

#do_start_transaction ⇒ Object 



138
139
140
141
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 138

def do_start_transaction
    connection.autocommit(false)
    curr[:in_transaction] = true
end

#dump_table_data(table, stream) ⇒ Object 



409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 409

def dump_table_data(table, stream)
    connection do |c|
        res = c.query("select * from #{table}")
        num = res.num_rows
        if num > 0
            fields = res.fetch_fields
            stream << "INSERT INTO `#{table}` (#{fields.map{ |f| "`#{f.name}`"}.join(', ')})\n"
            stream << "VALUES\n"
            cnt = 0
            while row = res.fetch_row
                cnt += 1
                stream << "("
                fields.each_with_index do |f, i|
                    stream << dump_value(row[i], f)
                    stream << ", " if i < fields.length - 1
                end
                stream << ")"
                if cnt < num
                    stream << ",\n"
                else
                    stream << ";\n"
                end
            end
            stream << "\n\n"
        end
    end
end

#dump_value(val, field) ⇒ Object 



437
438
439
440
441
442
443
444
445
446
447
448
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 437

def dump_value(val, field)
    return 'NULL' if val.nil?
    type =  self.class.field_types[field.type]
    if ['CHAR', 'VARCHAR', 'BLOB', 'TINY_BLOB', 'MEDIUM_BLOB', 'LONG_BLOB'].include?(type)
        val = val.gsub("'", "''").gsub("\n", '\n').gsub("\r", '\r')
        return "'#{val}'"
    elsif ['DATE', 'TIME', 'DATETIME'].include?(type)
        return "'#{val}'"
    else
        return val.to_s
    end
end

#execute(sql, *bind_vars) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 168

def execute(sql, *bind_vars)
   begin
       if (bind_vars && bind_vars.length > 0)
           debug_vars = bind_vars.map{|var| var = var.to_s; var && var.length > 50 ? var[0..50]+"...(#{var.length-50} chars more)" : var}
       end
       curr[:last_executed] = [sql, bind_vars]
       if (Spider.conf.get('storage.db.replace_debug_vars'))
           cnt = -1
           debug("mysql #{curr[:conn]} executing: "+sql.gsub('?'){ debug_vars[cnt+=1] })
       else
           debug_vars_str = debug_vars ? debug_vars.join(', ') : ''
           debug("mysql #{curr[:conn]} executing:\n#{sql}\n[#{debug_vars_str}]")
       end
       query_start
       stmt = connection.prepare(sql)
       curr[:stmt] = stmt
       res = stmt.execute(*bind_vars)
       have_result = (stmt.field_count == 0 ? false : true)
       if (have_result)
           result_meta = stmt.result_metadata
           fields = result_meta.fetch_fields
           result = []
           while (a = res.fetch)
               h = {}
               fields.each_index{ |i| h[fields[i].name] = a[i]}
               if block_given?
                   yield h
               else
                   result << h
               end
           end
           if (curr[:last_query_type] == :select)
               rows_res = connection.query("select FOUND_ROWS()")
               curr[:total_rows] = rows_res.fetch_row[0].to_i
           end
       end
       curr[:last_insert_id] = connection.insert_id
       curr[:last_query_type] = nil
       if (have_result)
           unless block_given?
               result.extend(StorageResult)
               return result
           end
       else
           return res
       end
   rescue => exc
       release
       if (exc.message =~ /Duplicate entry/)
           raise Spider::Model::Storage::DuplicateKey
       else
           raise exc
       end
   ensure
       query_finished
       release if curr[:conn] && !in_transaction?
   end
end

#execute_statement(stmt, *bind_vars) ⇒ Object 



232
233
234
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 232

def execute_statement(stmt, *bind_vars)
    stmt.execute(*bind_vars)
end

#function(func) ⇒ Object 



314
315
316
317
318
319
320
321
322
323
324
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 314

def function(func)
    return super unless func.func_name == :concat
    fields = func.elements.map{ |func_el|
        if (func_el.is_a?(Spider::QueryFuncs::Function))
            function(func_el)
        else
            func.mapper_fields[func_el]
        end
    }
    return "CONCAT(#{fields.map{ |f| "COALESCE(#{f}, '')" }.join(', ')})"
end

#get_table_create_sql(table) ⇒ Object 



399
400
401
402
403
404
405
406
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 399

def get_table_create_sql(table)
    sql = nil
    connection do |c|
        res = c.query("SHOW CREATE TABLE #{table}")
        sql = res.fetch_row[1]
    end
    sql
end

#in_transaction? ⇒ Boolean

Returns:

  • (Boolean) 


148
149
150
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 148

def in_transaction?
    return curr[:in_transaction] ? true : false
end

#last_insert_id ⇒ Object 



272
273
274
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 272

def last_insert_id
    curr[:last_insert_id]
end

#list_tables ⇒ Object

Methods to get information from the db #



330
331
332
333
334
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 330

def list_tables
    connection do |c|
        return c.list_tables
    end
end

#parse_url(url) ⇒ Object 



133
134
135
136
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 133

def parse_url(url)
    @host, @user, @pass, @db_name, @port, @sock = self.class.parse_url(url)
    @connection_params = [@host, @user, @pass, @db_name, @port, @sock]
end

#prepare(sql) ⇒ Object 



227
228
229
230
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 227

def prepare(sql)
    debug("mysql preparing: #{sql}")
    return curr[:stmt] = connection.prepare(sql)
end

#prepare_value(type, value) ⇒ Object 



240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 240

def prepare_value(type, value)
    value = super(type, value)
    return value unless value
    case type.name
    when 'String'
        return value.to_s
    when 'Date'
        return value.strftime("%Y-%m-%d")
    when 'DateTime'
        return value.strftime("%Y-%m-%dT%H:%M:%S")
    when 'Time'
        return value.strftime("%H:%M:%S")
    when 'Fixnum'
        return value.to_i
    end
    return value
end

#release ⇒ Object 



92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 92

def release
    begin
        #Spider::Logger.debug("MYSQL #{self.object_id} in thread #{Thread.current} releasing connection #{@conn}")
        @conn.autocommit(true) if @conn && !Spider.conf.get('storage.db.shared_connection')
        super
    rescue => exc
        Spider::Logger.error("MYSQL #{self.object_id} in thread #{Thread.current} exception #{exc.message} while trying to release connection #{@conn}")
        self.class.remove_connection(@conn, @connection_params)
        @conn = nil
    end
end

#rollback_savepoint(name = nil) ⇒ Object 



163
164
165
166
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 163

def rollback_savepoint(name=nil)
    connection.query("ROLLBACK TO #{name}")
    super
end

#savepoint(name) ⇒ Object 



143
144
145
146
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 143

def savepoint(name)
    connection.query("SAVEPOINT #{name}")
    super
end

#schema_field_date_equal?(current, field) ⇒ Boolean

Returns:

  • (Boolean) 


512
513
514
515
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 512

def schema_field_date_equal?(current, field)
    # FIXME
    return true
end

#schema_field_datetime_equal?(current, field) ⇒ Boolean

Returns:

  • (Boolean) 


517
518
519
520
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 517

def schema_field_datetime_equal?(current, field)
    # FIXME
    return true
end

#schema_field_float_equal?(current, field) ⇒ Boolean

Returns:

  • (Boolean) 


522
523
524
525
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 522

def schema_field_float_equal?(current, field)
    # FIXME
    return true
end

#schema_field_int_equal?(current, field) ⇒ Boolean

Returns:

  • (Boolean) 


502
503
504
505
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 502

def schema_field_int_equal?(current, field)
    # FIXME
    return true
end

#schema_field_text_equal?(current, field) ⇒ Boolean

Returns:

  • (Boolean) 


507
508
509
510
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 507

def schema_field_text_equal?(current, field)
    # FIXME
    return true
end

#schema_field_varchar_equal?(current, field) ⇒ Boolean

Returns:

  • (Boolean) 


527
528
529
530
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 527

def schema_field_varchar_equal?(current, field)
    # FIXME
    return true
end

#sql_create_table(create) ⇒ Object 



308
309
310
311
312
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 308

def sql_create_table(create)
    sqls = super
    sqls[0] += " ENGINE=#{@configuration['default_engine']}" if @configuration['default_engine']
    sqls
end

#sql_select(query) ⇒ Object 



281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 281

def sql_select(query)
    curr[:last_query_type] = :select
    bind_vars = query[:bind_vars] || []
    tables_sql, tables_values = sql_tables(query)
    sql = "SELECT "
    sql += "SQL_CALC_FOUND_ROWS " unless query[:query_type] == :count
    if query[:joins] && query[:joins].values.map{ |h| h.values }.flatten.select{ |v| v[:type] == :left}.length > 0
        sql += "DISTINCT "
    end
    sql += "#{sql_keys(query)} FROM #{tables_sql} "
    bind_vars += tables_values
    where, vals = sql_condition(query)
    bind_vars += vals
    sql += "WHERE #{where} " if where && !where.empty?
    order = sql_order(query)
    sql += "ORDER BY #{order} " if order && !order.empty?
    limit = sql_limit(query)
    sql += limit if limit
    return sql, bind_vars
end

#sql_table_field(name, type, attributes) ⇒ Object 



302
303
304
305
306
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 302

def sql_table_field(name, type, attributes)
    sql = super
    sql += " AUTO_INCREMENT" if attributes[:autoincrement]
    return sql
end

#table_exists?(table) ⇒ Boolean

Returns:

  • (Boolean) 


387
388
389
390
391
392
393
394
395
396
397
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 387

def table_exists?(table)
    begin
        connection do |c|
            c.query("select * from #{table} where 1=0")
        end
        Spider.logger.debug("TABLE EXISTS #{table}")
        return true
    rescue ::Mysql::Error
        return false
    end
end

#table_name(name) ⇒ Object

Schema methods



452
453
454
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 452

def table_name(name)
    super.downcase
end

#total_rows ⇒ Object 



236
237
238
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 236

def total_rows
    return curr[:total_rows]
end

#value_to_mapper(type, value) ⇒ Object 



258
259
260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/spiderfw/model/storage/db/adapters/mysql.rb', line 258

def value_to_mapper(type, value)
    return unless value
    case type.name
    when 'DateTime'
        @@time_offset ||= DateTime.now.offset
        return type.civil(value.year, value.month, value.day, value.hour, value.minute, value.second, @@time_offset)
    when 'Date'
        return type.civil(value.year, value.month, value.day)
    when 'Time'
        return type.local(2000, 1, 1, value.hour, value.minute, value.second)
    end
    return super(type, value)
end