Class: Bitcoin::Storage::Backends::SequelStore

Inherits:
StoreBase
  • Object
show all
Defined in:
lib/bitcoin/storage/sequel/sequel_store.rb

Overview

Storage backend using Sequel to connect to arbitrary SQL databases. Inherits from StoreBase and implements its interface.

Constant Summary collapse

DEFAULT_CONFIG = 
{ mode: :full, cache_head: false }

Constants inherited from StoreBase

Bitcoin::Storage::Backends::StoreBase::MAIN, Bitcoin::Storage::Backends::StoreBase::ORPHAN, Bitcoin::Storage::Backends::StoreBase::SCRIPT_TYPES, Bitcoin::Storage::Backends::StoreBase::SEQUEL_ADAPTERS, Bitcoin::Storage::Backends::StoreBase::SIDE

Instance Attribute Summary collapse

Attributes inherited from StoreBase

#config, #log

Instance Method Summary collapse

Methods inherited from StoreBase

#add_watched_address, #backend_name, #check_metadata, #get_balance, #get_locator, #get_txouts_for_address, #import, #in_sync?, #init_sequel_store, #migrate, #new_block, #new_tx, #rescan, #sqlite_pragmas, #store_addr, #store_block, #update_block

Constructor Details

#initialize(config, *args) ⇒ SequelStore

create sequel store with given config



18
19
20
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 18

def initialize config, *args
  super config, *args
end

Instance Attribute Details

#db ⇒ Object

sequel database connection



13
14
15
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 13

def db
  @db
end

Instance Method Details

#check_consistency(count = 1000) ⇒ Object

check data consistency of the top count blocks. validates that

  • the block hash computed from the stored data is the same

  • the prev_hash is the same as the previous blocks’ hash

  • the merkle root computed from all transactions is correct



416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 416

def check_consistency count = 1000
  return  if get_depth < 1 || count <= 0
  depth = get_depth
  count = depth - 1  if count == -1
  count = depth - 1  if count >= depth
  log.info { "Checking consistency of last #{count} blocks..." }
  prev_blk = get_block_by_depth(depth - count - 1)
  (depth - count).upto(depth).each do |depth|
    blk = get_block_by_depth(depth)
    raise "Block hash #{blk.depth} invalid!"  unless blk.hash == blk.recalc_block_hash
    raise "Prev hash #{blk.depth} invalid!"  unless blk.prev_block.reverse.hth == prev_blk.hash
    raise "Merkle root #{blk.depth} invalid!"  unless blk.verify_mrkl_root
    print "#{blk.hash} #{blk.depth} OK\r"
    prev_blk = blk
  end
  log.info { "Last #{count} blocks are consistent." }
end

#connect ⇒ Object

connect to database



23
24
25
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 23

def connect
  super
end

#delete_tx(hash) ⇒ Object

delete transaction TODO: also delete blk_tx mapping



210
211
212
213
214
215
216
217
218
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 210

def delete_tx(hash)
  log.debug { "Deleting tx #{hash} since all its outputs are spent" }
  @db.transaction do
    tx = get_tx(hash)
    tx.in.each {|i| @db[:txin].where(:id => i.id).delete }
    tx.out.each {|o| @db[:txout].where(:id => o.id).delete }
    @db[:tx].where(:id => tx.id).delete
  end
end

#get_block(blk_hash) ⇒ Object

get block for given blk_hash



251
252
253
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 251

def get_block(blk_hash)
  wrap_block(@db[:blk][:hash => blk_hash.htb.blob])
end

#get_block_by_depth(depth) ⇒ Object

get block by given depth



256
257
258
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 256

def get_block_by_depth(depth)
  wrap_block(@db[:blk][:depth => depth, :chain => MAIN])
end

#get_block_by_id(block_id) ⇒ Object

get block by given id



275
276
277
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 275

def get_block_by_id(block_id)
  wrap_block(@db[:blk][:id => block_id])
end

#get_block_by_prev_hash(prev_hash) ⇒ Object

get block by given prev_hash



261
262
263
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 261

def get_block_by_prev_hash(prev_hash)
  wrap_block(@db[:blk][:prev_hash => prev_hash.htb.blob, :chain => MAIN])
end

#get_block_by_tx(tx_hash) ⇒ Object

get block by given tx_hash



266
267
268
269
270
271
272
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 266

def get_block_by_tx(tx_hash)
  tx = @db[:tx][:hash => tx_hash.htb.blob]
  return nil  unless tx
  parent = @db[:blk_tx][:tx_id => tx[:id]]
  return nil  unless parent
  wrap_block(@db[:blk][:id => parent[:blk_id]])
end

#get_depth ⇒ Object

get depth of MAIN chain



242
243
244
245
246
247
248
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 242

def get_depth
  depth = (@config[:cache_head] && @head) ? @head.depth :
    @depth = @db[:blk].filter(:chain => MAIN).order(:depth).last[:depth] rescue nil

  return -1  unless depth
  depth
end

#get_head ⇒ Object

get head block (highest block from the MAIN chain)



231
232
233
234
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 231

def get_head
  (@config[:cache_head] && @head) ? @head :
    @head = wrap_block(@db[:blk].filter(:chain => MAIN).order(:depth).last)
end

#get_head_hash ⇒ Object 



236
237
238
239
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 236

def get_head_hash
  (@config[:cache_head] && @head) ? @head.hash :
    @head = @db[:blk].filter(:chain => MAIN).order(:depth).last[:hash].hth
end

#get_idx_from_tx_hash(tx_hash) ⇒ Object

Grab the position of a tx in a given block



336
337
338
339
340
341
342
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 336

def get_idx_from_tx_hash(tx_hash)
  tx = @db[:tx][:hash => tx_hash.htb.blob]
  return nil  unless tx
  parent = @db[:blk_tx][:tx_id => tx[:id]]
  return nil  unless parent
  return parent[:idx]
end

#get_tx(tx_hash) ⇒ Object

get transaction for given tx_hash



280
281
282
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 280

def get_tx(tx_hash)
  wrap_tx(@db[:tx][:hash => tx_hash.htb.blob])
end

#get_tx_by_id(tx_id) ⇒ Object

get transaction by given tx_id



285
286
287
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 285

def get_tx_by_id(tx_id)
  wrap_tx(@db[:tx][:id => tx_id])
end

#get_txin_for_txout(tx_hash, txout_idx) ⇒ Object

get corresponding Models::TxIn for the txout in transaction tx_hash with index txout_idx



291
292
293
294
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 291

def get_txin_for_txout(tx_hash, txout_idx)
  tx_hash = tx_hash.htb_reverse.blob
  wrap_txin(@db[:txin][:prev_out => tx_hash, :prev_out_index => txout_idx])
end

#get_txout_by_id(txout_id) ⇒ Object 



296
297
298
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 296

def get_txout_by_id(txout_id)
  wrap_txout(@db[:txout][:id => txout_id])
end

#get_txout_for_txin(txin) ⇒ Object

get corresponding Models::TxOut for txin



301
302
303
304
305
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 301

def get_txout_for_txin(txin)
  tx = @db[:tx][:hash => txin.prev_out.reverse.blob]
  return nil  unless tx
  wrap_txout(@db[:txout][:tx_idx => txin.prev_out_index, :tx_id => tx[:id]])
end

#get_txouts_for_hash160(hash160, unconfirmed = false) ⇒ Object

get all Models::TxOut matching given hash160



314
315
316
317
318
319
320
321
322
323
324
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 314

def get_txouts_for_hash160(hash160, unconfirmed = false)
  addr = @db[:addr][:hash160 => hash160]
  return []  unless addr
  txouts = @db[:addr_txout].where(:addr_id => addr[:id])
    .map{|t| @db[:txout][:id => t[:txout_id]] }
    .map{|o| wrap_txout(o) }
  unless unconfirmed
    txouts.select!{|o| @db[:blk][:id => o.get_tx.blk_id][:chain] == MAIN rescue false }
  end
  txouts
end

#get_txouts_for_name_hash(hash) ⇒ Object 



326
327
328
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 326

def get_txouts_for_name_hash(hash)
  @db[:names].filter(hash: hash).map {|n| get_txout_by_id(n[:txout_id]) }
end

#get_txouts_for_pk_script(script) ⇒ Object

get all Models::TxOut matching given script



308
309
310
311
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 308

def get_txouts_for_pk_script(script)
  txouts = @db[:txout].filter(:pk_script => script.blob).order(:id)
  txouts.map{|txout| wrap_txout(txout)}
end

#get_unconfirmed_tx ⇒ Object

get all unconfirmed Models::TxOut



331
332
333
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 331

def get_unconfirmed_tx
  @db[:unconfirmed].map{|t| wrap_tx(t)}
end

#has_block(blk_hash) ⇒ Object

check if block blk_hash exists



221
222
223
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 221

def has_block(blk_hash)
  !!@db[:blk].where(:hash => blk_hash.htb.blob).get(1)
end

#has_tx(tx_hash) ⇒ Object

check if transaction tx_hash exists



226
227
228
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 226

def has_tx(tx_hash)
  !!@db[:tx].where(:hash => tx_hash.htb.blob).get(1)
end

#parse_script(txout, i) ⇒ Object

parse script and collect address/txout mappings to index



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 110

def parse_script txout, i
  addrs, names = [], []

  script = Bitcoin::Script.new(txout.pk_script) rescue nil
  if script
    if script.is_hash160? || script.is_pubkey?
      addrs << [i, script.get_hash160]
    elsif script.is_multisig?
      script.get_multisig_pubkeys.map do |pubkey|
        addrs << [i, Bitcoin.hash160(pubkey.unpack("H*")[0])]
      end
    elsif Bitcoin.namecoin? && script.is_namecoin?
      addrs << [i, script.get_hash160]
      names << [i, script]
    else
      log.warn { "Unknown script type"}# #{tx.hash}:#{txout_idx}" }
    end
    script_type = SCRIPT_TYPES.index(script.type)
  else
    log.error { "Error parsing script"}# #{tx.hash}:#{txout_idx}" }
    script_type = SCRIPT_TYPES.index(:unknown)
  end
  [script_type, addrs, names]
end

#persist_addrs(addrs) ⇒ Object

bulk-store addresses and txout mappings



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 136

def persist_addrs addrs
  addr_txouts, new_addrs = [], []
  addrs.group_by {|_, a| a }.each do |hash160, txouts|
    if existing = @db[:addr][:hash160 => hash160]
      txouts.each {|id, _| addr_txouts << [existing[:id], id] }
    else
      new_addrs << [hash160, txouts.map {|id, _| id }]
    end
  end
  new_addr_ids = @db[:addr].insert_multiple(new_addrs.map {|hash160, txout_id|
    { hash160: hash160 } })
  new_addr_ids.each.with_index do |addr_id, idx|
    new_addrs[idx][1].each do |txout_id|
      addr_txouts << [addr_id, txout_id]
    end
  end
  @db[:addr_txout].insert_multiple(addr_txouts.map {|addr_id, txout_id|
    { addr_id: addr_id, txout_id: txout_id }})
end

#persist_block(blk, chain, depth, prev_work = 0) ⇒ Object

persist given block blk to storage.



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 34

def persist_block blk, chain, depth, prev_work = 0
  @db.transaction do
    attrs = {
      :hash => blk.hash.htb.blob,
      :depth => depth,
      :chain => chain,
      :version => blk.ver,
      :prev_hash => blk.prev_block.reverse.blob,
      :mrkl_root => blk.mrkl_root.reverse.blob,
      :time => blk.time,
      :bits => blk.bits,
      :nonce => blk.nonce,
      :blk_size => blk.to_payload.bytesize,
      :work => (prev_work + blk.block_work).to_s
    }
    attrs[:aux_pow] = blk.aux_pow.to_payload.blob  if blk.aux_pow
    existing = @db[:blk].filter(:hash => blk.hash.htb.blob)
    if existing.any?
      existing.update attrs
      block_id = existing.first[:id]
    else
      block_id = @db[:blk].insert(attrs)
      blk_tx, new_tx, addrs, names = [], [], [], []

      # store tx
      blk.tx.each.with_index do |tx, idx|
        existing = @db[:tx][hash: tx.hash.htb.blob]
        existing ? blk_tx[idx] = existing[:id] : new_tx << [tx, idx]
      end
      new_tx_ids = @db[:tx].insert_multiple(new_tx.map {|tx, _| tx_data(tx) })
      new_tx_ids.each.with_index {|tx_id, idx| blk_tx[new_tx[idx][1]] = tx_id }

      @db[:blk_tx].insert_multiple(blk_tx.map.with_index {|id, idx|
        { blk_id: block_id, tx_id: id, idx: idx } })

      # store txins
      txin_ids = @db[:txin].insert_multiple(new_tx.map.with_index {|tx, tx_idx|
        tx, _ = *tx
        tx.in.map.with_index {|txin, txin_idx|
          txin_data(new_tx_ids[tx_idx], txin, txin_idx) } }.flatten)

      # store txouts
      txout_i = 0
      txout_ids = @db[:txout].insert_multiple(new_tx.map.with_index {|tx, tx_idx|
        tx, _ = *tx
        tx.out.map.with_index {|txout, txout_idx|
          script_type, a, n = *parse_script(txout, txout_i)
          addrs += a; names += n; txout_i += 1
          txout_data(new_tx_ids[tx_idx], txout, txout_idx, script_type) } }.flatten)

      # store addrs
      persist_addrs addrs.map {|i, h| [txout_ids[i], h]}
      names.each {|i, script| store_name(script, txout_ids[i]) }
    end
    @head = wrap_block(attrs.merge(id: block_id))  if chain == MAIN
    @db[:blk].where(:prev_hash => blk.hash.htb.blob, :chain => ORPHAN).each do |b|
      log.debug { "connecting orphan #{b[:hash].hth}" }
      begin
        store_block(get_block(b[:hash].hth))
      rescue SystemStackError
        EM.defer { store_block(get_block(b[:hash].hth)) }  if EM.reactor_running?
      end
    end
    return depth, chain
  end
end

#reorg(new_side, new_main) ⇒ Object 



101
102
103
104
105
106
107
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 101

def reorg new_side, new_main
  @db.transaction do
    @db[:blk].where(hash: new_side.map {|h| h.htb.blob }).update(chain: SIDE)
    new_main.each {|b| get_block(b).validator(self).validate(raise_errors: true) }  unless @config[:skip_validation]
    @db[:blk].where(hash: new_main.map {|h| h.htb.blob }).update(chain: MAIN)
  end
end

#reset ⇒ Object

reset database; delete all data



28
29
30
31
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 28

def reset
  [:blk, :blk_tx, :tx, :txin, :txout, :addr, :addr_txout, :names].each {|table| @db[table].delete }
  @head = nil
end

#store_tx(tx, validate = true) ⇒ Object

store transaction tx



165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 165

def store_tx(tx, validate = true)
  @log.debug { "Storing tx #{tx.hash} (#{tx.to_payload.bytesize} bytes)" }
  tx.validator(self).validate(raise_errors: true)  if validate
  @db.transaction do
    transaction = @db[:tx][:hash => tx.hash.htb.blob]
    return transaction[:id]  if transaction
    tx_id = @db[:tx].insert(tx_data(tx))
    tx.in.each_with_index {|i, idx| store_txin(tx_id, i, idx)}
    tx.out.each_with_index {|o, idx| store_txout(tx_id, o, idx)}
    tx_id
  end
end

#store_txin(tx_id, txin, idx) ⇒ Object

store input txin



188
189
190
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 188

def store_txin(tx_id, txin, idx)
  @db[:txin].insert(txin_data(tx_id, txin, idx))
end

#store_txout(tx_id, txout, idx) ⇒ Object

store output txout



200
201
202
203
204
205
206
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 200

def store_txout(tx_id, txout, idx)
  script_type, addrs, names = *parse_script(txout, idx)
  txout_id = @db[:txout].insert(txout_data(tx_id, txout, idx, script_type))
  persist_addrs addrs.map {|i, h| [txout_id, h] }
  names.each {|i, script| store_name(script, txout_id) }
  txout_id
end

#tx_data(tx) ⇒ Object

prepare transaction data for storage



157
158
159
160
161
162
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 157

def tx_data tx
  { hash: tx.hash.htb.blob,
    version: tx.ver, lock_time: tx.lock_time,
    coinbase: tx.in.size == 1 && tx.in[0].coinbase?,
    tx_size: tx.payload.bytesize }
end

#txin_data(tx_id, txin, idx) ⇒ Object

prepare txin data for storage



179
180
181
182
183
184
185
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 179

def txin_data tx_id, txin, idx
  { tx_id: tx_id, tx_idx: idx,
    script_sig: txin.script_sig.blob,
    prev_out: txin.prev_out.blob,
    prev_out_index: txin.prev_out_index,
    sequence: txin.sequence.unpack("V")[0] }
end

#txout_data(tx_id, txout, idx, script_type) ⇒ Object

prepare txout data for storage



193
194
195
196
197
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 193

def txout_data tx_id, txout, idx, script_type
  { tx_id: tx_id, tx_idx: idx,
    pk_script: txout.pk_script.blob,
    value: txout.value, type: script_type }
end

#wrap_block(block) ⇒ Object

wrap given block into Models::Block



345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 345

def wrap_block(block)
  return nil  unless block

  data = {:id => block[:id], :depth => block[:depth], :chain => block[:chain], :work => block[:work].to_i, :hash => block[:hash].hth}
  blk = Bitcoin::Storage::Models::Block.new(self, data)

  blk.ver = block[:version]
  blk.prev_block = block[:prev_hash].reverse
  blk.mrkl_root = block[:mrkl_root].reverse
  blk.time = block[:time].to_i
  blk.bits = block[:bits]
  blk.nonce = block[:nonce]

  blk.aux_pow = Bitcoin::P::AuxPow.new(block[:aux_pow])  if block[:aux_pow]

  db[:blk_tx].filter(blk_id: block[:id]).join(:tx, id: :tx_id)
    .order(:idx).each {|tx| blk.tx << wrap_tx(tx, block[:id]) }

  blk.recalc_block_hash
  blk
end

#wrap_tx(transaction, block_id = nil) ⇒ Object

wrap given transaction into Models::Transaction



368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 368

def wrap_tx(transaction, block_id = nil)
  return nil  unless transaction

  block_id ||= @db[:blk_tx].join(:blk, id: :blk_id)
    .where(tx_id: transaction[:id], chain: 0).first[:blk_id] rescue nil

  data = {id: transaction[:id], blk_id: block_id}
  tx = Bitcoin::Storage::Models::Tx.new(self, data)

  inputs = db[:txin].filter(:tx_id => transaction[:id]).order(:tx_idx)
  inputs.each { |i| tx.add_in(wrap_txin(i)) }

  outputs = db[:txout].filter(:tx_id => transaction[:id]).order(:tx_idx)
  outputs.each { |o| tx.add_out(wrap_txout(o)) }
  tx.ver = transaction[:version]
  tx.lock_time = transaction[:lock_time]
  tx.hash = tx.hash_from_payload(tx.to_payload)
  tx
end

#wrap_txin(input) ⇒ Object

wrap given input into Models::TxIn



389
390
391
392
393
394
395
396
397
398
399
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 389

def wrap_txin(input)
  return nil  unless input
  data = {:id => input[:id], :tx_id => input[:tx_id], :tx_idx => input[:tx_idx]}
  txin = Bitcoin::Storage::Models::TxIn.new(self, data)
  txin.prev_out = input[:prev_out]
  txin.prev_out_index = input[:prev_out_index]
  txin.script_sig_length = input[:script_sig].bytesize
  txin.script_sig = input[:script_sig]
  txin.sequence = [input[:sequence]].pack("V")
  txin
end

#wrap_txout(output) ⇒ Object

wrap given output into Models::TxOut



402
403
404
405
406
407
408
409
410
 
# File 'lib/bitcoin/storage/sequel/sequel_store.rb', line 402

def wrap_txout(output)
  return nil  unless output
  data = {:id => output[:id], :tx_id => output[:tx_id], :tx_idx => output[:tx_idx],
    :hash160 => output[:hash160], :type => SCRIPT_TYPES[output[:type]]}
  txout = Bitcoin::Storage::Models::TxOut.new(self, data)
  txout.value = output[:value]
  txout.pk_script = output[:pk_script]
  txout
end