Class: OpencBot::ManualIncrementer

Inherits:
BaseIncrementer show all
Includes:
ScraperWiki
Defined in:
lib/openc_bot/incrementers/base.rb

Constant Summary collapse

ITEMS_TABLE = 
"items"

Instance Method Summary collapse

Methods inherited from BaseIncrementer

#db_name, #each, #log_progress, new, #position_file_name, #progress_percent, #read_current, #reset_current, #resumable, #write_current

Constructor Details

#initialize(name, opts = {}) ⇒ ManualIncrementer

Returns a new instance of ManualIncrementer.



115
116
117
118
119
120
121
122
123
 
# File 'lib/openc_bot/incrementers/base.rb', line 115

def initialize(name, opts={})
  super(name, opts)
  raise "Fields must be defined for this Record" if opts[:fields].nil?
  query = "CREATE TABLE IF NOT EXISTS #{ITEMS_TABLE} (#{opts[:fields].join(',')}, _id INTEGER PRIMARY KEY)"
  sqlite_magic_connection.execute query
  query = "CREATE UNIQUE INDEX IF NOT EXISTS #{opts[:fields].join('_')} " +
    "ON #{ITEMS_TABLE} (#{opts[:fields].join(',')})"
  sqlite_magic_connection.execute query
end

Instance Method Details

#add_row(val) ⇒ Object 



163
164
165
166
 
# File 'lib/openc_bot/incrementers/base.rb', line 163

def add_row(val)
  sqlite_magic_connection.insert_or_update(
    val.keys, val, ITEMS_TABLE, :update_unique_keys => true)
end

#count_all_items ⇒ Object 



181
182
183
184
185
186
 
# File 'lib/openc_bot/incrementers/base.rb', line 181

def count_all_items
  begin
    select("count(*) as count FROM #{ITEMS_TABLE}").first['count']
  rescue SqliteMagic::NoSuchTable
  end
end

#count_processed_items(start_id) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/openc_bot/incrementers/base.rb', line 168

def count_processed_items(start_id)
  if start_id
    begin
      result = select("count(*) as count FROM #{ITEMS_TABLE} WHERE _id < #{start_id}").first
      result && result['count']
    rescue SqliteMagic::NoSuchTable
      0
    end
  else
    0
  end
end

#enum(*args) ⇒ Object 



158
159
160
161
 
# File 'lib/openc_bot/incrementers/base.rb', line 158

def enum(*args)
  self.populated = true
  each
end

#increment_yielder(start_row = nil) ⇒ Object 



131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/openc_bot/incrementers/base.rb', line 131

def increment_yielder(start_row=nil)
  start_id = start_row && start_row["_id"].to_i
  @expected_count = count_all_items
  @count = count_processed_items(start_id)
  loop do
    result = read_batch(start_id).each do |row|
      yield row
      start_id = row["_id"].to_i + 1
    end
    raise StopIteration if result.empty?
  end
end

#populated ⇒ Object 



144
145
146
147
148
149
150
 
# File 'lib/openc_bot/incrementers/base.rb', line 144

def populated
  begin
    result = select("populated FROM misc").first['populated']
    result && result == "true"
  rescue SqliteMagic::NoSuchTable
  end
end

#populated=(val) ⇒ Object 



152
153
154
155
156
 
# File 'lib/openc_bot/incrementers/base.rb', line 152

def populated=(val)
  if val && val == "true" || val == true
    save_sqlite([:populated], {:populated => "true"}, "misc")
  end
end

#read_batch(start_id = nil) ⇒ Object 



188
189
190
191
192
193
194
195
 
# File 'lib/openc_bot/incrementers/base.rb', line 188

def read_batch(start_id=nil)
  sql = "* FROM #{ITEMS_TABLE}"
  if start_id
    sql += " WHERE _id >= #{start_id}"
  end
  sql += " LIMIT 100"
  select(sql)
end

#resuming_enum(enum) ⇒ Object

override superclass definition for more efficient version



198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
 
# File 'lib/openc_bot/incrementers/base.rb', line 198

def resuming_enum(enum)
  current_row = read_current && read_current != "" && JSON.parse(read_current)
  if current_row
    enum = Enumerator.new do |yielder|
      increment_yielder(current_row) do |result|
        write_current(result.to_json)
        yielder.yield(result)
        @count += 1
        log_progress(progress_percent)
      end
      reset_current
    end.lazy
  end
  enum
end

#single_transaction {|_self| ... } ⇒ Object

Yields:

  • (_self)

Yield Parameters:



109
110
111
112
113
 
# File 'lib/openc_bot/incrementers/base.rb', line 109

def single_transaction
  sqlite_magic_connection.execute("BEGIN TRANSACTION")
  yield(self)
  sqlite_magic_connection.execute("COMMIT")
end

#sqlite_magic_connection ⇒ Object

Override default in ScraperWiki gem



126
127
128
129
 
# File 'lib/openc_bot/incrementers/base.rb', line 126

def sqlite_magic_connection
  db = File.expand_path(File.join(@app_path, 'db', "#{db_name}.db"))
  @sqlite_magic_connection ||= SqliteMagic::Connection.new(db)
end