Class: Akane::Storages::Bigquery

Inherits:
AbstractStorage
  • Object
show all
Defined in:
lib/akane/storages/bigquery.rb

Defined Under Namespace

Classes: Stop

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Bigquery

Returns a new instance of Bigquery.



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/akane/storages/bigquery.rb', line 11

def initialize(*)
  super

  @client, @api = AkaneBigquery.make_bigquery_client(@config)

  @project_id = @config['project_id']
  @dataset_id = @config['dataset_id']

  @lock = Mutex.new
  @thread = nil

  @flush_interval = @config['flush_interval'] ? @config['flush_interval'].to_i : 60
  @flush_threshold = @config['flush_threshold'] ? @config['flush_threshold'].to_i : 1000

  @pending_inserts = []
  @failing_inserts = []
  @pending_inserts_lock = Mutex.new

  swap_buffers # initialize
  start
end

Instance Method Details

#bq_insert(table, row) ⇒ Object 



37
38
39
40
41
42
 
# File 'lib/akane/storages/bigquery.rb', line 37

def bq_insert(table, row)
  @lock.synchronize do
    @buffers[table] << row
  end
  self
end

#exitable? ⇒ Boolean

Returns:

  • (Boolean) 


53
54
55
 
# File 'lib/akane/storages/bigquery.rb', line 53

def exitable?
  @stop && (@thread ? @thread.alive? : true)
end

#mark_as_deleted(account, user_id, tweet_id) ⇒ Object 



109
110
111
112
113
114
115
116
117
 
# File 'lib/akane/storages/bigquery.rb', line 109

def mark_as_deleted(, user_id, tweet_id)
  bq_insert(:deletions,
    'user_id'.freeze => user_id,
    'user_id_str'.freeze => user_id.to_s,
    'tweet_id'.freeze => tweet_id,
    'tweet_id_str'.freeze => tweet_id.to_s,
    'deleted_at'.freeze => Time.now.to_i,
  )
end

#name ⇒ Object 



33
34
35
 
# File 'lib/akane/storages/bigquery.rb', line 33

def name
  @name ||= "bigquery:#{@project_id}/#{@dataset_id}"
end

#record_event(account, event) ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/akane/storages/bigquery.rb', line 119

def record_event(, event)
  source = event['source'.freeze]
  target = event['target'.freeze]
  target_object = event['target_object'.freeze]

  source_id = source[:id]
  target_id = target[:id]

  unless source_id && target_id
    @logger.warn "Discarding event because source and target id is missing: #{event.inspect}"
    return
  end

  hash = Hash[
    event.map { |k,v| [k, v && v.respond_to?(:attrs) ? v.attrs : nil] }
  ]

  row = {
    'json'.freeze => hash.to_json,
    'event'.freeze => event['event'.freeze],
    'source_id'.freeze => source_id,
    'source_id_str'.freeze => source_id.to_s,
    'target_id'.freeze => target_id,
    'target_id_str'.freeze => target_id.to_s,
    'created_at'.freeze => Time.now.to_i
  }

  if target_object && target_object[:id]
    id = target_object[:id]
    row['target_object_id'.freeze] = id
    row['target_object_id_str'.freeze] = id.to_s
  end

  p row
  bq_insert :events, row
end

#record_message(account, message) ⇒ Object 



156
157
 
# File 'lib/akane/storages/bigquery.rb', line 156

def record_message(, message)
end

#record_tweet(account, tweet) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/akane/storages/bigquery.rb', line 64

def record_tweet(, tweet)
  hash = tweet.attrs
  row = {
    'json'.freeze => hash.to_json,
    'id_str'.freeze => hash[:id_str],
    'id'.freeze => hash[:id],
    'text'.freeze => hash[:text],
    'lang'.freeze => hash[:lang],
    'source'.freeze => hash[:source],
    'in_reply_to_status_id'.freeze => hash[:in_reply_to_status_id],
    'in_reply_to_status_id_str'.freeze => hash[:in_reply_to_status_id_str],
    'in_reply_to_user_id'.freeze => hash[:in_reply_to_user_id],
    'in_reply_to_user_id_str'.freeze => hash[:in_reply_to_user_id_str],
    'in_reply_to_screen_name'.freeze => hash[:in_reply_to_screen_name],
    'user'.freeze => {
      'id_str'.freeze => hash[:user][:id_str],
      'id'.freeze => hash[:user][:id],
      'name'.freeze => hash[:user][:name],
      'screen_name'.freeze => hash[:user][:screen_name],
      'protected'.freeze => hash[:user][:protected],
    },
    'created_at'.freeze => Time.parse(hash[:created_at]).to_i
  }

  if hash['coordinates'.freeze]
    row['coordinates_longitude'.freeze], row['coordinates_latitude'.freeze] = \
      hash[:coordinates][:coordinates]
  end

  if hash[:place]
    place = hash[:place]
    row['place'.freeze] = {
      'id'.freeze => place[:id],
      'country'.freeze => place[:country],
      'country_code'.freeze => place[:country_code],
      'name'.freeze => place[:name],
      'full_name'.freeze => place[:full_name],
      'place_type'.freeze => place[:place_type],
      'url'.freeze => place[:url],
    }
  end

  bq_insert :tweets, row
end

#start ⇒ Object 



44
45
46
47
48
49
50
51
 
# File 'lib/akane/storages/bigquery.rb', line 44

def start
  @lock.synchronize do
    unless @thread
      @thread = Thread.new(&method(:worker_loop))
      @stop = false
    end
  end
end

#status ⇒ Object 



159
160
161
 
# File 'lib/akane/storages/bigquery.rb', line 159

def status
  @buffers ? @buffers.map{ |table, buf| "#{table}=#{buf.size}" }.join(', ') + " | #{@failing_inserts.size} failures, #{@pending_inserts.size} inserts" : "-"
end

#stop! ⇒ Object 



57
58
59
60
61
62
 
# File 'lib/akane/storages/bigquery.rb', line 57

def stop!
  @lock.synchronize do
    super
    @thread.raise(Stop) if @thread
  end
end