Module: Blazer
- Defined in:
- lib/blazer.rb,
lib/blazer/engine.rb,
lib/blazer/result.rb,
lib/blazer/version.rb,
lib/blazer/data_source.rb,
app/models/blazer/audit.rb,
app/models/blazer/check.rb,
app/models/blazer/query.rb,
lib/blazer/excel_parser.rb,
app/models/blazer/record.rb,
lib/blazer/run_statement.rb,
app/models/blazer/dashboard.rb,
app/models/blazer/connection.rb,
lib/blazer/run_statement_job.rb,
app/helpers/blazer/base_helper.rb,
app/mailers/blazer/check_mailer.rb,
lib/blazer/adapters/sql_adapter.rb,
lib/blazer/adapters/base_adapter.rb,
app/mailers/blazer/slack_notifier.rb,
app/models/blazer/dashboard_query.rb,
lib/blazer/adapters/drill_adapter.rb,
lib/blazer/adapters/druid_adapter.rb,
lib/blazer/adapters/athena_adapter.rb,
lib/blazer/adapters/presto_adapter.rb,
lib/blazer/adapters/mongodb_adapter.rb,
lib/blazer/adapters/bigquery_adapter.rb,
lib/blazer/adapters/cassandra_adapter.rb,
lib/blazer/adapters/snowflake_adapter.rb,
app/controllers/blazer/base_controller.rb,
lib/generators/blazer/install_generator.rb,
app/controllers/blazer/checks_controller.rb,
app/controllers/blazer/queries_controller.rb,
lib/blazer/adapters/elasticsearch_adapter.rb,
app/controllers/blazer/dashboards_controller.rb
Defined Under Namespace
Modules: Adapters, BaseHelper, Generators Classes: Audit, BaseController, Check, CheckMailer, ChecksController, Connection, Dashboard, DashboardQuery, DashboardsController, DataSource, Engine, Error, ExcelParser, QueriesController, Query, Record, Result, RunStatement, RunStatementJob, SlackNotifier, TimeoutNotSupported
Constant Summary collapse
- TIMEOUT_MESSAGE =
"Query timed out :("
- TIMEOUT_ERRORS =
[ "canceling statement due to statement timeout", # postgres "canceling statement due to conflict with recovery", # postgres "cancelled on user's request", # redshift "canceled on user's request", # redshift "system requested abort", # redshift "maximum statement execution time exceeded" # mysql ]
- BELONGS_TO_OPTIONAL =
{}
- VERSION =
'2.0.5'
Class Attribute Summary collapse
-
.anomaly_checks ⇒ Object
Returns the value of attribute anomaly_checks.
-
.assignees ⇒ Object
Returns the value of attribute assignees.
-
.async ⇒ Object
Returns the value of attribute async.
-
.audit ⇒ Object
Returns the value of attribute audit.
-
.before_action ⇒ Object
Returns the value of attribute before_action.
-
.cache ⇒ Object
Returns the value of attribute cache.
-
.check_schedules ⇒ Object
Returns the value of attribute check_schedules.
-
.forecasting ⇒ Object
Returns the value of attribute forecasting.
-
.from_email ⇒ Object
Returns the value of attribute from_email.
-
.images ⇒ Object
Returns the value of attribute images.
-
.mapbox_access_token ⇒ Object
Returns the value of attribute mapbox_access_token.
-
.override_csp ⇒ Object
Returns the value of attribute override_csp.
-
.query_creatable ⇒ Object
Returns the value of attribute query_creatable.
-
.query_editable ⇒ Object
Returns the value of attribute query_editable.
-
.query_viewable ⇒ Object
Returns the value of attribute query_viewable.
-
.slack_webhook_url ⇒ Object
Returns the value of attribute slack_webhook_url.
-
.subdomain ⇒ Object
Returns the value of attribute subdomain.
-
.time_zone ⇒ Object
Returns the value of attribute time_zone.
-
.transform_statement ⇒ Object
Returns the value of attribute transform_statement.
-
.transform_variable ⇒ Object
Returns the value of attribute transform_variable.
- .user_class ⇒ Object
- .user_method ⇒ Object
-
.user_name ⇒ Object
Returns the value of attribute user_name.
Class Method Summary collapse
- .adapters ⇒ Object
- .data_sources ⇒ Object
- .extract_vars(statement) ⇒ Object
- .register_adapter(name, adapter) ⇒ Object
- .run_check(check) ⇒ Object
- .run_checks(schedule: nil) ⇒ Object
- .send_failing_after_checks(schedule: nil) ⇒ Object
- .send_failing_checks ⇒ Object
- .settings ⇒ Object
- .slack? ⇒ Boolean
Class Attribute Details
.anomaly_checks ⇒ Object
Returns the value of attribute anomaly_checks.
51 52 53 |
# File 'lib/blazer.rb', line 51 def anomaly_checks @anomaly_checks end |
.assignees ⇒ Object
Returns the value of attribute assignees.
50 51 52 |
# File 'lib/blazer.rb', line 50 def assignees @assignees end |
.async ⇒ Object
Returns the value of attribute async.
53 54 55 |
# File 'lib/blazer.rb', line 53 def async @async end |
.audit ⇒ Object
Returns the value of attribute audit.
37 38 39 |
# File 'lib/blazer.rb', line 37 def audit @audit end |
.before_action ⇒ Object
Returns the value of attribute before_action.
42 43 44 |
# File 'lib/blazer.rb', line 42 def before_action @before_action end |
.cache ⇒ Object
Returns the value of attribute cache.
45 46 47 |
# File 'lib/blazer.rb', line 45 def cache @cache end |
.check_schedules ⇒ Object
Returns the value of attribute check_schedules.
48 49 50 |
# File 'lib/blazer.rb', line 48 def check_schedules @check_schedules end |
.forecasting ⇒ Object
Returns the value of attribute forecasting.
52 53 54 |
# File 'lib/blazer.rb', line 52 def forecasting @forecasting end |
.from_email ⇒ Object
Returns the value of attribute from_email.
43 44 45 |
# File 'lib/blazer.rb', line 43 def from_email @from_email end |
.images ⇒ Object
Returns the value of attribute images.
54 55 56 |
# File 'lib/blazer.rb', line 54 def images @images end |
.mapbox_access_token ⇒ Object
Returns the value of attribute mapbox_access_token.
49 50 51 |
# File 'lib/blazer.rb', line 49 def mapbox_access_token @mapbox_access_token end |
.override_csp ⇒ Object
Returns the value of attribute override_csp.
57 58 59 |
# File 'lib/blazer.rb', line 57 def override_csp @override_csp end |
.query_creatable ⇒ Object
Returns the value of attribute query_creatable.
59 60 61 |
# File 'lib/blazer.rb', line 59 def query_creatable @query_creatable end |
.query_editable ⇒ Object
Returns the value of attribute query_editable.
56 57 58 |
# File 'lib/blazer.rb', line 56 def query_editable @query_editable end |
.query_viewable ⇒ Object
Returns the value of attribute query_viewable.
55 56 57 |
# File 'lib/blazer.rb', line 55 def query_viewable @query_viewable end |
.slack_webhook_url ⇒ Object
Returns the value of attribute slack_webhook_url.
58 59 60 |
# File 'lib/blazer.rb', line 58 def slack_webhook_url @slack_webhook_url end |
.subdomain ⇒ Object
Returns the value of attribute subdomain.
44 45 46 |
# File 'lib/blazer.rb', line 44 def subdomain @subdomain end |
.time_zone ⇒ Object
Returns the value of attribute time_zone.
38 39 40 |
# File 'lib/blazer.rb', line 38 def time_zone @time_zone end |
.transform_statement ⇒ Object
Returns the value of attribute transform_statement.
46 47 48 |
# File 'lib/blazer.rb', line 46 def transform_statement @transform_statement end |
.transform_variable ⇒ Object
Returns the value of attribute transform_variable.
47 48 49 |
# File 'lib/blazer.rb', line 47 def transform_variable @transform_variable end |
.user_class ⇒ Object
89 90 91 92 93 94 |
# File 'lib/blazer.rb', line 89 def self.user_class if !defined?(@user_class) @user_class = settings.key?("user_class") ? settings["user_class"] : (User.name rescue nil) end @user_class end |
.user_method ⇒ Object
96 97 98 99 100 101 102 103 104 |
# File 'lib/blazer.rb', line 96 def self.user_method if !defined?(@user_method) @user_method = settings["user_method"] if user_class @user_method ||= "current_#{user_class.to_s.downcase.singularize}" end end @user_method end |
.user_name ⇒ Object
Returns the value of attribute user_name.
39 40 41 |
# File 'lib/blazer.rb', line 39 def user_name @user_name end |
Class Method Details
.adapters ⇒ Object
222 223 224 |
# File 'lib/blazer.rb', line 222 def self.adapters @adapters ||= {} end |
.data_sources ⇒ Object
117 118 119 120 121 122 123 124 125 126 |
# File 'lib/blazer.rb', line 117 def self.data_sources @data_sources ||= begin ds = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" } settings["data_sources"].each do |id, s| ds[id] = Blazer::DataSource.new(id, s) end ds.default = ds.values.first ds end end |
.extract_vars(statement) ⇒ Object
128 129 130 131 132 |
# File 'lib/blazer.rb', line 128 def self.extract_vars(statement) # strip commented out lines # and regex {1} or {1,2} statement.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq end |
.register_adapter(name, adapter) ⇒ Object
226 227 228 |
# File 'lib/blazer.rb', line 226 def self.register_adapter(name, adapter) adapters[name] = adapter end |
.run_check(check) ⇒ Object
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 |
# File 'lib/blazer.rb', line 148 def self.run_check(check) tries = 1 ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument| # try 3 times on timeout errors data_source = data_sources[check.query.data_source] statement = check.query.statement Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement while tries <= 3 result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query) if result.timed_out? Rails.logger.info "[blazer timeout] query=#{check.query.name}" tries += 1 sleep(10) elsif result.error.to_s.start_with?("PG::ConnectionBad") data_source.reconnect Rails.logger.info "[blazer reconnect] query=#{check.query.name}" tries += 1 sleep(10) else break end end begin check.reload # in case state has changed since job started check.update_state(result) rescue ActiveRecord::RecordNotFound # check deleted end # TODO use proper logfmt Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}" instrument[:statement] = statement instrument[:data_source] = data_source instrument[:state] = check.state instrument[:rows] = result.rows.try(:size) instrument[:error] = result.error instrument[:tries] = tries end end |
.run_checks(schedule: nil) ⇒ Object
139 140 141 142 143 144 145 146 |
# File 'lib/blazer.rb', line 139 def self.run_checks(schedule: nil) checks = Blazer::Check.includes(:query) checks = checks.where(schedule: schedule) if schedule checks.find_each do |check| next if check.state == "disabled" Safely.safely { run_check(check) } end end |
.send_failing_after_checks(schedule: nil) ⇒ Object
134 135 136 137 |
# File 'lib/blazer.rb', line 134 def self.send_failing_after_checks(schedule: nil) run_checks(schedule) send_failing_checks end |
.send_failing_checks ⇒ Object
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 |
# File 'lib/blazer.rb', line 192 def self.send_failing_checks emails = {} slack_channels = {} Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check| check.split_emails.each do |email| (emails[email] ||= []) << check end check.split_slack_channels.each do |channel| (slack_channels[channel] ||= []) << check end end emails.each do |email, checks| Safely.safely do Blazer::CheckMailer.failing_checks(email, checks).deliver_now end end slack_channels.each do |channel, checks| Safely.safely do Blazer::SlackNotifier.failing_checks(channel, checks) end end end |
.settings ⇒ Object
106 107 108 109 110 111 112 113 114 115 |
# File 'lib/blazer.rb', line 106 def self.settings @settings ||= begin path = Rails.root.join("config", "blazer.yml").to_s if File.exist?(path) YAML.load(ERB.new(File.read(path)).result) else {} end end end |
.slack? ⇒ Boolean
218 219 220 |
# File 'lib/blazer.rb', line 218 def self.slack? slack_webhook_url.present? end |