Module: Blazer
- Defined in:
- lib/blazer.rb,
lib/blazer/engine.rb,
lib/blazer/result.rb,
lib/blazer/version.rb,
lib/blazer/statement.rb,
lib/blazer/data_source.rb,
app/models/blazer/audit.rb,
app/models/blazer/check.rb,
app/models/blazer/query.rb,
lib/blazer/check_mailer.rb,
app/models/blazer/record.rb,
app/models/blazer/upload.rb,
lib/blazer/run_statement.rb,
lib/blazer/slack_notifier.rb,
app/models/blazer/dashboard.rb,
app/models/blazer/connection.rb,
lib/blazer/run_statement_job.rb,
app/helpers/blazer/base_helper.rb,
lib/blazer/adapters/sql_adapter.rb,
lib/blazer/adapters/base_adapter.rb,
lib/blazer/adapters/hive_adapter.rb,
lib/blazer/adapters/soda_adapter.rb,
app/models/blazer/dashboard_query.rb,
lib/blazer/adapters/drill_adapter.rb,
lib/blazer/adapters/druid_adapter.rb,
lib/blazer/adapters/neo4j_adapter.rb,
lib/blazer/adapters/spark_adapter.rb,
lib/blazer/adapters/athena_adapter.rb,
lib/blazer/adapters/ignite_adapter.rb,
lib/blazer/adapters/presto_adapter.rb,
lib/blazer/adapters/mongodb_adapter.rb,
app/models/blazer/uploads_connection.rb,
lib/blazer/adapters/bigquery_adapter.rb,
lib/blazer/adapters/influxdb_adapter.rb,
lib/blazer/adapters/cassandra_adapter.rb,
lib/blazer/adapters/snowflake_adapter.rb,
app/controllers/blazer/base_controller.rb,
lib/blazer/adapters/opensearch_adapter.rb,
lib/blazer/adapters/salesforce_adapter.rb,
lib/generators/blazer/install_generator.rb,
lib/generators/blazer/uploads_generator.rb,
app/controllers/blazer/checks_controller.rb,
app/controllers/blazer/queries_controller.rb,
app/controllers/blazer/uploads_controller.rb,
lib/blazer/adapters/elasticsearch_adapter.rb,
app/controllers/blazer/dashboards_controller.rb
Defined Under Namespace
Modules: Adapters, BaseHelper, Generators Classes: Audit, BaseController, Check, CheckMailer, ChecksController, Connection, Dashboard, DashboardQuery, DashboardsController, DataSource, Engine, Error, QueriesController, Query, Record, Result, RunStatement, RunStatementJob, SlackNotifier, Statement, TimeoutNotSupported, Upload, UploadError, UploadsConnection, UploadsController
Constant Summary collapse
- VARIABLE_MESSAGE =
"Variable cannot be used in this position"
- TIMEOUT_MESSAGE =
"Query timed out :("
- TIMEOUT_ERRORS =
[ "canceling statement due to statement timeout", # postgres "canceling statement due to conflict with recovery", # postgres "cancelled on user's request", # redshift "canceled on user's request", # redshift "system requested abort", # redshift "maximum statement execution time exceeded" # mysql ]
- VERSION =
"2.6.5"
Class Attribute Summary collapse
-
.anomaly_checks ⇒ Object
Returns the value of attribute anomaly_checks.
-
.async ⇒ Object
Returns the value of attribute async.
-
.audit ⇒ Object
Returns the value of attribute audit.
-
.before_action ⇒ Object
Returns the value of attribute before_action.
-
.cache ⇒ Object
Returns the value of attribute cache.
-
.check_schedules ⇒ Object
Returns the value of attribute check_schedules.
-
.forecasting ⇒ Object
Returns the value of attribute forecasting.
-
.from_email ⇒ Object
Returns the value of attribute from_email.
-
.images ⇒ Object
Returns the value of attribute images.
-
.mapbox_access_token ⇒ Object
Returns the value of attribute mapbox_access_token.
-
.override_csp ⇒ Object
Returns the value of attribute override_csp.
-
.query_editable ⇒ Object
Returns the value of attribute query_editable.
-
.query_viewable ⇒ Object
Returns the value of attribute query_viewable.
-
.slack_oauth_token ⇒ Object
Returns the value of attribute slack_oauth_token.
-
.slack_webhook_url ⇒ Object
Returns the value of attribute slack_webhook_url.
-
.time_zone ⇒ Object
Returns the value of attribute time_zone.
-
.transform_statement ⇒ Object
Returns the value of attribute transform_statement.
-
.transform_variable ⇒ Object
Returns the value of attribute transform_variable.
- .user_class ⇒ Object
- .user_method ⇒ Object
-
.user_name ⇒ Object
Returns the value of attribute user_name.
Class Method Summary collapse
- .adapters ⇒ Object
- .archive_queries ⇒ Object
- .data_sources ⇒ Object
-
.extract_vars(statement) ⇒ Object
TODO move to Statement and remove in 3.0.0.
-
.monotonic_time ⇒ Object
private.
- .register_adapter(name, adapter) ⇒ Object
- .run_check(check) ⇒ Object
- .run_checks(schedule: nil) ⇒ Object
- .send_failing_checks ⇒ Object
- .settings ⇒ Object
- .slack? ⇒ Boolean
- .uploads? ⇒ Boolean
- .uploads_connection ⇒ Object
- .uploads_schema ⇒ Object
- .uploads_table_name(name) ⇒ Object
Class Attribute Details
.anomaly_checks ⇒ Object
Returns the value of attribute anomaly_checks.
65 66 67 |
# File 'lib/blazer.rb', line 65 def anomaly_checks @anomaly_checks end |
.async ⇒ Object
Returns the value of attribute async.
67 68 69 |
# File 'lib/blazer.rb', line 67 def async @async end |
.audit ⇒ Object
Returns the value of attribute audit.
54 55 56 |
# File 'lib/blazer.rb', line 54 def audit @audit end |
.before_action ⇒ Object
Returns the value of attribute before_action.
59 60 61 |
# File 'lib/blazer.rb', line 59 def before_action @before_action end |
.cache ⇒ Object
Returns the value of attribute cache.
61 62 63 |
# File 'lib/blazer.rb', line 61 def cache @cache end |
.check_schedules ⇒ Object
Returns the value of attribute check_schedules.
64 65 66 |
# File 'lib/blazer.rb', line 64 def check_schedules @check_schedules end |
.forecasting ⇒ Object
Returns the value of attribute forecasting.
66 67 68 |
# File 'lib/blazer.rb', line 66 def forecasting @forecasting end |
.from_email ⇒ Object
Returns the value of attribute from_email.
60 61 62 |
# File 'lib/blazer.rb', line 60 def from_email @from_email end |
.images ⇒ Object
Returns the value of attribute images.
68 69 70 |
# File 'lib/blazer.rb', line 68 def images @images end |
.mapbox_access_token ⇒ Object
Returns the value of attribute mapbox_access_token.
74 75 76 |
# File 'lib/blazer.rb', line 74 def mapbox_access_token @mapbox_access_token end |
.override_csp ⇒ Object
Returns the value of attribute override_csp.
71 72 73 |
# File 'lib/blazer.rb', line 71 def override_csp @override_csp end |
.query_editable ⇒ Object
Returns the value of attribute query_editable.
70 71 72 |
# File 'lib/blazer.rb', line 70 def query_editable @query_editable end |
.query_viewable ⇒ Object
Returns the value of attribute query_viewable.
69 70 71 |
# File 'lib/blazer.rb', line 69 def query_viewable @query_viewable end |
.slack_oauth_token ⇒ Object
Returns the value of attribute slack_oauth_token.
72 73 74 |
# File 'lib/blazer.rb', line 72 def slack_oauth_token @slack_oauth_token end |
.slack_webhook_url ⇒ Object
Returns the value of attribute slack_webhook_url.
73 74 75 |
# File 'lib/blazer.rb', line 73 def slack_webhook_url @slack_webhook_url end |
.time_zone ⇒ Object
Returns the value of attribute time_zone.
55 56 57 |
# File 'lib/blazer.rb', line 55 def time_zone @time_zone end |
.transform_statement ⇒ Object
Returns the value of attribute transform_statement.
62 63 64 |
# File 'lib/blazer.rb', line 62 def transform_statement @transform_statement end |
.transform_variable ⇒ Object
Returns the value of attribute transform_variable.
63 64 65 |
# File 'lib/blazer.rb', line 63 def transform_variable @transform_variable end |
.user_class ⇒ Object
100 101 102 103 104 105 |
# File 'lib/blazer.rb', line 100 def self.user_class if !defined?(@user_class) @user_class = settings.key?("user_class") ? settings["user_class"] : (User.name rescue nil) end @user_class end |
.user_method ⇒ Object
107 108 109 110 111 112 113 114 115 |
# File 'lib/blazer.rb', line 107 def self.user_method if !defined?(@user_method) @user_method = settings["user_method"] if user_class @user_method ||= "current_#{user_class.to_s.downcase.singularize}" end end @user_method end |
.user_name ⇒ Object
Returns the value of attribute user_name.
56 57 58 |
# File 'lib/blazer.rb', line 56 def user_name @user_name end |
Class Method Details
.adapters ⇒ Object
245 246 247 |
# File 'lib/blazer.rb', line 245 def self.adapters @adapters ||= {} end |
.archive_queries ⇒ Object
253 254 255 256 257 258 259 |
# File 'lib/blazer.rb', line 253 def self.archive_queries raise "Audits must be enabled to archive" unless Blazer.audit raise "Missing status column - see https://github.com/ankane/blazer#23" unless Blazer::Query.column_names.include?("status") viewed_query_ids = Blazer::Audit.where("created_at > ?", 90.days.ago).group(:query_id).count.keys.compact Blazer::Query.active.where.not(id: viewed_query_ids).update_all(status: "archived") end |
.data_sources ⇒ Object
128 129 130 131 132 133 134 135 136 |
# File 'lib/blazer.rb', line 128 def self.data_sources @data_sources ||= begin ds = Hash.new { |hash, key| raise Blazer::Error, "Unknown data source: #{key}" } settings["data_sources"].each do |id, s| ds[id] = Blazer::DataSource.new(id, s) end ds end end |
.extract_vars(statement) ⇒ Object
TODO move to Statement and remove in 3.0.0
139 140 141 142 143 |
# File 'lib/blazer.rb', line 139 def self.extract_vars(statement) # strip commented out lines # and regex {1} or {1,2} statement.to_s.gsub(/\-\-.+/, "").gsub(/\/\*.+\*\//m, "").scan(/\{\w*?\}/i).map { |v| v[1...-1] }.reject { |v| /\A\d+(\,\d+)?\z/.match(v) || v.empty? }.uniq end |
.monotonic_time ⇒ Object
private
262 263 264 |
# File 'lib/blazer.rb', line 262 def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC) end |
.register_adapter(name, adapter) ⇒ Object
249 250 251 |
# File 'lib/blazer.rb', line 249 def self.register_adapter(name, adapter) adapters[name] = adapter end |
.run_check(check) ⇒ Object
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 |
# File 'lib/blazer.rb', line 154 def self.run_check(check) tries = 1 ActiveSupport::Notifications.instrument("run_check.blazer", check_id: check.id, query_id: check.query.id, state_was: check.state) do |instrument| # try 3 times on timeout errors statement = check.query.statement_object data_source = statement.data_source while tries <= 3 result = data_source.run_statement(statement, refresh_cache: true, check: check, query: check.query) if result.timed_out? Rails.logger.info "[blazer timeout] query=#{check.query.name}" tries += 1 sleep(10) elsif result.error.to_s.start_with?("PG::ConnectionBad") data_source.reconnect Rails.logger.info "[blazer reconnect] query=#{check.query.name}" tries += 1 sleep(10) else break end end begin check.reload # in case state has changed since job started check.update_state(result) rescue ActiveRecord::RecordNotFound # check deleted end # TODO use proper logfmt Rails.logger.info "[blazer check] query=#{check.query.name} state=#{check.state} rows=#{result.rows.try(:size)} error=#{result.error}" # should be no variables instrument[:statement] = statement.bind_statement instrument[:data_source] = data_source instrument[:state] = check.state instrument[:rows] = result.rows.try(:size) instrument[:error] = result.error instrument[:tries] = tries end end |
.run_checks(schedule: nil) ⇒ Object
145 146 147 148 149 150 151 152 |
# File 'lib/blazer.rb', line 145 def self.run_checks(schedule: nil) checks = Blazer::Check.includes(:query) checks = checks.where(schedule: schedule) if schedule checks.find_each do |check| next if check.state == "disabled" Safely.safely { run_check(check) } end end |
.send_failing_checks ⇒ Object
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 |
# File 'lib/blazer.rb', line 198 def self.send_failing_checks emails = {} slack_channels = {} Blazer::Check.includes(:query).where(state: ["failing", "error", "timed out", "disabled"]).find_each do |check| check.split_emails.each do |email| (emails[email] ||= []) << check end check.split_slack_channels.each do |channel| (slack_channels[channel] ||= []) << check end end emails.each do |email, checks| Safely.safely do Blazer::CheckMailer.failing_checks(email, checks).deliver_now end end slack_channels.each do |channel, checks| Safely.safely do Blazer::SlackNotifier.failing_checks(channel, checks) end end end |
.settings ⇒ Object
117 118 119 120 121 122 123 124 125 126 |
# File 'lib/blazer.rb', line 117 def self.settings @settings ||= begin path = Rails.root.join("config", "blazer.yml").to_s if File.exist?(path) YAML.load(ERB.new(File.read(path)).result) else {} end end end |
.slack? ⇒ Boolean
224 225 226 |
# File 'lib/blazer.rb', line 224 def self.slack? slack_oauth_token.present? || slack_webhook_url.present? end |
.uploads? ⇒ Boolean
228 229 230 |
# File 'lib/blazer.rb', line 228 def self.uploads? settings.key?("uploads") end |
.uploads_connection ⇒ Object
232 233 234 235 |
# File 'lib/blazer.rb', line 232 def self.uploads_connection raise "Empty url for uploads" unless settings.dig("uploads", "url") Blazer::UploadsConnection.connection end |
.uploads_schema ⇒ Object
237 238 239 |
# File 'lib/blazer.rb', line 237 def self.uploads_schema settings.dig("uploads", "schema") || "uploads" end |
.uploads_table_name(name) ⇒ Object
241 242 243 |
# File 'lib/blazer.rb', line 241 def self.uploads_table_name(name) uploads_connection.quote_table_name("#{uploads_schema}.#{name}") end |