Class: LogStash::Outputs::LMLogs

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/outputs/lmlogs.rb

Overview

An example output that does nothing.

Defined Under Namespace

Classes: InvalidHTTPConfigError

Constant Summary collapse

@@MAX_PAYLOAD_SIZE = 
8*1024*1024
@@CONSOLE_LOGS =

For developer debugging.

false

Instance Method Summary collapse

Instance Method Details

#client ⇒ Object 



159
160
161
 
# File 'lib/logstash/outputs/lmlogs.rb', line 159

def client
  @client ||= make_client
end

#client_config ⇒ Object

def register



115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/logstash/outputs/lmlogs.rb', line 115

def client_config
  c = {
      connect_timeout: @connect_timeout,
      socket_timeout: @socket_timeout,
      request_timeout: @request_timeout,
      follow_redirects: @follow_redirects,
      automatic_retries: @automatic_retries,
      retry_non_idempotent: @retry_non_idempotent,
      check_connection_timeout: @validate_after_inactivity,
      pool_max: @pool_max,
      pool_max_per_route: @pool_max_per_route,
      cookies: @cookies,
      keepalive: @keepalive
  }

  if @proxy
    # Symbolize keys if necessary
    c[:proxy] = @proxy.is_a?(Hash) ?
                    @proxy.reduce({}) {|memo,(k,v)| memo[k.to_sym] = v; memo} :
                    @proxy
  end

  if @access_id
    if !@access_key || !@access_key.value
      raise ::LogStash::ConfigurationError, "access_id '#{@access_id}' specified without access_key!"
    end

    # Symbolize keys if necessary
    # c[:auth] = {
    #     :user => @access_id,
    #     :password => @access_key.value,
    #     :eager => true
    # }
  end
 log_debug("manticore client config: ", :client => c)              
  return c
end

#close ⇒ Object 



164
165
166
 
# File 'lib/logstash/outputs/lmlogs.rb', line 164

def close
  @client.close
end

#generate_auth_string(body) ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/logstash/outputs/lmlogs.rb', line 169

def generate_auth_string(body)
  timestamp = DateTime.now.strftime('%Q')
  hash_this = "POST#{timestamp}#{body}/log/ingest"
  sign_this = OpenSSL::HMAC.hexdigest(
                OpenSSL::Digest.new('sha256'),
                "#{@access_key.value}",
                hash_this
              )
  signature = Base64.strict_encode64(sign_this)
  "LMv1 #{@access_id}:#{signature}:#{timestamp}"
end

#isValidPayloadSize(documents, lmlogs_event, max_payload_size) ⇒ Object 



289
290
291
292
293
294
295
296
297
 
# File 'lib/logstash/outputs/lmlogs.rb', line 289

def isValidPayloadSize(documents,lmlogs_event,max_payload_size)
  if (documents.to_json.bytesize + lmlogs_event.to_json.bytesize) >  max_payload_size 
        send_batch(documents)
        documents = []
        
  end
  documents.push(lmlogs_event)
  return documents
end

#log_debug(message, *opts) ⇒ Object 



243
244
245
246
247
248
249
 
# File 'lib/logstash/outputs/lmlogs.rb', line 243

def log_debug(message, *opts)
  if @@CONSOLE_LOGS
    puts "[#{DateTime::now}] [logstash.outputs.lmlogs] [DEBUG] #{message} #{opts.to_s}"
  elsif debug
    @logger.debug(message, *opts)
  end
end

#log_failure(message, opts) ⇒ Object 



285
286
287
 
# File 'lib/logstash/outputs/lmlogs.rb', line 285

def log_failure(message, opts)
  @logger.error("[HTTP Output Failure] #{message}", opts)
end

#multi_receive(events) ⇒ Object 



252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/logstash/outputs/lmlogs.rb', line 252

def multi_receive(events)
  if events.length() > 0
    log_debug(events.to_json)
 end

  events.each_slice(@batch_size) do |chunk|
    documents = []
    chunk.each do |event|
      event_json = JSON.parse(event.to_json)
      lmlogs_event = event_json
      lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
      lmlogs_event["event"].delete("original") # remove redundant log field

      lmlogs_event["message"] = event.get(@message_key).to_s

      lmlogs_event["_lm.resourceId"] = {}
      lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)

      if @keep_timestamp
        lmlogs_event["timestamp"] = event.get("@timestamp")
      end
      
      if @timestamp_is_key
        lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
      end

      documents = isValidPayloadSize(documents,lmlogs_event,@@MAX_PAYLOAD_SIZE)

    end
    send_batch(documents)
  end
end

#register ⇒ Object 



105
106
107
108
109
110
111
112
113
 
# File 'lib/logstash/outputs/lmlogs.rb', line 105

def register
  @total = 0
  @total_failed = 0
  logger.info("Initialized LogicMonitor output plugin with configuration",
              :host => @host)
  logger.info("Max Payload Size: ", 
              :size => @@MAX_PAYLOAD_SIZE)

end

#send_batch(events) ⇒ Object 



181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/logstash/outputs/lmlogs.rb', line 181

def send_batch(events)
  log_debug("Started sending logs to LM: ", 
                :time => Time::now.utc)
  url = "https://" + @portal_name + ".logicmonitor.com/rest/log/ingest"
  body = events.to_json
  auth_string = generate_auth_string(body)
  request = client.post(url, {
      :body => body,
      :headers => {
              "Content-Type" => "application/json",
              "User-Agent" => "LM Logs Logstash Plugin",
              "Authorization" => "#{auth_string}"
      }
  })

  request.on_success do |response|
    if response.code == 202
      @total += events.length
      log_debug("Successfully sent ",
                    :response_code => response.code,
                    :batch_size => events.length,
                    :total_sent => @total,
                    :time => Time::now.utc)
    elsif response.code == 207
      log_failure(
        "207 HTTP code - some of the events successfully parsed, some not. ",
        :response_code => response.code,
        :url => url,
        :response_body => response.body,
        :total_failed => @total_failed)
    else
      @total_failed += 1
      log_failure(
          "Encountered non-202/207 HTTP code #{response.code}",
          :response_code => response.code,
          :url => url,
          :response_body => response.body,
          :total_failed => @total_failed)
    end
  end

  request.on_failure do |exception|
    @total_failed += 1
    log_failure("The request failed. ",
                :url => url,
                :method => @http_method,
                :message => exception.message,
                :class => exception.class.name,
                :backtrace => exception.backtrace,
                :total_failed => @total_failed
    )
  end

  log_debug("Completed sending logs to LM",
                :total => @total,
                :time => Time::now.utc)
  request.call

rescue Exception => e
  @logger.error("[Exception=] #{e.message} #{e.backtrace}")
end