Module: OciLogAnalyticsResourcesDiscovery

Extended by:
Util::Logging
Defined in:
lib/oci_loganalytics_resources_discovery.rb

Constant Summary collapse

@@oci_la_log_source_name = 
'Kubernetes Objects Discovery Payload Logs'
@@oci_la_log_path = 
'UNDEFINED'

Constants included from Util::Logging

Util::Logging::SEV_LABEL, Util::Logging::TRACE

Class Method Summary collapse

Methods included from Util::Logging

logger, logger=

Class Method Details

.filter_event_logs(kubernetes_objects_payload, collect_warning_events_only) ⇒ Object 



208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 208

def filter_event_logs(kubernetes_objects_payload, collect_warning_events_only)
  cluster_events_processed_time = Util::StateManager.state.last_timestamp
  new_processed_time = cluster_events_processed_time
  filtered_events = []
  logger.info("Filtering events with EPOCH cut-off : #{cluster_events_processed_time} - (#{Time.at(cluster_events_processed_time).getutc})")
  begin
    eventlist_response = JSON.parse(kubernetes_objects_payload[:raw_response][:events], symbolize_names: true)
    eventslist = eventlist_response[:items]
    eventslist.each do |event|
      # Skip all non warning events if collect_warning_events_only == true
      next if collect_warning_events_only && event[:type] != 'Warning'

      # if lastTimeStamp is present, use it, otherwise fallback to creationTimestamp
      event_timestamp = if !event[:lastTimestamp].nil?
                          event[:lastTimestamp]
                        elsif !event[:metadata][:creationTimestamp].nil?
                          event[:metadata][:creationTimestamp]
                        else
                          logger.warn("Not able to find event timestamp in following event (uid) - #{event[:uid]}")
                          next
                        end

      event_timestamp = Time.parse(event_timestamp).to_i

      if event_timestamp > cluster_events_processed_time
        filtered_events.push(event)
        new_processed_time = event_timestamp if event_timestamp > new_processed_time
      end
    end
    logger.info("Event Filter Stats: #{filtered_events.length} out of #{eventslist.length} events included.")
    Util::StateManager.state.last_timestamp = new_processed_time
  rescue StandardError => e
    logger.error("Failed to filter event logs - #{e}")
    raise e
  end
  eventlist_response[:items] = filtered_events
  kubernetes_objects_payload[:raw_response][:events] = JSON.dump(eventlist_response)
  kubernetes_objects_payload
end

.get_auth_object ⇒ Object 



150
151
152
153
154
155
156
157
158
159
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 150

def get_auth_object
  begin
    Util::OCIClients.initialize_auth_config(@auth_config_hash)
  rescue StandardError => e
    logger.error("Error occurred in creating authentication object - #{e}")
    raise e
  end
  @auth_object = Util::OCIClients.get_auth_config_object
  logger.debug('Successfully loaded the OCI auth config.')
end

.get_infra_resources_payload ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 183

def get_infra_resources_payload
  logger.debug('Discovering Infrastructure Resources')
  infra_resources_payload = nil

  unless @auth_object.nil?
    begin
      infra_resources_payload = InfraResources.get_infra_resources(@auth_object, @cluster_config_hash[:oci_la_cluster_entity_id])
    rescue StandardError => e
      logger.error("Error in getting infrastructure resources: #{e}")
    end
  end
  infra_resources_payload
end

.get_kube_clients ⇒ Object 



161
162
163
164
165
166
167
168
169
170
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 161

def get_kube_clients
  begin
    Util::KubeClient.create_clients(@kube_config_hash)
  rescue StandardError => e
    logger.error("Error occurred in creating kubeclients - #{e}")
    raise e
  end
  @kube_clients = Util::KubeClient.get_clients
  logger.debug('Kubeclients created successfully.')
end

.get_objects_resources_payload ⇒ Object

Raises:

  • (StandardError) 


197
198
199
200
201
202
203
204
205
206
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 197

def get_objects_resources_payload
  logger.info('Discovering Kubernetes Objects.')
  raise StandardError, 'Kubeclients not initialized.' if @kube_clients.nil?

  ObjectsResources.get_objects_resources_helper(
    @cluster_config_hash[:kubernetes_cluster_id],
    @cluster_config_hash[:kubernetes_cluster_name],
    @app_config_hash
  )
end

.get_oci_clients(options) ⇒ Object 



172
173
174
175
176
177
178
179
180
181
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 172

def get_oci_clients(options)
  begin
    Util::OCIClients.create_clients(@auth_object, options)
  rescue StandardError => e
    logger.error("Error occurred in creating OCI clients - #{e}")
    raise e
  end
  @oci_clients = Util::OCIClients.get_clients
  logger.debug('OCI clients created successfully.')
end

.get_zippedstream(kubernetes_object_payload) ⇒ Object 



248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 248

def get_zippedstream(kubernetes_object_payload)
  logger.debug('Creating Zipped stream for object log collection payload')
  begin
    zippedstream = Zip::OutputStream.write_buffer do |zos|
      kubernetes_object_payload[:raw_response].each do |key, value|
        next if value.nil? || value.empty?

        next_entry = "k8s.#{key}.json"
        zos.put_next_entry(next_entry)

        zos.write(value)
      end
    end
    zippedstream.rewind
  rescue StandardError => e
    logger.error("Failed to generate zipped stream of discovery payload - #{e}")
    raise e
  end
  zippedstream
end

.initialize(cluster_config_hash:, auth_config_hash:, kube_config_hash:, app_config_hash:) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 47

def initialize(cluster_config_hash:, auth_config_hash:, kube_config_hash:, app_config_hash:)
  @cluster_config_hash = cluster_config_hash
  @auth_config_hash = auth_config_hash
  @kube_config_hash = kube_config_hash
  @app_config_hash = app_config_hash

  # OCI.logger = Util::Logging.logger

  get_auth_object

  options = { mode: app_config_hash[:mode] }
  get_oci_clients(options)

  if @app_config_hash[:mode] == 'object'
    @snapshot_id = Time.now.to_i
    get_kube_clients
    Util::StateManager.init(@cluster_config_hash[:kubernetes_resourcename_prefix], @cluster_config_hash[:kubernetes_cluster_namespace])
    Util::KubectlOps.set_chunk_limit @app_config_hash[:chunk_limit]
    logger.debug("Chunk limit set to - #{@app_config_hash[:chunk_limit]}")
  end
end

.initiate_infra_discovery ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 69

def initiate_infra_discovery
  logger.info('Initiating Kubernetes infrastructure discovery.')
  begin
    infra_objects_payload = get_infra_resources_payload
  rescue StandardError => e
    logger.error("Error occurred while fetching infrastructure resources. Error: #{e}")
    raise StandardError, 'Error occurred while fetching infrastructure resources.'
  end

  logger.info('Infrastructure discovery payload created.')
  logger.trace("Infrastructure discovery payload: \n#{Yajl.dump(infra_objects_payload.to_hash)}")

  if @app_config_hash[:skip_upload]
    logger.warn('--skip_upload Flag is set: Skipping payload upload to OCI Logging Analytics.')
  else
    if !infra_objects_payload.nil?
      # TODO: Upload implemetation pending.
    else
      logger.warn('No infrastructure objects discovered.')
    end
  end
end

.initiate_object_discovery ⇒ Object 



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 92

def initiate_object_discovery
  kubernetes_objects_payload = get_objects_resources_payload
  if @app_config_hash[:collect_warning_events_only]
    logger.info('Only Warning event logs will be collected as the following flag is set: --collect_warning_events_only')
  end
  kubernetes_objects_payload = filter_event_logs(kubernetes_objects_payload,
                                                 @app_config_hash[:collect_warning_events_only])

  logger.info('Kubernetes objects discovery payload created.')
  logger.trace("Object Discovery Payload: \n#{Yajl.dump(kubernetes_objects_payload[:processed_objects].to_hash)}")
  logger.trace("Object Log Collection Payload: \n#{Yajl.dump(kubernetes_objects_payload[:raw_response].to_hash)}")

  if @app_config_hash[:skip_upload]
    logger.warn('--skip_upload Flag is set: Skipping payload upload to OCI logging analytics.')
  else
    upload_failure = false

    begin
      # Upload object logs with relevant header parameters
      logger.info('Uploading Object Logs to OCI')
      object_logs_opts = {
        payload_type: 'ZIP',
        discovery_data_type: 'K8S_OBJECTS',
        opc_meta_properties: "cluster-entity-id:#{@cluster_config_hash[:oci_la_cluster_entity_id]};sub-type:object_logs;snapshot-id:#{@snapshot_id};log-group-id:#{@cluster_config_hash[:oci_la_log_group_id]};cluster-id:#{@cluster_config_hash[:kubernetes_cluster_id]};cluster-name:#{@cluster_config_hash[:kubernetes_cluster_name]}"
      }
      zip_object_log_payload = get_zippedstream(kubernetes_objects_payload)
      upload_data_via_discovery_api(zip_object_log_payload, object_logs_opts) unless zip_object_log_payload.nil?
      logger.info('Successfully uploaded object logs to OCI')

      # Update the Events tracker config map only after successful objects logs upload
      Util::StateManager.update_state_configmap
    rescue StandardError
      logger.error('Error occurred while uploading object logs to OCI')
      upload_failure = true
    end

    unless @cluster_config_hash[:oci_la_cluster_entity_id].nil?
      begin
        # Upload object discovery payload with relevant header parameters
        logger.info('Uploading Object Discovery Payload to OCI')
        object_discovery_opts = {
          payload_type: 'JSON',
          discovery_data_type: 'K8S_OBJECTS',
          opc_meta_properties: "cluster-entity-id:#{@cluster_config_hash[:oci_la_cluster_entity_id]};sub-type:discovery_payload;cluster-id:#{@cluster_config_hash[:kubernetes_cluster_id]};cluster-name:#{@cluster_config_hash[:kubernetes_cluster_name]}"
        }
        json_object_discovery_payload = Yajl.dump(kubernetes_objects_payload[:processed_objects].to_hash)
        upload_data_via_discovery_api(json_object_discovery_payload, object_discovery_opts)
        logger.info('Successfully uploaded object discovery payload to OCI')
      rescue StandardError
        logger.error('Error occurred while uploading object discovery payload to OCI')
        upload_failure = true
      end
    end

    raise StandardError, 'Error occurred while uploading payloads to OCI' if upload_failure
  end
end

.upload_data_via_discovery_api(payload, opts) ⇒ Object 



269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
 
# File 'lib/oci_loganalytics_resources_discovery.rb', line 269

def upload_data_via_discovery_api(payload, opts)
  response = if @app_config_hash[:enable_raw_request]
               Util::LogAnalytics.upload_discovery_data_raw_request(@cluster_config_hash[:oci_la_namespace],
                                                             payload,
                                                             opts,
                                                             @auth_object)
             else
               Util::LogAnalytics.upload_discovery_data(@cluster_config_hash[:oci_la_namespace],
                                                 payload,
                                                 opts)
             end
  unless response.nil?
    resp_headers = response.headers
    logger.info("Response OPC object ID: #{resp_headers['opc-object-id']}")
    logger.info("Response OPC request ID: #{resp_headers['opc-request-id']}")
  end
rescue OCI::Errors::ServiceError => e
  error_code = e.status_code
  logger.error("Error code '#{error_code}' encountered while uploading payload, error: #{e}")
  raise e
rescue StandardError => e
  logger.error("Error in uploading data - #{e}")
  raise e
end