Class: Fluent::GoogleCloudOutput

Inherits:
BufferedOutput
  • Object
show all
Defined in:
lib/fluent/plugin/out_google_cloud.rb

Defined Under Namespace

Modules: Platform

Constant Summary collapse

APPENGINE_SERVICE =

Constants for service names.

'appengine.googleapis.com'
COMPUTE_SERVICE = 
'compute.googleapis.com'
DATAFLOW_SERVICE = 
'dataflow.googleapis.com'
EC2_SERVICE = 
'ec2.amazonaws.com'
LOGGING_SCOPE =

Name of the the Google cloud logging write scope.

'https://www.googleapis.com/auth/logging.write'
METADATA_SERVICE_ADDR =

Address of the metadata service.

'169.254.169.254'

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ GoogleCloudOutput

Returns a new instance of GoogleCloudOutput.



70
71
72
73
74
75
76
77
78
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 70

def initialize
  super
  require 'cgi'
  require 'google/api_client'
  require 'google/api_client/auth/compute_service_account'
  require 'googleauth'
  require 'json'
  require 'open-uri'
end

Instance Attribute Details

#common_labels ⇒ Object (readonly)

Returns the value of attribute common_labels.



68
69
70
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 68

def common_labels
  @common_labels
end

#gae_backend_name ⇒ Object (readonly)

Returns the value of attribute gae_backend_name.



65
66
67
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 65

def gae_backend_name
  @gae_backend_name
end

#gae_backend_version ⇒ Object (readonly)

Returns the value of attribute gae_backend_version.



66
67
68
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 66

def gae_backend_version
  @gae_backend_version
end

#project_id ⇒ Object (readonly)

Expose attr_readers to make testing of metadata more direct than only testing it indirectly through metadata sent with logs.



61
62
63
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 61

def project_id
  @project_id
end

#running_on_managed_vm ⇒ Object (readonly)

Returns the value of attribute running_on_managed_vm.



64
65
66
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 64

def running_on_managed_vm
  @running_on_managed_vm
end

#service_name ⇒ Object (readonly)

Returns the value of attribute service_name.



67
68
69
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 67

def service_name
  @service_name
end

#vm_id ⇒ Object (readonly)

Returns the value of attribute vm_id.



63
64
65
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 63

def vm_id
  @vm_id
end

#zone ⇒ Object (readonly)

Returns the value of attribute zone.



62
63
64
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 62

def zone
  @zone
end

Instance Method Details

#configure(conf) ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 80

def configure(conf)
  super

  if !@auth_method.nil?
    $log.warn ('auth_method is deprecated; please migrate to using ' +
               'Application Default Credentials.')
    if @auth_method == 'private_key'
      if !@private_key_email
        raise Fluent::ConfigError, ('"private_key_email" must be ' +
            'specified if auth_method is "private_key"')
      elsif !@private_key_path
        raise Fluent::ConfigError, ('"private_key_path" must be ' +
            'specified if auth_method is "private_key"')
      elsif !@private_key_passphrase
        raise Fluent::ConfigError, ('"private_key_passphrase" must be ' +
            'specified if auth_method is "private_key"')
      end
    end
  end

  # TODO: Send instance tags and/or hostname as labels as well?
  @common_labels = {}

  # set attributes from metadata (unless overriden by static config)
  @platform = detect_platform
  case @platform
  when Platform::GCE
    if @project_id.nil?
      @project_id = fetch_gce_metadata('project/project-id')
    end
    if @zone.nil?
      # this returns "projects/<number>/zones/<zone>"; we only want
      # the part after the final slash.
      fully_qualified_zone = fetch_gce_metadata('instance/zone')
      @zone = fully_qualified_zone.rpartition('/')[2]
    end
    if @vm_id.nil?
      @vm_id = fetch_gce_metadata('instance/id')
    end
  when Platform::EC2
    metadata = fetch_ec2_metadata
    if @zone.nil? && metadata.has_key?('availabilityZone')
      @zone = 'aws:' + metadata['availabilityZone']
    end
    if @vm_id.nil? && metadata.has_key?('instanceId')
      @vm_id = metadata['instanceId']
    end
    if metadata.has_key?('accountId')
      common_labels["#{EC2_SERVICE}/account_id"] = metadata['accountId']
    end
  when Platform::OTHER
    # do nothing
  else
    raise Fluent::ConfigError, 'Unknown platform ' + @platform
  end

  # all metadata parameters must now be set
  unless @project_id && @zone && @vm_id
    missing = []
    missing << "project_id" unless @project_id
    missing << "zone" unless @zone
    missing << "vm_id" unless @vm_id
    raise Fluent::ConfigError,
      ('Unable to obtain metadata parameters: ' + missing.join(' '))
  end

  # Default this to false; it is only overwritten if we detect Managed VM.
  @running_on_managed_vm = false

  # Set labels, etc. based on the config
  case @platform
  when Platform::GCE
    @service_name = COMPUTE_SERVICE
    # Check for specialized GCE environments (Managed VM or Dataflow).
    # TODO: Add config options for these to allow for running outside GCE?
    attributes = fetch_gce_metadata('instance/attributes/').split
    if (attributes.include?('gae_backend_name') &&
        attributes.include?('gae_backend_version'))
      # Managed VM
      @running_on_managed_vm = true
      @gae_backend_name =
          fetch_gce_metadata('instance/attributes/gae_backend_name')
      @gae_backend_version =
          fetch_gce_metadata('instance/attributes/gae_backend_version')
      @service_name = APPENGINE_SERVICE
      common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
      common_labels["#{APPENGINE_SERVICE}/version_id"] =
        @gae_backend_version
    elsif (attributes.include?('job_id'))
      # Dataflow
      @service_name = DATAFLOW_SERVICE
      @dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
      common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
    end
    # include GCE labels unless we're running on dataflow, which
    # uses their own labels exclusively.
    if (@service_name != DATAFLOW_SERVICE)
      common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
      common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
    end
  when Platform::EC2
    @service_name = EC2_SERVICE
    common_labels["#{EC2_SERVICE}/resource_type"] = 'instance'
    common_labels["#{EC2_SERVICE}/resource_id"] = @vm_id
  when Platform::OTHER
    # Use COMPUTE_SERVICE as the default environment.
    @service_name = COMPUTE_SERVICE
    common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
    common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
  end
end

#format(tag, time, record) ⇒ Object 



205
206
207
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 205

def format(tag, time, record)
  [tag, time, record].to_msgpack
end

#shutdown ⇒ Object 



201
202
203
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 201

def shutdown
  super
end

#start ⇒ Object 



192
193
194
195
196
197
198
199
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 192

def start
  super

  init_api_client()

  @successful_call = false
  @timenanos_warning = false
end

#write(chunk) ⇒ Object 



209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
 
# File 'lib/fluent/plugin/out_google_cloud.rb', line 209

def write(chunk)
  # Group the entries since we have to make one call per tag.
  grouped_entries = {}
  chunk.msgpack_each do |tag, *arr|
    if !grouped_entries.has_key?(tag)
      grouped_entries[tag] = []
    end
    grouped_entries[tag].push(arr)
  end

  grouped_entries.each do |tag, arr|
    write_log_entries_request = {
      'commonLabels' => @common_labels,
      'entries' => [],
    }
    arr.each do |time, record|
      next unless record.is_a? Hash
      if (record.has_key?('timestamp') &&
          record['timestamp'].has_key?('seconds') &&
          record['timestamp'].has_key?('nanos'))
        ts_secs = record['timestamp']['seconds']
        ts_nanos = record['timestamp']['nanos']
        record.delete('timestamp')
      elsif (record.has_key?('timestampSeconds') &&
             record.has_key?('timestampNanos'))
        ts_secs = record['timestampSeconds']
        ts_nanos = record['timestampNanos']
        record.delete('timestampSeconds')
        record.delete('timestampNanos')
      elsif (record.has_key?('timeNanos'))
        # This is deprecated since the precision is insufficient.
        # Use timestampSeconds/timestampNanos instead
        ts_secs = (record['timeNanos'] / 1000000000).to_i
        ts_nanos = record['timeNanos'] % 1000000000
        record.delete('timeNanos')
        if (!@timenanos_warning)
          # Warn the user this is deprecated, but only once to avoid spam.
          @timenanos_warning = true
          $log.warn ("timeNanos is deprecated - please use " +
                     "timestampSeconds and timestampNanos instead.")
        end
      else
        timestamp = Time.at(time)
        ts_secs = timestamp.tv_sec
        ts_nanos = timestamp.tv_nsec
      end
      entry = {
        'metadata' => {
          'serviceName' => @service_name,
          'projectId' => @project_id,
          'zone' => @zone,
          'timestamp' => {
            'seconds' => ts_secs,
            'nanos' => ts_nanos
          },
        },
      }
      if record.has_key?('severity')
        entry['metadata']['severity'] = parse_severity(record['severity'])
        record.delete('severity')
      else
        entry['metadata']['severity'] = 'DEFAULT'
      end

      # use textPayload if the only remainaing key is 'message',
      # otherwise use a struct.
      if (record.size == 1 && record.has_key?('message'))
        entry['textPayload'] = record['message']
      else
        entry['structPayload'] = record
      end
      write_log_entries_request['entries'].push(entry)
    end
    # Don't send an empty request if we rejected all the entries.
    next if write_log_entries_request['entries'].empty?

    # Add a prefix to VMEngines logs to prevent namespace collisions,
    # and also escape the log name.
    log_name = CGI::escape(@running_on_managed_vm ?
                           "#{APPENGINE_SERVICE}/#{tag}" : tag)
    url = ('https://logging.googleapis.com/v1beta3/projects/' +
           "#{@project_id}/logs/#{log_name}/entries:write")
    begin
      client = api_client()
      request = client.generate_request({
        :uri => url,
        :body_object => write_log_entries_request,
        :http_method => 'POST',
        :authenticated => true
      })
      client.execute!(request)
      # Let the user explicitly know when the first call succeeded,
      # to aid with verification and troubleshooting.
      if (!@successful_call)
        @successful_call = true
        $log.info "Successfully sent to Google Cloud Logging API."
      end
    # Allow most exceptions to propagate, which will cause fluentd to
    # retry (with backoff), but in some cases we catch the error and
    # drop the request (we will emit a log message in those cases).
    rescue Google::APIClient::ClientError => error
      # Most ClientErrors indicate a problem with the request itself and
      # should not be retried, unless it is an authentication issue, in
      # which case we will retry the request via re-raising the exception.
      if (is_retriable_client_error(error))
        raise error
      end
      log_write_failure(write_log_entries_request, error)
    rescue JSON::GeneratorError => error
      # This happens if the request contains illegal characters;
      # do not retry it because it will fail repeatedly.
      log_write_failure(write_log_entries_request, error)
    end
  end
end