Class: Fluent::Plugin::MongoOutput

Inherits:
Output
  • Object
show all
Includes:
LoggerSupport, MongoAuth, MongoAuthParams
Defined in:
lib/fluent/plugin/out_mongo.rb

Direct Known Subclasses

MongoOutputReplset

Constant Summary collapse

DEFAULT_BUFFER_TYPE = 
"memory"
LIMIT_BEFORE_v1_8 =

Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.

2 * 1024 * 1024
LIMIT_AFTER_v1_8 =

2MB = 4MB / 2

8 * 1024 * 1024

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from LoggerSupport

#configure_logger, included

Methods included from MongoAuth

#authenticate

Methods included from MongoAuthParams

included

Constructor Details

#initialize ⇒ MongoOutput

Returns a new instance of MongoOutput.



64
65
66
67
68
69
70
 
# File 'lib/fluent/plugin/out_mongo.rb', line 64

def initialize
  super

  @nodes = nil
  @client_options = {}
  @collection_options = {capped: false}
end

Instance Attribute Details

#client_options ⇒ Object (readonly)

Returns the value of attribute client_options.



62
63
64
 
# File 'lib/fluent/plugin/out_mongo.rb', line 62

def client_options
  @client_options
end

#collection_options ⇒ Object (readonly)

Returns the value of attribute collection_options.



62
63
64
 
# File 'lib/fluent/plugin/out_mongo.rb', line 62

def collection_options
  @collection_options
end

Instance Method Details

#configure(conf) ⇒ Object

8MB = 16MB / 2

Raises:

  • (Fluent::ConfigError) 


76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/fluent/plugin/out_mongo.rb', line 76

def configure(conf)
  if conf.has_key?('buffer_chunk_limit')
    configured_chunk_limit_size = Fluent::Config.size_value(conf['buffer_chunk_limit'])
    estimated_limit_size = LIMIT_AFTER_v1_8
    estimated_limit_size_conf = '8m'
    if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
      estimated_limit_size = LIMIT_BEFORE_v1_8
      estimated_limit_size_conf = '2m'
    end
    if configured_chunk_limit_size > estimated_limit_size
      log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{estimated_limit_size_conf}"
      conf['buffer_chunk_limit'] = estimated_limit_size_conf
    end
  else
    if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
      conf['buffer_chunk_limit'] = '2m'
    else
      conf['buffer_chunk_limit'] = '8m'
    end
  end
  # 'config_set_default :include_time_key, true' is ignored in compat_parameters_convert so need manual setting
  if conf.elements('inject').empty?
    if conf.has_key?('include_time_key')
      if Fluent::Config.bool_value(conf['include_time_key']) && !conf.has_key?('time_key')
        conf['time_key'] = 'time'
      end
    else
      conf['time_key'] = 'time'
    end
  end

  compat_parameters_convert(conf, :inject)

  super

  if @connection_string.nil? && @database.nil?
    raise Fluent::ConfigError,  "connection_string or database parameter is required"
  end

  unless @ignore_invalid_record
    log.warn "Since v0.8, invalid record detection will be removed because mongo driver v2.x and API spec don't provide it. You may lose invalid records, so you should not send such records to mongo plugin"
  end

  if conf.has_key?('tag_mapped')
    log.warn "'tag_mapped' feature is replaced with built-in config placeholder. Please consider to use 'collection ${tag}'."
    @collection = '${tag}'
  end
  raise Fluent::ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')

  if conf.has_key?('capped')
    raise Fluent::ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
    @collection_options[:capped] = true
    @collection_options[:size] = Fluent::Config.size_value(conf['capped_size'])
    @collection_options[:max] = Fluent::Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
  end

  if remove_tag_prefix = conf['remove_tag_prefix']
    @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
  end

  @client_options[:write] = {j: @journaled}
  @client_options[:write].merge!({w: @write_concern}) unless @write_concern.nil?
  @client_options[:ssl] = @ssl

  if @ssl
    @client_options[:ssl_cert] = @ssl_cert
    @client_options[:ssl_key] = @ssl_key
    @client_options[:ssl_key_pass_phrase] = @ssl_key_pass_phrase
    @client_options[:ssl_verify] = @ssl_verify
    @client_options[:ssl_ca_cert] = @ssl_ca_cert
  end
  @nodes = ["#{@host}:#{@port}"] if @nodes.nil?

  configure_logger(@mongo_log_level)

  log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
end

#emit(tag, es, chain) ⇒ Object 



166
167
168
169
170
171
172
 
# File 'lib/fluent/plugin/out_mongo.rb', line 166

def emit(tag, es, chain)
  if @tag_mapped
    super(tag, es, chain, tag)
  else
    super(tag, es, chain)
  end
end

#format(tag, time, record) ⇒ Object 



174
175
176
 
# File 'lib/fluent/plugin/out_mongo.rb', line 174

def format(tag, time, record)
  [time, record].to_msgpack
end

#formatted_to_msgpack_binary ⇒ Object 



178
179
180
 
# File 'lib/fluent/plugin/out_mongo.rb', line 178

def formatted_to_msgpack_binary
  true
end

#multi_workers_ready? ⇒ Boolean

Returns:

  • (Boolean) 


182
183
184
 
# File 'lib/fluent/plugin/out_mongo.rb', line 182

def multi_workers_ready?
  true
end

#shutdown ⇒ Object 



161
162
163
164
 
# File 'lib/fluent/plugin/out_mongo.rb', line 161

def shutdown
  @client.close
  super
end

#start ⇒ Object 



154
155
156
157
158
159
 
# File 'lib/fluent/plugin/out_mongo.rb', line 154

def start
  @client = client
  @client = authenticate(@client)
  @collections = {}
  super
end

#write(chunk) ⇒ Object 



186
187
188
189
 
# File 'lib/fluent/plugin/out_mongo.rb', line 186

def write(chunk)
  collection_name = extract_placeholders(@collection, chunk.metadata)
  operate(format_collection_name(collection_name), collect_records(chunk))
end