Class: Fluent::Plugin::MongoOutput

Inherits:
Output
  • Object
show all
Includes:
LoggerSupport, MongoAuth, MongoAuthParams
Defined in:
lib/fluent/plugin/out_mongo.rb

Direct Known Subclasses

MongoOutputReplset

Constant Summary collapse

DEFAULT_BUFFER_TYPE = 
"memory"
LIMIT_BEFORE_v1_8 =

Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.

2 * 1024 * 1024
LIMIT_AFTER_v1_8 =

2MB = 4MB / 2

8 * 1024 * 1024

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from LoggerSupport

#configure_logger, included

Methods included from MongoAuth

#authenticate

Methods included from MongoAuthParams

included

Constructor Details

#initialize ⇒ MongoOutput

Returns a new instance of MongoOutput.



72
73
74
75
76
77
78
 
# File 'lib/fluent/plugin/out_mongo.rb', line 72

def initialize
  super

  @nodes = nil
  @client_options = {}
  @collection_options = {capped: false}
end

Instance Attribute Details

#client_options ⇒ Object (readonly)

Returns the value of attribute client_options.



70
71
72
 
# File 'lib/fluent/plugin/out_mongo.rb', line 70

def client_options
  @client_options
end

#collection_options ⇒ Object (readonly)

Returns the value of attribute collection_options.



70
71
72
 
# File 'lib/fluent/plugin/out_mongo.rb', line 70

def collection_options
  @collection_options
end

Instance Method Details

#configure(conf) ⇒ Object

8MB = 16MB / 2

Raises:

  • (Fluent::ConfigError) 


84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/fluent/plugin/out_mongo.rb', line 84

def configure(conf)
  if conf.has_key?('buffer_chunk_limit')
    configured_chunk_limit_size = Fluent::Config.size_value(conf['buffer_chunk_limit'])
    estimated_limit_size = LIMIT_AFTER_v1_8
    estimated_limit_size_conf = '8m'
    if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
      estimated_limit_size = LIMIT_BEFORE_v1_8
      estimated_limit_size_conf = '2m'
    end
    if configured_chunk_limit_size > estimated_limit_size
      log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{estimated_limit_size_conf}"
      conf['buffer_chunk_limit'] = estimated_limit_size_conf
    end
  else
    if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
      conf['buffer_chunk_limit'] = '2m'
    else
      conf['buffer_chunk_limit'] = '8m'
    end
  end
  # 'config_set_default :include_time_key, true' is ignored in compat_parameters_convert so need manual setting
  if conf.elements('inject').empty?
    if conf.has_key?('include_time_key')
      if Fluent::Config.bool_value(conf['include_time_key']) && !conf.has_key?('time_key')
        conf['time_key'] = 'time'
      end
    else
      conf['time_key'] = 'time'
    end
  end

  compat_parameters_convert(conf, :inject)

  super

  if @auth_mech && !Mongo::Auth::SOURCES.has_key?(@auth_mech.to_sym)
    raise Fluent::ConfigError, Mongo::Auth::InvalidMechanism.new(@auth_mech.to_sym)
  end

  if @connection_string.nil? && @database.nil?
    raise Fluent::ConfigError,  "connection_string or database parameter is required"
  end

  if conf.has_key?('tag_mapped')
    log.warn "'tag_mapped' feature is replaced with built-in config placeholder. Please consider to use 'collection ${tag}'."
    @collection = '${tag}'
  end
  raise Fluent::ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')

  if conf.has_key?('capped')
    raise Fluent::ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
    @collection_options[:capped] = true
    @collection_options[:size] = Fluent::Config.size_value(conf['capped_size'])
    @collection_options[:max] = Fluent::Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
  end

  if remove_tag_prefix = conf['remove_tag_prefix']
    @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
  end

  @client_options[:write] = {j: @journaled}
  @client_options[:write].merge!({w: @write_concern}) unless @write_concern.nil?
  @client_options[:ssl] = @ssl

  if @ssl
    @client_options[:ssl_cert] = @ssl_cert
    @client_options[:ssl_key] = @ssl_key
    @client_options[:ssl_key_pass_phrase] = @ssl_key_pass_phrase
    @client_options[:ssl_verify] = @ssl_verify
    @client_options[:ssl_ca_cert] = @ssl_ca_cert
  end
  @nodes = ["#{@host}:#{@port}"] if @nodes.nil?

  configure_logger(@mongo_log_level)

  log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
end

#formatted_to_msgpack_binary ⇒ Object 



174
175
176
 
# File 'lib/fluent/plugin/out_mongo.rb', line 174

def formatted_to_msgpack_binary
  true
end

#multi_workers_ready? ⇒ Boolean

Returns:

  • (Boolean) 


178
179
180
 
# File 'lib/fluent/plugin/out_mongo.rb', line 178

def multi_workers_ready?
  true
end

#shutdown ⇒ Object 



169
170
171
172
 
# File 'lib/fluent/plugin/out_mongo.rb', line 169

def shutdown
  @client.close
  super
end

#start ⇒ Object 



162
163
164
165
166
167
 
# File 'lib/fluent/plugin/out_mongo.rb', line 162

def start
  @client = client
  @client = authenticate(@client)
  @collections = {}
  super
end

#write(chunk) ⇒ Object 



182
183
184
185
186
187
 
# File 'lib/fluent/plugin/out_mongo.rb', line 182

def write(chunk)
  collection_name = extract_placeholders(@collection, chunk)
  # In connection_string case, we shouldn't handle extract_placeholers for @database.
  database_name = extract_placeholders(@database, chunk) unless @connection_string
  operate(database_name, format_collection_name(collection_name), collect_records(chunk))
end