Class: LogStash::Inputs::S3
- Inherits:
-
Base
- Object
- Base
- LogStash::Inputs::S3
- Includes:
- PluginMixins::AwsConfig::V2
- Defined in:
- lib/logstash/inputs/s3.rb
Overview
Files ending in ‘.gz` are handled as gzip’ed files.
Defined Under Namespace
Modules: SinceDB
Instance Method Summary collapse
- #backup_to_bucket(object) ⇒ Object
- #backup_to_dir(filename) ⇒ Object
- #list_new_files ⇒ Object
- #process_files(queue) ⇒ Object
- #register ⇒ Object
- #run(queue) ⇒ Object
- #stop ⇒ Object
Instance Method Details
#backup_to_bucket(object) ⇒ Object
153 154 155 156 157 158 159 160 161 |
# File 'lib/logstash/inputs/s3.rb', line 153 def backup_to_bucket(object) unless @backup_to_bucket.nil? backup_key = "#{@backup_add_prefix}#{object.key}" @backup_bucket.object(backup_key).copy_from(:copy_source => "#{object.bucket_name}/#{object.key}") if @delete object.delete() end end end |
#backup_to_dir(filename) ⇒ Object
164 165 166 167 168 |
# File 'lib/logstash/inputs/s3.rb', line 164 def backup_to_dir(filename) unless @backup_to_dir.nil? FileUtils.cp(filename, @backup_to_dir) end end |
#list_new_files ⇒ Object
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 |
# File 'lib/logstash/inputs/s3.rb', line 124 def list_new_files objects = {} found = false begin @s3bucket.objects(:prefix => @prefix).each do |log| found = true @logger.debug("S3 input: Found key", :key => log.key) if ignore_filename?(log.key) @logger.debug('S3 input: Ignoring', :key => log.key) elsif log.content_length <= 0 @logger.debug('S3 Input: Object Zero Length', :key => log.key) elsif !sincedb.newer?(log.last_modified) @logger.debug('S3 Input: Object Not Modified', :key => log.key) elsif log.storage_class.start_with?('GLACIER') @logger.debug('S3 Input: Object Archived to Glacier', :key => log.key) else objects[log.key] = log.last_modified @logger.debug("S3 input: Adding to objects[]", :key => log.key) @logger.debug("objects[] length is: ", :length => objects.length) end end @logger.info('S3 input: No files found in bucket', :prefix => prefix) unless found rescue Aws::Errors::ServiceError => e @logger.error("S3 input: Unable to list objects in bucket", :prefix => prefix, :message => e.) end objects.keys.sort {|a,b| objects[a] <=> objects[b]} end |
#process_files(queue) ⇒ Object
171 172 173 174 175 176 177 178 179 180 181 182 |
# File 'lib/logstash/inputs/s3.rb', line 171 def process_files(queue) objects = list_new_files objects.each do |key| if stop? break else @logger.debug("S3 input processing", :bucket => @bucket, :key => key) process_log(queue, key) end end end |
#register ⇒ Object
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 |
# File 'lib/logstash/inputs/s3.rb', line 83 def register require "fileutils" require "digest/md5" require "aws-sdk-resources" @logger.info("Registering s3 input", :bucket => @bucket, :region => @region) s3 = get_s3object @s3bucket = s3.bucket(@bucket) unless @backup_to_bucket.nil? @backup_bucket = s3.bucket(@backup_to_bucket) begin s3.client.head_bucket({ :bucket => @backup_to_bucket}) rescue Aws::S3::Errors::NoSuchBucket s3.create_bucket({ :bucket => @backup_to_bucket}) end end unless @backup_to_dir.nil? Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir) end FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory) if !@watch_for_new_files && original_params.include?('interval') logger.warn("`watch_for_new_files` has been disabled; `interval` directive will be ignored.") end end |
#run(queue) ⇒ Object
115 116 117 118 119 120 121 |
# File 'lib/logstash/inputs/s3.rb', line 115 def run(queue) @current_thread = Thread.current Stud.interval(@interval) do process_files(queue) stop unless @watch_for_new_files end end |
#stop ⇒ Object
185 186 187 188 189 190 |
# File 'lib/logstash/inputs/s3.rb', line 185 def stop # @current_thread is initialized in the `#run` method, # this variable is needed because the `#stop` is a called in another thread # than the `#run` method and requiring us to call stop! with a explicit thread. Stud.stop!(@current_thread) end |