Class: LogStash::Inputs::S3
- Inherits:
-
Base
- Object
- Base
- LogStash::Inputs::S3
- Includes:
- PluginMixins::AwsConfig::V2
- Defined in:
- lib/logstash/inputs/s3.rb
Overview
Files ending in ‘.gz` are handled as gzip’ed files.
Defined Under Namespace
Modules: SinceDB
Instance Method Summary collapse
-
#backup_to_bucket(object) ⇒ Object
def fetch_new_files.
- #backup_to_dir(filename) ⇒ Object
-
#list_new_files ⇒ Object
def run.
- #process_files(queue) ⇒ Object
- #register ⇒ Object
- #run(queue) ⇒ Object
-
#stop ⇒ Object
def process_files.
Instance Method Details
#backup_to_bucket(object) ⇒ Object
def fetch_new_files
155 156 157 158 159 160 161 162 163 |
# File 'lib/logstash/inputs/s3.rb', line 155 def backup_to_bucket(object) unless @backup_to_bucket.nil? backup_key = "#{@backup_add_prefix}#{object.key}" @backup_bucket.object(backup_key).copy_from(:copy_source => "#{object.bucket_name}/#{object.key}") if @delete object.delete() end end end |
#backup_to_dir(filename) ⇒ Object
165 166 167 168 169 |
# File 'lib/logstash/inputs/s3.rb', line 165 def backup_to_dir(filename) unless @backup_to_dir.nil? FileUtils.cp(filename, @backup_to_dir) end end |
#list_new_files ⇒ Object
def run
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
# File 'lib/logstash/inputs/s3.rb', line 128 def list_new_files objects = {} found = false begin @s3bucket.objects(:prefix => @prefix).each do |log| found = true @logger.debug('Found key', :key => log.key) if ignore_filename?(log.key) @logger.debug('Ignoring', :key => log.key) elsif log.content_length <= 0 @logger.debug('Object Zero Length', :key => log.key) elsif !sincedb.newer?(log.last_modified) @logger.debug('Object Not Modified', :key => log.key) elsif (log.storage_class == 'GLACIER' || log.storage_class == 'DEEP_ARCHIVE') && !file_restored?(log.object) @logger.debug('Object Archived to Glacier', :key => log.key) else objects[log.key] = log.last_modified @logger.debug("Added to objects[]", :key => log.key, :length => objects.length) end end @logger.info('No files found in bucket', :prefix => prefix) unless found rescue Aws::Errors::ServiceError => e @logger.error("Unable to list objects in bucket", :exception => e.class, :message => e., :backtrace => e.backtrace, :prefix => prefix) end objects.keys.sort {|a,b| objects[a] <=> objects[b]} end |
#process_files(queue) ⇒ Object
171 172 173 174 175 176 177 178 179 180 181 |
# File 'lib/logstash/inputs/s3.rb', line 171 def process_files(queue) objects = list_new_files objects.each do |key| if stop? break else process_log(queue, key) end end end |
#register ⇒ Object
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
# File 'lib/logstash/inputs/s3.rb', line 89 def register require "fileutils" require "digest/md5" require "aws-sdk-resources" @logger.info("Registering", :bucket => @bucket, :region => @region) s3 = get_s3object @s3bucket = s3.bucket(@bucket) unless @backup_to_bucket.nil? @backup_bucket = s3.bucket(@backup_to_bucket) begin s3.client.head_bucket({ :bucket => @backup_to_bucket}) rescue Aws::S3::Errors::NoSuchBucket s3.create_bucket({ :bucket => @backup_to_bucket}) end end unless @backup_to_dir.nil? Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir) end FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory) if !@watch_for_new_files && original_params.include?('interval') logger.warn("`watch_for_new_files` has been disabled; `interval` directive will be ignored.") end end |
#run(queue) ⇒ Object
120 121 122 123 124 125 126 |
# File 'lib/logstash/inputs/s3.rb', line 120 def run(queue) @current_thread = Thread.current Stud.interval(@interval) do process_files(queue) stop unless @watch_for_new_files end end |
#stop ⇒ Object
def process_files
183 184 185 186 187 188 |
# File 'lib/logstash/inputs/s3.rb', line 183 def stop # @current_thread is initialized in the `#run` method, # this variable is needed because the `#stop` is a called in another thread # than the `#run` method and requiring us to call stop! with a explicit thread. Stud.stop!(@current_thread) end |