9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
# File 'lib/s3-object-processor/cli.rb', line 9
def initialize(&config)
@reports = {}
instance_eval &config
cli_setup = @cli_setup
cli_process_setup = @cli_process_setup
settings = ::CLI.new do
description 'Set header of S3 object'
option :key_id,
short: :i,
description: 'AWS access key ID',
default_label: 'AWS_SECRET_KEY_ID environment variable',
default: ENV['AWS_ACCESS_KEY_ID'],
required: true
option :key_secret,
short: :s,
description: 'AWS access key secret',
default_label: 'AWS_SECRET_ACCESS_KEY environment variable',
default: ENV['AWS_SECRET_ACCESS_KEY'],
required: true
switch :no_https,
description: 'use plain HTTP S3 connections'
option :bucket,
short: :b,
description: 'bucket to process',
required: true
option :prefix,
short: :p,
description: 'process only objects of key starting with given prefix'
option :lister_fetch_size,
description: 'fetch no more that that number of keys per request',
cast: Integer,
default: 200
option :lister_backlog,
description: 'maximum length of to be processed key queue',
cast: Integer,
default: 1000
option :key_list,
description: 'file with keys to process (one per line)',
default_label: 'process all keys in S3 bucket',
cast: Pathname
option :reporter_backlog,
description: 'maximum length of to be processed report queue',
cast: Integer,
default: 1000
option :reporter_summary_interval,
description: 'pring summary every some number of processed objects',
cast: Integer,
default: 100
option :reporter_average_contribution,
description: 'how much does last average calculation contribute in the printed value - less => more stable',
cast: Float,
default: 0.10
option :workers,
short: :t,
description: 'number of processing threads to start',
cast: Integer,
default: 10
switch :noop,
short: :n,
description: 'do not change any object; just say what would be done'
switch :debug,
short: :d,
description: 'log at DEBUG level'
option :max_keys,
description: 'stop after processing this amout of keys',
cast: Integer
instance_eval &cli_setup if cli_setup
end.parse! do |settings|
instance_eval &cli_process_setup if cli_process_setup
end
log = Logger.new(STDERR)
log.level = settings.debug ? Logger::DEBUG : Logger::INFO
log.debug(settings.inspect)
trap 'QUIT' do
Thread.list.each do |thread|
STDERR.puts "Thread-#{thread.object_id.to_s(36)}"
STDERR.puts thread.backtrace.join("\n \\_ ")
end
end
BucketProcessor.new(settings.key_id, settings.key_secret, settings.bucket,
no_https: settings.no_https,
log: log,
workers: settings.workers,
max_keys: settings.max_keys,
lister_fetch_size: settings.lister_fetch_size,
lister_backlog: settings.lister_backlog,
key_list: settings.key_list && settings.key_list.readlines.map(&:strip),
reporter_backlog: settings.reporter_backlog,
reporter_summary_interval: settings.reporter_summary_interval,
reporter_average_contribution: settings.reporter_average_contribution,
reports: @reports
) do |bucket, key, reporter|
@processor.call(bucket, key, settings, log, reporter)
end
.run(settings.prefix)
end
|