5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
# File 'lib/ralf/option_parser.rb', line 5
def self.parse(args, output = $stdout)
options = {}
opts = ::OptionParser.new do |opts|
opts.banner = <<USAGE_END
Usage: #{$0} [options]
Download and merge Amazon S3 bucket log files for a specified date range and
output a Common Log File. Ralf is an acronym for Retrieve Amazon Log Files.
Ralf downloads bucket log files to local cache directories, merges the Amazon Log
Files and converts them to Common Log Format.
Example: #{$0} --range month --now yesterday --output-file '/var/log/amazon/:year/:month/:bucket.log'
AWS credentials (Access Key Id and Secret Access Key) are required to access
S3 buckets. For security reasons these credentials can only be specified in a
configuration file (see --config-file) or through the environment using the
AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
USAGE_END
opts.separator ""
opts.separator "Log selection options:"
opts.on("-l", "--[no-]list", "List buckets that have logging enabled. Does not process log files.") do |value|
options[:list] = value
end
opts.on("-b", "--buckets x,y,z", Array, "Buckets for which to process log files. Defaults to all log-enabled buckets.") do |buckets|
options[:buckets] = buckets.compact
end
opts.on("-r", "--range BEGIN[,END]", Array, "Date or date range to process. Defaults to 'today'.") do |range|
options[:range] = range.compact
end
log_selection_help =<<LOG_SELECTION_HELP
Date to use as base for range. Defaults to 'today'.
You can use Chronic expressions for '--range' and '--now'. See http://chronic.rubyforge.org.
Example: --range 'last week'
All days of previous week.
Example: --range 'this week'
Beginning of this week (sunday) upto and including today.
Example: --range '2010-01-01','2010-04-30'
First four months of this year.
Example: --range 'this month' --now yesterday
This will select log files from the beginning of yesterday's month upto and including yesterday.
The --buckets, --range and --now options are optional. If unspecified, (incomplete)
logging for today will be processed for all buckets (that have logging enabled).
This is equivalent to specifying "--range 'today'" and "--now 'today'".
LOG_SELECTION_HELP
opts.on("-t", "--now TIME", log_selection_help) do |now|
options[:now] = now
end
opts.separator "Output options:"
output_file_help =<<OUTPUT_FILE_HELP
Output file, e.g. '/var/log/s3/:year/:month/:bucket.log'. Required.
The --output-file format uses the last day of the range specified by (--range)
to determine the filename. E.g. when the format contains ':year/:month/:day' and
the range is 2010-01-15..2010-02-14, then the output file will be '2010/02/14'.
OUTPUT_FILE_HELP
opts.on("-o", "--output-file FORMAT", output_file_help) do |format|
options[:output_file] = format
end
cache_dir_help =<<CACHE_DIR_HELP
Directory name(s) in which to cache downloaded log files. Optional.
The --cache-dir format expands to as many directory names as needed for the
range specified by --range. E.g. "/var/run/s3_cache/:year/:month/:day/:bucket"
expands to 31 directories for range 2010-01-01..2010-01-31.
Defaults to '~/.ralf/:bucket' or '/var/log/ralf/:bucket' (when running as root).
CACHE_DIR_HELP
opts.on("-x", "--cache-dir FORMAT", cache_dir_help) do |format|
options[:cache_dir] = format
end
opts.separator "Config file options:"
config_file_help =<<CONFIG_FILE_HELP
Path to file with configuration settings (in YAML format).
Configuration settings are read from the (-c) specified configuration file
or from ~/.ralf.conf or from /etc/ralf.conf (when running as root).
Command-line options override settings read from the configuration file.
The configuration file must be in YAML format. Each command-line options has an
equivalent setting in a configuration file replacing dash (-) by underscore(_).
The Amazon Access Key Id and Secret Access Key can only be specified in the
Example:
output_file: /var/log/amazon_s3/:year:month/:bucket.log
aws_access_key_id: my_access_key_id
aws_secret_access_key: my_secret_access_key
To only use command-line options simply specify -c or --config-file without
an argument.
CONFIG_FILE_HELP
opts.on("-c", "--config-file [FILE]", config_file_help) do |file|
options[:config_file] = file
end
opts.separator "Debug options:"
opts.on("-d", "--[no-]debug [aws]", "Show debug messages.") do |aws|
options[:debug] = aws || true
end
opts.separator ""
opts.separator "Common options:"
opts.on_tail("-h", "--help", "Show this message.") do
output.puts opts
return nil
end
opts.on_tail("-v", "--version", "Show version.") do
output.print File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION'))
return nil
end
end
remaining = opts.parse!(args)
opts.warn "Warning: unused arguments: #{remaining.join(' ')}" unless remaining.empty?
options
end
|