Class: RestFtpDaemon::Job

Inherits:
Object
  • Object
show all
Includes:
BmcDaemonLib::LoggerHelper, NewRelic::Agent::Instrumentation::ControllerInstrumentation, CommonHelpers
Defined in:
lib/rest-ftp-daemon/job.rb,
lib/rest-ftp-daemon/jobs/errors.rb

Direct Known Subclasses

JobDummy, JobTransfer, JobVideo

Constant Summary collapse

IMPORTED =

Fields to be imported from params

%w(type priority pool label priority source target overwrite notify mkdir tempfile video_options video_custom)
ERRORS =

Common errors

{
# oops_invalid_argument:         Errno::EINVAL,
oops_runtime_error:            RuntimeError,

job_timeout:              RestFtpDaemon::JobTimeout,
source_not_supported:     RestFtpDaemon::SourceUnsupported,
source_not_found:         RestFtpDaemon::SourceNotFound,
target_file_exists:       RestFtpDaemon::TargetFileExists,
target_directory_error:   RestFtpDaemon::TargetDirectoryError,
target_permission_error:  RestFtpDaemon::TargetPermissionError,
target_not_supported:     RestFtpDaemon::TargetUnsupported,
assertion_failed:         RestFtpDaemon::AssertionFailed,

conn_socket_error:        SocketError,
conn_eof:                 EOFError,
conn_failed:              Errno::ENOTCONN,
conn_host_is_down:        Errno::EHOSTDOWN,
conn_broken_pipe:         Errno::EPIPE,
conn_unreachable:         Errno::ENETUNREACH,
conn_reset_by_peer:       Errno::ECONNRESET,
conn_refused:             Errno::ECONNREFUSED,
conn_timed_out_1:         Timeout::Error,
conn_timed_out_2:         Net::ReadTimeout,
conn_timed_out_3:         Errno::ETIMEDOUT,

ftp_connection_error:     Net::FTPConnectionError,
ftp_perm_error:           Net::FTPPermError,
ftp_reply_error:          Net::FTPReplyError,
ftp_temp_error:           Net::FTPTempError,
ftp_proto_error:          Net::FTPProtoError,
ftp_error:                Net::FTPError,

sftp_exception:           Net::SFTP::StatusException,
sftp_key_mismatch:        Net::SSH::HostKeyMismatch,
sftp_auth_failed:         Net::SSH::AuthenticationFailed,
sftp_openssl_error:       OpenSSL::SSL::SSLError,

s3_no_such_waiter:        Aws::Waiters::Errors::NoSuchWaiterError,
s3_failure_state_error:   Aws::Waiters::Errors::FailureStateError,
s3_too_many_attempts:     Aws::Waiters::Errors::TooManyAttemptsError,
s3_waiter_unexpected:     Aws::Waiters::Errors::UnexpectedError,
s3_waiter_failed:         Aws::Waiters::Errors::WaiterFailed,

#s3_not_found:             Aws::S3::Errors::NotFound,
s3_permanent_redirect:    Aws::S3::Errors::PermanentRedirect,
s3_no_such_key:           Aws::S3::Errors::NoSuchKey,
s3_no_such_bucket:        Aws::S3::Errors::NoSuchBucket,
s3_no_such_upload:        Aws::S3::Errors::NoSuchUpload,
s3_error:                 Aws::S3::Errors::ServiceError,

video_missing_binary:     RestFtpDaemon::VideoMissingBinary,
video_not_found:          RestFtpDaemon::VideoNotFound,
video_movie_error:        RestFtpDaemon::VideoMovieError,
video_ffmpeg_error:       FFMPEG::Error,

# rescue Encoding::UndefinedConversionError => exception
#   return oops :ended, exception, "encoding_error", true
}

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from CommonHelpers

#dashboard_url, #exception_to_error, #format_bytes, #identifier, #underscore

Constructor Details

#initialize(job_id = nil, params = {}) ⇒ Job

Returns a new instance of Job.

Raises:



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/rest-ftp-daemon/job.rb', line 46

def initialize job_id = nil, params = {}
  # Minimal init
  @infos = {}
  @mutex = Mutex.new

  # Skip if no job_id passed or null (mock Job)
  return if job_id.nil?

  # Init context
  @id           = job_id.to_s
  @updated_at   = nil
  @error        = nil
  @status       = nil
  @tentatives   = 0
  @wid          = nil
  @created_at   = Time.now

  # Logger # FIXME: should be :jobs
  log_pipe      :transfer

  # Prepare configuration
  @config       = Conf[:transfer] || {}
  @endpoints    = Conf[:endpoints] || {}
  @pools        = Conf[:pools] || {}

  # Import query params
  set_info INFO_PARAMS, params
  IMPORTED.each do |field|
    instance_variable_set "@#{field}", params[field]
  end

  # Check if pool name exists
  @pool = DEFAULT_POOL unless @pools.keys.include?(@pool)

  # Prepare sources/target
  raise RestFtpDaemon::JobAttributeMissing, "source" unless params[:source]
  @source_loc = Location.new(params[:source])

  raise RestFtpDaemon::JobAttributeMissing, "target" unless params[:target]
  @target_loc = Location.new(params[:target])

  # We're done!
  log_info "initialized", {
    source: @source_loc.uri,
    target: @target_loc.uri,
    pool: @pool,
    }
end

Instance Attribute Details

#created_at ⇒ Object (readonly)

Returns the value of attribute created_at.



30
31
32
 
# File 'lib/rest-ftp-daemon/job.rb', line 30

def created_at
  @created_at
end

#created_since ⇒ Object (readonly)

Returns the value of attribute created_since.



35
36
37
 
# File 'lib/rest-ftp-daemon/job.rb', line 35

def created_since
  @created_since
end

#error ⇒ Object (readonly)

Returns the value of attribute error.



26
27
28
 
# File 'lib/rest-ftp-daemon/job.rb', line 26

def error
  @error
end

#finished_at ⇒ Object (readonly)

Returns the value of attribute finished_at.



33
34
35
 
# File 'lib/rest-ftp-daemon/job.rb', line 33

def finished_at
  @finished_at
end

#finished_in ⇒ Object (readonly)

Returns the value of attribute finished_in.



37
38
39
 
# File 'lib/rest-ftp-daemon/job.rb', line 37

def finished_in
  @finished_in
end

#id ⇒ Object (readonly)

Returns the value of attribute id.



25
26
27
 
# File 'lib/rest-ftp-daemon/job.rb', line 25

def id
  @id
end

#infos ⇒ Object (readonly)

Returns the value of attribute infos.



39
40
41
 
# File 'lib/rest-ftp-daemon/job.rb', line 39

def infos
  @infos
end

#source_loc ⇒ Object (readonly)

Returns the value of attribute source_loc.



22
23
24
 
# File 'lib/rest-ftp-daemon/job.rb', line 22

def source_loc
  @source_loc
end

#started_at ⇒ Object (readonly)

Returns the value of attribute started_at.



32
33
34
 
# File 'lib/rest-ftp-daemon/job.rb', line 32

def started_at
  @started_at
end

#started_since ⇒ Object (readonly)

Returns the value of attribute started_since.



36
37
38
 
# File 'lib/rest-ftp-daemon/job.rb', line 36

def started_since
  @started_since
end

#status ⇒ Object (readonly)

Returns the value of attribute status.



27
28
29
 
# File 'lib/rest-ftp-daemon/job.rb', line 27

def status
  @status
end

#target_loc ⇒ Object (readonly)

Returns the value of attribute target_loc.



23
24
25
 
# File 'lib/rest-ftp-daemon/job.rb', line 23

def target_loc
  @target_loc
end

#tentatives ⇒ Object (readonly)

Returns the value of attribute tentatives.



28
29
30
 
# File 'lib/rest-ftp-daemon/job.rb', line 28

def tentatives
  @tentatives
end

#updated_at ⇒ Object (readonly)

Returns the value of attribute updated_at.



31
32
33
 
# File 'lib/rest-ftp-daemon/job.rb', line 31

def updated_at
  @updated_at
end

#wid ⇒ Object

Class options



20
21
22
 
# File 'lib/rest-ftp-daemon/job.rb', line 20

def wid
  @wid
end

Instance Method Details

#after ⇒ Object 



161
162
 
# File 'lib/rest-ftp-daemon/job.rb', line 161

def after
end

#before ⇒ Object 



157
158
 
# File 'lib/rest-ftp-daemon/job.rb', line 157

def before
end

#get_info(name) ⇒ Object 



203
204
205
206
207
 
# File 'lib/rest-ftp-daemon/job.rb', line 203

def get_info name
  @mutex.synchronize do
    @infos[name]
  end
end

#oops_end(what, exception) ⇒ Object 



193
194
195
196
 
# File 'lib/rest-ftp-daemon/job.rb', line 193

def oops_end what, exception
  Rollbar.error exception, "oops_end [#{what}]: #{exception.class.name}: #{exception.message}"
  oops :ended, exception, what
end

#reset ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/rest-ftp-daemon/job.rb', line 95

def reset
  # Update job status
  set_status JOB_STATUS_PREPARING

  # Increment run cours
  @tentatives +=1
  @updated_at = Time.now
  @started_at   = nil
  @finished_at  = nil

  # Job has been prepared, reset infos
  set_status JOB_STATUS_PREPARED
  @infos = {}

  # Update job status, send first notification
  set_status JOB_STATUS_QUEUED
  set_error nil
  client_notify :queued
  log_info "reset notify[queued] tentative[#{@tentatives}]"
end

#set_info(name, value) ⇒ Object 



209
210
211
212
213
214
215
 
# File 'lib/rest-ftp-daemon/job.rb', line 209

def set_info name, value
  @mutex.synchronize do
    @infos || {}
    @infos[name] = debug_value_utf8(value)
  end
  touch_job
end

#source_uri ⇒ Object 



164
165
166
 
# File 'lib/rest-ftp-daemon/job.rb', line 164

def source_uri
  @source_loc.uri if @source_loc
end

#start ⇒ Object

Process job



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/rest-ftp-daemon/job.rb', line 117

def start
  # Check prerequisites
  raise RestFtpDaemon::AssertionFailed, "run/source_loc" unless @source_loc
  raise RestFtpDaemon::AssertionFailed, "run/target_loc" unless @target_loc

  # Remember when we started
  @started_at = Time.now

  # Notify we start working
  log_info "client_notify [started]"
  current_signal = :started
  set_status JOB_STATUS_WORKING
  client_notify :started

  # Before work
  log_debug "do_before"
  current_signal = :started
  do_before

  # Do the hard work
  log_debug "do_work"
  current_signal = :ended
  do_work

  # Finalize all this
  log_debug "do_after"
  current_signal = :ended
  do_after

rescue StandardError => exception
  Rollbar.error exception, "job [#{error}]: #{exception.class.name}: #{exception.message}"
  return oops current_signal, exception

else
  # All done !
  set_status JOB_STATUS_FINISHED
  log_info "client_notify [ended]"
  client_notify :ended
end

#target_uri ⇒ Object 



168
169
170
 
# File 'lib/rest-ftp-daemon/job.rb', line 168

def target_uri
  @target_loc.uri if @target_loc
end

#targethost ⇒ Object 



198
199
200
201
 
# File 'lib/rest-ftp-daemon/job.rb', line 198

def targethost
  @target_loc.host unless @target_loc.nil?
  #get_info :target_host
end

#weight ⇒ Object 



172
173
174
175
176
177
178
 
# File 'lib/rest-ftp-daemon/job.rb', line 172

def weight
  @weight = [
    - @tentatives.to_i,
    + @priority.to_i,
    - @created_at.to_i,
    ]
end

#work ⇒ Object 



159
160
 
# File 'lib/rest-ftp-daemon/job.rb', line 159

def work
end