82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
# File 'lib/dapp/kube/dapp/command/deploy.rb', line 82
def kube_run_deploy(release)
log_process("Deploy release #{release.name}") do
helm_status_res = shellout("helm status #{release.name}")
release_status = nil
if helm_status_res.status.success?
status_line = helm_status_res.stdout.lines.find {|l| l.start_with? "STATUS: "}
release_status = status_line.partition(": ")[2].strip if status_line
end
release_exists = nil
if not helm_status_res.status.success?
release_exists = false
kube_create_helm_auto_purge_trigger_file(release.name)
elsif ["FAILED", "PENDING_INSTALL"].include? release_status
release_exists = true
if File.exists? kube_helm_auto_purge_trigger_file_path(release.name)
log_process("Purge helm release #{release.name}") do
shellout!("helm delete --purge #{release.name}")
end
release_exists = false
end
else
if File.exists? kube_helm_auto_purge_trigger_file_path(release.name)
log_warning "[WARN] Will not purge helm release #{release.name}: expected FAILED or PENDING_INSTALL release status, got #{release_status}"
end
release_exists = true
kube_delete_helm_auto_purge_trigger_file(release.name)
end
kube_flush_hooks_jobs(release)
watch_hooks_by_type = release.jobs.values
.reduce({}) do |res, job|
if job.annotations['dapp/watch-logs'].to_s == 'true'
job.annotations['helm.sh/hook'].to_s.split(',').each do |hook_type|
res[hook_type] ||= []
res[hook_type] << job
end
end
res
end
.tap do |res|
res.values.each do |jobs|
jobs.sort_by! {|job| job.annotations['helm.sh/hook-weight'].to_i}
end
end
watch_hooks = if release_exists
watch_hooks_by_type['pre-upgrade'].to_a + watch_hooks_by_type['post-upgrade'].to_a
else
watch_hooks_by_type['pre-install'].to_a + watch_hooks_by_type['post-install'].to_a
end
watch_hooks_thr = nil
watch_hooks_condition_mutex = ::Mutex.new
watch_hooks_condition = ::ConditionVariable.new
deploy_has_began = false
unless dry_run? and watch_hooks.any?
watch_hooks_thr = Thread.new do
watch_hooks_condition_mutex.synchronize do
while not deploy_has_began do
watch_hooks_condition.wait(watch_hooks_condition_mutex)
end
end
begin
watch_hooks.each do |job|
begin
Kubernetes::Manager::Job.new(self, job.name).watch_till_done!
rescue ::Exception => e
sentry_exception(e, extra: {"job-spec" => job.spec})
raise
end
end rescue Kubernetes::Error::Default => e
$stderr.puts(::Dapp::Dapp.paint_string(::Dapp::Helper::NetStatus.message(e), :warning))
end
end end
deployment_managers = release.deployments.values
.map {|deployment| Kubernetes::Manager::Deployment.new(self, deployment.name)}
deployment_managers.each(&:before_deploy)
log_process("#{release_exists ? "Upgrade" : "Install"} helm release #{release.name}") do
watch_hooks_condition_mutex.synchronize do
deploy_has_began = true
watch_hooks_condition.signal
end
cmd_res = if release_exists
release.upgrade_helm_release
else
release.install_helm_release
end
if cmd_res.error?
if cmd_res.stderr.end_with? "has no deployed releases\n"
log_warning "[WARN] Helm release #{release.name} is in improper state: #{cmd_res.stderr}"
log_warning "[WARN] Helm release #{release.name} will be removed with `helm delete --purge` on the next run of `dapp kube deploy`"
kube_create_helm_auto_purge_trigger_file(release.name)
end
raise ::Dapp::Error::Command, code: :kube_helm_failed, data: {output: (cmd_res.stdout + cmd_res.stderr).strip}
else
kube_delete_helm_auto_purge_trigger_file(release.name)
watch_hooks_thr.join if !dry_run? && watch_hooks_thr && watch_hooks_thr.alive?
log_info((cmd_res.stdout + cmd_res.stderr).strip)
end
end
deployment_managers.each(&:after_deploy)
unless dry_run?
begin
::Timeout::timeout(self.options[:timeout] || 300) do
deployment_managers.each {|deployment_manager|
if deployment_manager.should_watch?
deployment_manager.watch_till_ready!
end
}
end
rescue ::Timeout::Error
raise ::Dapp::Error::Command, code: :kube_deploy_timeout
end
end
end
end
|