41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
# File 'lib/dapp/kube/kubernetes/manager/deployment.rb', line 41
def watch_till_ready!
dapp.log_process("Watch deployment '#{name}' till ready") do
known_events_by_pod = {}
known_log_timestamps_by_pod_and_container = {}
d = @deployment_before_deploy || Kubernetes::Client::Resource::Deployment.new(dapp.kubernetes.deployment(name))
loop do
d_revision = d.annotations['deployment.kubernetes.io/revision']
dapp.log_step("[#{Time.now}] Poll deployment '#{d.name}' status")
dapp.with_log_indent do
dapp.log_info("Replicas: #{_field_value_for_log(d.status['replicas'])}")
dapp.log_info("Updated replicas: #{_field_value_for_log(d.status['updatedReplicas'])}")
dapp.log_info("Available replicas: #{_field_value_for_log(d.status['availableReplicas'])}")
dapp.log_info("Unavailable replicas: #{_field_value_for_log(d.status['unavailableReplicas'])}")
dapp.log_info("Ready replicas: #{_field_value_for_log(d.status['readyReplicas'])} / #{_field_value_for_log(d.replicas)}")
end
rs = nil
if d_revision
rs = dapp.kubernetes.replicaset_list['items']
.map {|spec| Kubernetes::Client::Resource::Replicaset.new(spec)}
.select do |_rs|
Array(_rs.metadata['ownerReferences']).any? do |owner_reference|
owner_reference['uid'] == d.metadata['uid']
end
end
.select do |_rs|
rs_revision = _rs.annotations['deployment.kubernetes.io/revision']
(rs_revision and (d_revision == rs_revision))
end
.sort_by do |_rs|
if creation_timestamp = _rs.metadata['creationTimestamp']
Time.parse(creation_timestamp)
else
Time.now
end
end.first
end
if rs
dapp.with_log_indent do
dapp.log_step("Current ReplicaSet '#{rs.name}' status")
dapp.with_log_indent do
dapp.log_info("Replicas: #{_field_value_for_log(rs.status['replicas'])}")
dapp.log_info("Fully labeled replicas: #{_field_value_for_log(rs.status['fullyLabeledReplicas'])}")
dapp.log_info("Available replicas: #{_field_value_for_log(rs.status['availableReplicas'])}")
dapp.log_info("Ready replicas: #{_field_value_for_log(rs.status['readyReplicas'])} / #{_field_value_for_log(d.replicas)}")
end
end
rs_pods = dapp.kubernetes.pod_list['items']
.map {|spec| Kubernetes::Client::Resource::Pod.new(spec)}
.select do |pod|
Array(pod.metadata['ownerReferences']).any? do |owner_reference|
owner_reference['uid'] == rs.metadata['uid']
end
end
dapp.with_log_indent do
dapp.log_step("Pods:") if rs_pods.any?
rs_pods.each do |pod|
dapp.with_log_indent do
dapp.log_step(pod.name)
known_events_by_pod[pod.name] ||= []
pod_events = dapp.kubernetes
.event_list(fieldSelector: "involvedObject.uid=#{pod.uid}")['items']
.map {|spec| Kubernetes::Client::Resource::Event.new(spec)}
.reject do |event|
known_events_by_pod[pod.name].include? event.uid
end
if pod_events.any?
dapp.with_log_indent do
dapp.log_step("Last events:")
pod_events.each do |event|
dapp.with_log_indent do
dapp.log_info("[#{event.metadata['creationTimestamp']}] #{event.spec['message']}")
end
known_events_by_pod[pod.name] << event.uid
end
end
end
dapp.with_log_indent do
pod.containers_names.each do |container_name|
known_log_timestamps_by_pod_and_container[pod.name] ||= {}
known_log_timestamps_by_pod_and_container[pod.name][container_name] ||= Set.new
since_time = nil
if [nil, "True"].include? pod.ready_condition_status
since_time = @deploy_began_at.utc.iso8601(9) if @deploy_began_at
end
log_lines_by_time = []
begin
log_lines_by_time = dapp.kubernetes.pod_log(pod.name, container: container_name, timestamps: true, sinceTime: since_time)
.lines.map(&:strip)
.map {|line|
timestamp, _, data = line.partition(' ')
unless known_log_timestamps_by_pod_and_container[pod.name][container_name].include? timestamp
known_log_timestamps_by_pod_and_container[pod.name][container_name].add timestamp
[timestamp, data]
end
}.compact
rescue Kubernetes::Client::Error::Pod::ContainerCreating, Kubernetes::Client::Error::Pod::PodInitializing
next
rescue Kubernetes::Client::Error::Base => err
dapp.log_warning("#{dapp.log_time}Error while fetching pod's #{pod.name} logs: #{err.message}", stream: dapp.service_stream)
next
end
if log_lines_by_time.any?
dapp.log_step("Last container '#{container_name}' log:")
dapp.with_log_indent do
log_lines_by_time.each do |timestamp, line|
dapp.log("[#{timestamp}] #{line}")
end
end
end
end
end
pod_manager = Kubernetes::Manager::Pod.new(dapp, pod.name)
pod_manager.check_readiness_condition_if_available!(pod)
end end end end
if d_revision && d.replicas && d.replicas == 0
break
end
if d_revision && d.replicas && rs
break if is_deployment_ready(d) && is_replicaset_ready(d, rs)
end
sleep 5
d = Kubernetes::Client::Resource::Deployment.new(dapp.kubernetes.deployment(d.name))
end
end
end
|