Module: Dapp::Dimg::Dapp::Command::CleanupRepo

Included in:
Dapp
Defined in:
lib/dapp/dimg/dapp/command/cleanup_repo.rb

Constant Summary collapse

GIT_TAGS_LIMIT_POLICY = 
10
EXPIRY_DATE_PERIOD_POLICY = 
60 * 60 * 24 * 30
GIT_COMMITS_LIMIT_POLICY = 
50
GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY = 
60 * 60 * 24 * 30

Instance Method Summary collapse

Instance Method Details

#actual_detailed_dimgs_images_by_scheme(registry) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 36

def actual_detailed_dimgs_images_by_scheme(registry)
  {}.tap do |detailed_dimgs_images_by_scheme|
    tagging_schemes.each { |scheme| detailed_dimgs_images_by_scheme[scheme] = [] }
    repo_detailed_dimgs_images(registry).each do |image|
      image_repository = [option_repo, image[:dimg]].compact.join('/')
      image_name = [image_repository, image[:tag]].join(':')

      should_be_ignored = deployed_docker_images.include?(image_name)

      if should_be_ignored
        log "Keep in repo image that is being used in kubernetes: #{image_name}"
        next
      end

      (detailed_dimgs_images_by_scheme[image[:labels]['dapp-tag-scheme']] ||= []) << image
    end
  end
end

#cleanup_repo ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 11

def cleanup_repo
  lock_repo(repo = option_repo) do
    log_step_with_indent(repo) do
      log_step_with_indent('Searching for images being used in kubernetes clusters in one of the kube-config contexts') do
        deployed_docker_images.each do |deployed_img|
          log(deployed_img)
        end
      end

      registry = dimg_registry(repo)

      if git_own_repo_exist?
        cleanup_repo_by_nonexistent_git_primitive(registry, actual_detailed_dimgs_images_by_scheme(registry))
        cleanup_repo_by_policies(registry, actual_detailed_dimgs_images_by_scheme(registry))
      end

      begin
        repo_dimgs      = repo_dimgs_images(registry)
        repo_dimgstages = repo_dimgstages_images(registry)
        repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages)
      end if with_stages?
    end
  end
end

#cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme) ⇒ Object 



77
78
79
80
81
82
83
84
85
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 77

def cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme)
  nonexist_repo_images = repo_dimgs_images_by_scheme[dapp_tag_scheme]
    .select { |dimg_image| dimg_image[:labels]['dapp-tag-scheme'] == dapp_tag_scheme }
    .select { |dimg_image| !(yield dimg_image) }

  log_step_with_indent(:"#{dapp_tag_scheme.split('_').join(' ')} nonexistent") do
    nonexist_repo_images.each { |repo_image| delete_repo_image(registry, repo_image) }
  end unless nonexist_repo_images.empty?
end

#cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 55

def cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme)
  %w(git_tag git_branch git_commit).each do |scheme|
    cleanup_repo_by_nonexistent_git_base(registry, detailed_dimgs_images_by_scheme, scheme) do |detailed_dimg_image|
      case scheme
        when 'git_tag'    then consistent_git_tags.include?(detailed_dimg_image[:tag])
        when 'git_branch' then consistent_git_remote_branches.include?(detailed_dimg_image[:tag])
        when 'git_commit' then git_own_repo.commit_exists?(detailed_dimg_image[:tag])
        else
          raise
      end
    end unless detailed_dimgs_images_by_scheme[scheme].empty?
  end
end

#cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object 



87
88
89
90
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 87

def cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme)
  cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme)
  cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme)
end

#cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:) ⇒ Object 



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 112

def cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:)
  sorted_detailed_dimgs_images = detailed_dimgs_images.sort_by { |dimg| dimg[:created_at] }.reverse

  expired_dimgs_images, not_expired_dimgs_images = sorted_detailed_dimgs_images.partition do |dimg_image|
    dimg_image[:created_at] < expiry_date_policy
  end

  log_step_with_indent(:"git #{log_primitive} date policy (before #{DateTime.strptime(expiry_date_policy.to_s, '%s')})") do
    expired_dimgs_images.each { |dimg| delete_repo_image(registry, dimg) }
  end unless expired_dimgs_images.empty?

  not_expired_dimgs_images
    .each_with_object({}) { |dimg, images_by_dimg| (images_by_dimg[dimg[:dimg]] ||= []) << dimg }
    .each do |dimg_name, images|
    next if images[limit_policy..-1].nil?
    log_step_with_indent(:"git #{log_primitive} limit policy (> #{limit_policy}) (`#{dimg_name || 'nameless'}` dimg)") do
      images[limit_policy..-1].each { |dimg| delete_repo_image(registry, dimg) }
    end
  end
end

#cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object 



102
103
104
105
106
107
108
109
110
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 102

def cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme)
  cleanup_repo_by_policies_base(
    registry,
    detailed_dimgs_images_by_scheme['git_commit'].select { |dimg| git_own_repo.commit_exists?(dimg[:tag]) },
    expiry_date_policy: git_commits_expiry_date_policy,
    limit_policy:       git_commits_limit_policy,
    log_primitive:      'commit'
  )
end

#cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object 



92
93
94
95
96
97
98
99
100
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 92

def cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme)
  cleanup_repo_by_policies_base(
    registry,
    detailed_dimgs_images_by_scheme['git_tag'].select { |dimg| consistent_git_tags.include?(dimg[:tag]) },
    expiry_date_policy: git_tags_expiry_date_policy,
    limit_policy:       git_tags_limit_policy,
    log_primitive:      'tag'
  )
end

#consistent_git_remote_branches ⇒ Object 



73
74
75
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 73

def consistent_git_remote_branches
  @consistent_git_remote_branches ||= git_own_repo.remote_branches.map(&method(:consistent_uniq_slugify))
end

#consistent_git_tags ⇒ Object 



69
70
71
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 69

def consistent_git_tags
  git_tag_by_consistent_tag_name.keys
end

#cronjob_images(client) ⇒ Object

cronjob items[] spec jobTemplate spec template spec containers[] image



233
234
235
236
237
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 233

def cronjob_images(client)
  client.cronjob_list['items'].map do |item|
    item['spec']['jobTemplate']['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#daemonset_images(client) ⇒ Object

daemonsets items[] spec template spec containers[] image



240
241
242
243
244
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 240

def daemonset_images(client)
  client.daemonset_list['items'].map do |item|
    item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#deployed_docker_images ⇒ Object 



173
174
175
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 173

def deployed_docker_images
  @deployed_docker_images ||= search_deployed_docker_images
end

#deployment_images(client) ⇒ Object

deployment items[] spec template spec containers[] image



247
248
249
250
251
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 247

def deployment_images(client)
  client.deployment_list['items'].map do |item|
    item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#expiry_date_policy_value(env_key, default:) ⇒ Object 



149
150
151
152
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 149

def expiry_date_policy_value(env_key, default:)
  expiry_date_period_policy = policy_value(env_key, default: default)
  Time.now.to_i - expiry_date_period_policy
end

#git_commits_expiry_date_policy ⇒ Object 



141
142
143
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 141

def git_commits_expiry_date_policy
  @git_commits_expiry_date_policy ||= expiry_date_policy_value('GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY', default: GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY)
end

#git_commits_limit_policy ⇒ Object 



145
146
147
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 145

def git_commits_limit_policy
  @git_commits_limit_policy ||= policy_value('GIT_COMMITS_LIMIT_POLICY', default: GIT_COMMITS_LIMIT_POLICY)
end

#git_tag_by_consistent_git_tag(consistent_git_tag) ⇒ Object 



165
166
167
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 165

def git_tag_by_consistent_git_tag(consistent_git_tag)
  git_tag_by_consistent_tag_name[consistent_git_tag]
end

#git_tag_by_consistent_tag_name ⇒ Object 



169
170
171
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 169

def git_tag_by_consistent_tag_name
  @git_consistent_tags ||= git_own_repo.tags.map { |t| [consistent_uniq_slugify(t), t] }.to_h
end

#git_tags_expiry_date_policy ⇒ Object 



133
134
135
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 133

def git_tags_expiry_date_policy
  @git_tags_expiry_date_policy ||= expiry_date_policy_value('EXPIRY_DATE_PERIOD_POLICY', default: EXPIRY_DATE_PERIOD_POLICY)
end

#git_tags_limit_policy ⇒ Object 



137
138
139
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 137

def git_tags_limit_policy
  @git_tags_limit_policy ||= policy_value('GIT_TAGS_LIMIT_POLICY', default: GIT_TAGS_LIMIT_POLICY)
end

#job_images(client) ⇒ Object

job items[] spec template spec containers[] image



254
255
256
257
258
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 254

def job_images(client)
  client.job_list['items'].map do |item|
    item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#pod_images(client) ⇒ Object

pod items[] spec containers[] image



226
227
228
229
230
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 226

def pod_images(client)
  client.pod_list['items'].map do |item|
    item['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#policy_value(env_key, default:) ⇒ Object 



154
155
156
157
158
159
160
161
162
163
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 154

def policy_value(env_key, default:)
  return default if (val = ENV[env_key]).nil?

  if val.to_i.to_s == val
    val.to_i
  else
    log_warning("WARNING: `#{env_key}` value `#{val}` is ignored (using default value `#{default}`)!")
    default
  end
end

#replicaset_images(client) ⇒ Object

replicasets items[] spec template spec containers[] image



261
262
263
264
265
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 261

def replicaset_images(client)
  client.replicaset_list['items'].map do |item|
    item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#replicationcontroller_images(client) ⇒ Object

replicationcontroller items[] spec template spec containers[] image



275
276
277
278
279
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 275

def replicationcontroller_images(client)
  client.replicationcontroller_list['items'].map do |item|
    item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#search_deployed_docker_images ⇒ Object 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 177

def search_deployed_docker_images
  return [] if without_kube?

  config = ::Dapp::Kube::Kubernetes::Config.new_auto_if_available
  return [] if config.nil?

  config.context_names.
    map {|context_name|
      client = ::Dapp::Kube::Kubernetes::Client.new(
        config,
        context_name,
        "default",
        timeout: options[:kubernetes_timeout],
      )
      search_deployed_docker_images_from_kube(client)
    }.flatten.sort.uniq
end

#search_deployed_docker_images_from_kube(client) ⇒ Object 



195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 195

def search_deployed_docker_images_from_kube(client)
  namespaces = []
  # check connectivity for 2 seconds
  begin
    namespaces = client.namespace_list(excon_parameters: {:connect_timeout => 30})
  rescue Excon::Error::Timeout
    raise ::Dapp::Error::Default, code: :kube_connect_timeout
  end

  # get images from containers from pods from all namespaces.
  namespaces['items'].map do |item|
    item['metadata']['name']
  end.map do |ns|
    [].tap do |arr|
      client.with_namespace(ns) do
        arr << pod_images(client)
        arr << deployment_images(client)
        arr << replicaset_images(client)
        arr << statefulset_images(client)
        arr << daemonset_images(client)
        arr << job_images(client)
        arr << cronjob_images(client)
        arr << replicationcontroller_images(client)
      end
    end
  end.flatten.select do |img|
    img.start_with? option_repo
  end.sort.uniq
end

#statefulset_images(client) ⇒ Object

replicasets items[] spec template spec containers[] image



268
269
270
271
272
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 268

def statefulset_images(client)
  client.statefulset_list['items'].map do |item|
    item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] }
  end
end

#without_kube? ⇒ Boolean

Returns:

  • (Boolean) 


281
282
283
 
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 281

def without_kube?
  !!options[:without_kube]
end