Module: Pkg::Util::Ship

Defined in:
lib/packaging/util/ship.rb

Class Method Summary collapse

Class Method Details

.collect_packages(pkg_exts, excludes = []) ⇒ Object

rubocop:disable Metrics/MethodLength



6
7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/packaging/util/ship.rb', line 6

def collect_packages(pkg_exts, excludes = []) # rubocop:disable Metrics/MethodLength
  pkgs = pkg_exts.map { |ext| Dir.glob(ext) }.flatten
  return [] if pkgs.empty?
  excludes.each do |exclude|
    pkgs.delete_if { |p| p.match(exclude) }
  end if excludes
  if pkgs.empty?
    $stdout.puts "No packages with (#{pkg_exts.join(', ')}) extensions found staged in 'pkg'"
    $stdout.puts "Maybe your excludes argument (#{excludes}) is too restrictive?"
  end
  pkgs
end

.create_rolling_repo_link(platform_tag, staging_server, repo_path) ⇒ Object 



208
209
210
211
212
213
214
 
# File 'lib/packaging/util/ship.rb', line 208

def create_rolling_repo_link(platform_tag, staging_server, repo_path)
  command = rolling_repo_link_command(platform_tag, repo_path)

  Pkg::Util::Net.remote_ssh_cmd(staging_server, command) unless command.nil?
rescue => e
  fail "Failed to create rolling repo link for '#{platform_tag}'.\n#{e}"
end

.reorganize_packages(pkgs, tmp, platform_independent = false) ⇒ Object

Takes a set of packages and reorganizes them into the final repo structure before they are shipping out to their final destination.

This assumes the working directory is a temporary directory that will later be cleaned up

If this is platform_independent the packages will not get reorganized, just copied under the tmp directory for more consistent workflows



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/packaging/util/ship.rb', line 27

def reorganize_packages(pkgs, tmp, platform_independent = false)
  new_pkgs = []
  pkgs.each do |pkg|
    if platform_independent
      path = 'pkg'
    else
      platform_tag = Pkg::Paths.tag_from_artifact_path(pkg)
      path = Pkg::Paths.artifacts_path(platform_tag, 'pkg')
    end
    FileUtils.mkdir_p File.join(tmp, path)
    FileUtils.cp pkg, File.join(tmp, path)
    new_pkgs << File.join(path, File.basename(pkg))
  end
  new_pkgs
end

.rolling_repo_link_command(platform_tag, repo_path) ⇒ Object 



178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/packaging/util/ship.rb', line 178

def rolling_repo_link_command(platform_tag, repo_path)
  base_path, link_path = Pkg::Paths.artifacts_base_path_and_link_path(platform_tag, repo_path)

  if link_path.nil?
    puts "No link target set, not creating rolling repo link for #{base_path}"
    return nil
  end

  cmd = "    if [ ! -d \#{base_path} ] ; then\n      echo \"Link target '\#{base_path}' does not exist; skipping\"\n      exit 0\n    fi\n    # If it's a link but pointing to the wrong place, remove the link\n    # This is likely to happen around the transition times, like puppet5 -> puppet6\n    if [ -L \#{link_path} ] && [ ! \#{base_path} -ef \#{link_path} ] ; then\n      rm \#{link_path}\n    # This is the link you're looking for, nothing to see here\n    elif [ -L \#{link_path} ] ; then\n      exit 0\n    # Don't want to delete it if it isn't a link, that could be destructive\n    # So, fail!\n    elif [ -e \#{link_path} ] ; then\n      echo \"\#{link_path} exists but isn't a link, I don't know what to do with this\" >&2\n      exit 1\n    fi\n    ln -s \#{base_path} \#{link_path}\n  CMD\nend\n"

.ship_debs(local_staging_directory, remote_path, opts = {}) ⇒ Object 



111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/packaging/util/ship.rb', line 111

def ship_debs(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/**/*.debian.tar.gz", "#{local_staging_directory}/**/*.orig.tar.gz" "#{local_staging_directory}/**/*.dsc", "#{local_staging_directory}/**/*.deb", "#{local_staging_directory}/**/*.changes"], Pkg::Config.apt_signing_server, remote_path, opts)

  # We need to iterate through all the supported platforms here because of
  # how deb repos are set up. Each codename will have its own link from the
  # current versioned repo (e.g. puppet5) to the rolling repo. The one thing
  # we don't care about is architecture, so we just grab the first supported
  # architecture for the code name we're working with at the moment. [written
  # by Melissa, copied by Molly]
  Pkg::Platforms.codenames.each do |codename|
    create_rolling_repo_link(Pkg::Platforms.codename_to_tags(codename)[0], Pkg::Config.apt_signing_server, remote_path)
  end
end

.ship_dmg(local_staging_directory, remote_path, opts = {}) ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/packaging/util/ship.rb', line 133

def ship_dmg(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/**/*.dmg"], Pkg::Config.dmg_staging_server, remote_path, opts)

  create_rolling_repo_link(Pkg::Platforms.generic_platform_tag('osx'), Pkg::Config.dmg_staging_server, remote_path)

  Pkg::Platforms.platform_tags_for_package_format('dmg').each do |platform_tag|
    # TODO remove the PC1 links when we no longer need to maintain them
    # [written by Melissa, copied by Molly]
    _, version, arch = Pkg::Platforms.parse_platform_tag(platform_tag)
    Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', "/opt/downloads/mac/#{version}/PC1/#{arch}", 'dmg')
    # Create the latest symlink for the current supported repo
    Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', Pkg::Paths.artifacts_path(platform_tag, remote_path), 'dmg')
  end
end

.ship_gem(local_staging_directory, remote_path, opts = {}) ⇒ Object 



170
171
172
 
# File 'lib/packaging/util/ship.rb', line 170

def ship_gem(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/*.gem*"], Pkg::Config.gem_host, remote_path, opts)
end

.ship_msi(local_staging_directory, remote_path, opts = {}) ⇒ Object 



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/packaging/util/ship.rb', line 154

def ship_msi(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/**/*.msi"], Pkg::Config.msi_staging_server, remote_path, opts)

  create_rolling_repo_link(Pkg::Platforms.generic_platform_tag('windows'), Pkg::Config.msi_staging_server, remote_path)
  # Create the symlinks for the latest supported repo
  Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', Pkg::Paths.artifacts_path(Pkg::Platforms.generic_platform_tag('windows'), remote_path), 'msi', arch: 'x64')
  Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', Pkg::Paths.artifacts_path(Pkg::Platforms.generic_platform_tag('windows'), remote_path), 'msi', arch: 'x86')

  # We provide symlinks to the latest package in a given directory. This
  # allows users to upgrade more easily to the latest version that we release
  # TODO remove the links to PC1 when we no longer ship to that repo [written
  # by Melissa, copied by Molly]
  Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', '/opt/downloads/windows', 'msi', arch: 'x64')
  Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', '/opt/downloads/windows', 'msi', arch: 'x86')
end

.ship_p5p(local_staging_directory, remote_path, opts = {}) ⇒ Object 



129
130
131
 
# File 'lib/packaging/util/ship.rb', line 129

def ship_p5p(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/**/*.p5p"], Pkg::Config.p5p_host, remote_path, opts)
end

.ship_pkgs(pkg_exts, staging_server, remote_path, opts = {}) ⇒ Object

Take local packages and restructure them to the desired final path before shipping to the staging server rubocop:disable Metrics/MethodLength, Metrics/AbcSize

Parameters:

  • pkg_exts (Array) —

    the file globs for the files you want to ship For example, something like ['pkg//*.rpm', 'pkg//*.deb'] to ship the rpms and debs

  • staging_server (String) —

    The hostname to ship the packages to

  • remote_path (String) —

    The base path to ship the packages to on the staging_server, for example '/opt/downloads/windows' or '/opt/repository/yum'

  • opts (Hash) (defaults to: {}) —

    Additional options that can be used when shipping packages

Options Hash (opts):

  • :excludes (Array) —

    File globs to exclude packages from shipping

  • :chattr (Boolean) —

    Whether or not to make the files immutable after shipping. Defaults to true.

  • :platform_independent (Boolean) —

    Whether or not the path the packages ship to has platform-dependent information in it. Defaults to false (most paths will be platform dependent), but set to true for gems and tarballs since those just land directly under /opt/downloads/



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/packaging/util/ship.rb', line 63

def ship_pkgs(pkg_exts, staging_server, remote_path, opts = {})
  options = {
    excludes: [],
    chattr: true,
    platform_independent: false }.merge(opts)

  # First find the packages to be shipped. We must find them before moving
  # to our temporary staging directory
  local_packages = collect_packages(pkg_exts, options[:excludes])
  return if local_packages.empty?

  tmpdir = Dir.mktmpdir
  staged_pkgs = reorganize_packages(local_packages, tmpdir, options[:platform_independent])

  puts staged_pkgs.sort
  puts "Do you want to ship the above files to (#{staging_server})?"
  if Pkg::Util.ask_yes_or_no
    extra_flags = ['--ignore-existing', '--delay-updates']
    extra_flags << '--dry-run' if ENV['DRYRUN']

    staged_pkgs.each do |pkg|
      Pkg::Util::Execution.retry_on_fail(times: 3) do
        sub_string = 'pkg'
        remote_pkg = pkg.sub(sub_string, remote_path)
        remote_basepath = File.dirname(remote_pkg)
        Pkg::Util::Net.remote_ssh_cmd(staging_server, "mkdir -p #{remote_basepath}")
        Pkg::Util::Net.rsync_to(
          File.join(tmpdir, pkg),
          staging_server,
          remote_basepath,
          extra_flags: extra_flags
        )

        Pkg::Util::Net.remote_set_ownership(staging_server, 'root', 'release', [remote_basepath, remote_pkg])
        Pkg::Util::Net.remote_set_permissions(staging_server, '775', [remote_basepath])
        Pkg::Util::Net.remote_set_permissions(staging_server, '0664', [remote_pkg])
        Pkg::Util::Net.remote_set_immutable(staging_server, [remote_pkg]) if options[:chattr]
      end
    end
  end
end

.ship_rpms(local_staging_directory, remote_path, opts = {}) ⇒ Object 



105
106
107
108
109
 
# File 'lib/packaging/util/ship.rb', line 105

def ship_rpms(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/**/*.rpm", "#{local_staging_directory}/**/*.srpm"], Pkg::Config.yum_staging_server, remote_path, opts)

  create_rolling_repo_link(Pkg::Platforms.generic_platform_tag('el'), Pkg::Config.yum_staging_server, remote_path)
end

.ship_svr4(local_staging_directory, remote_path, opts = {}) ⇒ Object 



125
126
127
 
# File 'lib/packaging/util/ship.rb', line 125

def ship_svr4(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/**/*.pkg.gz"], Pkg::Config.svr4_host, remote_path, opts)
end

.ship_swix(local_staging_directory, remote_path, opts = {}) ⇒ Object 



148
149
150
151
152
 
# File 'lib/packaging/util/ship.rb', line 148

def ship_swix(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/**/*.swix"], Pkg::Config.swix_staging_server, remote_path, opts)

  create_rolling_repo_link(Pkg::Platforms.generic_platform_tag('eos'), Pkg::Config.swix_staging_server, remote_path)
end

.ship_tar(local_staging_directory, remote_path, opts = {}) ⇒ Object 



174
175
176
 
# File 'lib/packaging/util/ship.rb', line 174

def ship_tar(local_staging_directory, remote_path, opts = {})
  ship_pkgs(["#{local_staging_directory}/*.tar.gz*"], Pkg::Config.tar_staging_server, remote_path, opts)
end

.test_ship(vm, ship_task) ⇒ Object 



216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
# File 'lib/packaging/util/ship.rb', line 216

def test_ship(vm, ship_task)
  command = 'getent group release || groupadd release'
  Pkg::Util::Net.remote_ssh_cmd(vm, command)
  hosts_to_override = %w(
    APT_HOST
    DMG_HOST
    GEM_HOST
    IPS_HOST
    MSI_HOST
    P5P_HOST
    SVR4_HOST
    SWIX_HOST
    TAR_HOST
    YUM_HOST
    APT_SIGNING_SERVER
    APT_STAGING_SERVER
    DMG_STAGING_SERVER
    MSI_STAGING_SERVER
    SWIX_STAGING_SERVER
    TAR_STAGING_SERVER
    YUM_STAGING_SERVER
    STAGING_SERVER
  )
  hosts_to_override.each do |host|
    ENV[host] = vm
  end
  Rake::Task[ship_task].invoke
end