Class: Simp::Rake::Build::Iso

Inherits:

Rake::TaskLib

• Object
• Rake::TaskLib
• Simp::Rake::Build::Iso

Includes:

Simp::Rake, Constants

Defined in:

lib/simp/rake/build/iso.rb

Instance Attribute Summary

Attributes included from Simp::Rake

#module_paths, #puppetfile

Instance Method Summary

• #define_tasks ⇒ Object
• #initialize(base_dir) ⇒ Iso constructor

  A new instance of Iso.

• #verbose ⇒ Object

Methods included from Constants

#distro_build_dir, #init_member_vars

Methods included from Simp::Rake

#clean_yaml, #encode_line, #get_cpu_limit, #help, #load_puppetfile, #run_pager

Methods included from CommandUtils

#which

Constructor Details

#initialize(base_dir) ⇒ Iso

Returns a new instance of Iso.

# File 'lib/simp/rake/build/iso.rb', line 12

def initialize( base_dir )
  init_member_vars( base_dir )

  define_tasks
end

Instance Method Details

#define_tasks ⇒ Object

# File 'lib/simp/rake/build/iso.rb', line 22

def define_tasks

  File.umask(0007)

  namespace :iso do
    task :prep do
      if $simp6
        @build_dir = $simp6_build_dir || @distro_build_dir
      end
    end

    # Remove packages from the given directory. The goal of this method is to help
    # get the distro to be as small as possible.
    # [:from_dir] Root directory to remove packages from (removes recursively)
    # [:exclude_dirs] Array of directories to not remove any packages from
    # [:exclude_pkgs] Array of packages to not remove
    def prune_packages(from_dir,exclude_dirs,exclude_pkgs,mkrepo='createrepo -p',use_hack=true)
      $stderr.puts "Starting to prune..."
      Dir.chdir(from_dir) do
        prune_count = 0

        Find.find('.') do |path|
          Find.prune if exclude_dirs.include?(File.basename(path))

          if File.basename(path) =~ /.*\.rpm/
            # Get the package name from the RPM.
            # Note: an alternative method may be to just simply check the names
            # of the RPMs themselves instead of the names of the packages.
            pkg = nil
            if use_hack
              # The proper way (defined below) is way too slow, so this hack helps
              # speed up the process by reading the file directly. If the code is
              # not working, attempt this without using the hack, just be ready
              # to wait a long time for the code to complete.
              pkgname = File.basename(path).split('-').first
              File.open(path,'r').each_line do |line|
                if encode_line(line) =~ /C\000(\S+\000)?(#{Regexp.escape(pkgname)}\S*)\000/
                  pkg = $2.split(/\000/).first
                  break
                end
              end
            else
              # Proper way to obtain the RPM's package name, but WAY too slow
              pkg = %x{rpm -qp --qf "%{NAME}" #{path} 2>/dev/null}.chomp
            end

            unless exclude_pkgs.include?(pkg)
              rm(path, :verbose => verbose)
              prune_count += 1
            end
          end
        end
        $stderr.puts "Info: Pruned #{prune_count} packages from #{from_dir}"

        if prune_count > 0
          # Recreate the now-pruned repos
          basepath = '.'
          if (File.basename(from_dir) =~ /^RHEL/)
            # This covers old versions of RHEL that don't follow the new
            # way of doing things.
            unless Dir.glob("Server/*.rpm").empty?
              basepath = 'Server'
            end
          end

          Dir.chdir(basepath) do
            cp(Dir.glob("repodata/*comps*.xml").first,"simp_comps.xml")
            sh %{#{mkrepo} -g simp_comps.xml .}
            rm("simp_comps.xml")
          end
        end
      end
    end # End of prune_packages

=begin
    desc <<-EOM
  Build the SIMP ISO(s).
   * :tarball - Path of the source tarball or directory containing the source
       tarballs.
   * :unpacked_dvds - Path of the directory containing the unpacked base OS
       directories. Default is the current directory.
   * :prune - Flag for whether unwanted packages should be pruned prior to
       building the ISO. Default is true.

   ENV vars:
     - Set `SIMP_ISO_verbose=yes` to report file operations as they happen.
       EOM
=end

    task :build,[:tarball,:unpacked_dvds,:prune] => [:prep] do |t,args|
      args.with_defaults(:unpacked_dvds => "#{@run_dir}", :prune => 'true')

      if args.tarball.nil?
        fail("Error: You must specify a source tarball or tarball directory!")
      else
        tarball = File.expand_path(args.tarball)

        unless File.exist?(tarball)
          fail("Error: Could not find tarball at '#{tarball}'!")
        end
      end

      reposync_location = File.join(@build_dir,'yum_data','reposync')
      reposync_active = !Dir.glob(File.join(reposync_location, '**', 'repomd.xml')).empty?

      tarfiles = File.directory?(tarball) ?
        Dir.glob("#{tarball}/*.tar.gz") : [tarball]
      vermap = YAML::load_file( File.join( File.dirname(__FILE__), 'vermap.yaml'))

      tarfiles.each do |tball|
        namepieces = File.basename(tarball,".tar.gz").split('-')

        # SIMP 6
        if namepieces[1] =~ /^\d/
          simpver = namepieces[1..2].join('-')
          baseos  = namepieces[3]
        else
          simpver = namepieces[3..-1].join('-')
          baseos  = namepieces[2]
        end

        iso_dirs = Dir.glob("#{File.expand_path(args.unpacked_dvds)}/#{baseos}*")
        if iso_dirs.empty?
          fail("Error: No unpacked DVD directories found for '#{baseos}' under '#{File.expand_path(args.unpacked_dvds)}'")
        end

        # Process each unpacked base OS ISO directory found
        iso_dirs.each do |dir|
          baseosver = '???'
          arch      = '???'

          # read the .treeinfo file (INI format)
          # ------------------------------------
          require 'puppet'
          require 'puppet/util/inifile'

          file = "#{dir}/.treeinfo"
          fail("ERROR: no file '#{file}'") unless File.file?(file)

          ini = Puppet::Util::IniConfig::PhysicalFile.new(file)
          ini.read
          sections = ini.sections.map{ |s| s.name }

          # NOTE: RHEL7 discs claim [general] section is deprecated.
          if sections.include?('general')
            h = Hash[ ini.get_section('general').entries.map{|k,v| [k,v]} ]
            arch      = h.fetch('arch', arch).strip
            baseosver = h.fetch('version', baseosver).strip
            baseosver += '.0' if (baseosver.count('.') < 1)
          end
          # ------------------------------------

          # Skip if SIMP version doesn't match target base OS version
          unless Array(vermap[simpver.split('.').first]).include?(baseosver.split('.').first)
            if verbose
              warn("Could not find SIMP version mapping for #{simpver} for Base OS #{baseosver}")
              warn("Do you need to update vermap.yaml in the Gem?")
            end

            next
          end

          mkrepo = baseosver.split('.').first == '5' ? 'createrepo -s sha -p' : 'createrepo -p'

          @simp_dvd_dirs.each do |clean_dir|
            if File.directory?("#{dir}/#{clean_dir}")
              rm_rf("#{dir}/#{clean_dir}", :verbose => verbose)
            elsif File.file?("#{dir}/#{clean_dir}")
              fail("Error: #{dir}/#{clean_dir} is a file, expecting directory!")
            end
          end


          repo_target_dir = dir

          # If we've pulled in reposync directories, we expect them to
          # completely overwrite the directory of the same name in the
          # target ISO so no pruning is required
          #
          # Note: CASE MATTERS on the directory names
          if reposync_active
            # We're working with the new EL8+ layout, so we need to target
            # the SimpRepos subdirectory
            repo_target_dir = File.join(dir,'SimpRepos')
            mkdir_p(repo_target_dir, :verbose => verbose)

            repos_to_overwrite = Dir.glob(File.join(reposync_location, '*'))
              .delete_if{|x| !File.directory?(x)}
              .map{|x| File.basename(x)}

            repos_to_overwrite.each do |repo|
              src = File.join(reposync_location, repo)
              target = File.join(dir, repo)

              if File.directory?(target)
                rm_rf(target, :verbose => verbose) if File.directory?(target)
              else
                target = File.join(repo_target_dir, repo)
              end

              cp_r(src, target, :verbose => verbose)
            end
          else
            # Prune unwanted packages
            begin
              system("tar --no-same-permissions -C #{repo_target_dir} -xzf #{tball} *simp_pkglist.txt")
            rescue
              # Does not matter if the command fails
            end

            pkglist_file = ENV.fetch(
              'SIMP_PKGLIST_FILE',
              File.join(dir,"#{baseosver.split('.').first}-simp_pkglist.txt")
            )

            puts
            puts '-'*80
            puts "### Pruning packages not in file '#{pkglist_file}'"
            puts
            puts '   (override this with `SIMP_PKGLIST_FILE=<file>`)'
            puts
            puts '-'*80
            puts

            if (args.prune.casecmp("false") != 0) && File.exist?(pkglist_file)
              exclude_pkgs = Array.new
              File.read(pkglist_file).each_line do |line|
                next if line =~ /^(\s+|#.*)$/
                exclude_pkgs.push(line.chomp)
              end
              prune_packages(dir,['SIMP','SimpRepos'],exclude_pkgs,mkrepo)
            end
          end

          reposync_only = (ENV.fetch('SIMP_BUILD_reposync_only', 'no') == 'yes')
          if reposync_only && !reposync_active
            fail("ERROR: Reposync-only requested, but no reposync content found")
          end

          if reposync_only
            puts
            puts '-'*80
            puts '### Reposync-Only Mode ###'
            puts
            puts '  Locally built packages will not be added'
            puts
            puts '-'*80
            puts

            # Only add the ISO modifications
            system(%(tar --no-same-permissions --exclude="*.rpm" -C #{dir} -xzf #{tball}))
          else
            # Add the SIMP code and ISO modifications
            system("tar --no-same-permissions -C #{dir} -xzf #{tball}")

            # Pop the SIMP directory from the tarball into the correct spot
            # FIXME: This is a hack
            unless dir == repo_target_dir
              simpdir = File.join(dir,'SIMP')

              if File.directory?(simpdir)
                 cp_r(simpdir, repo_target_dir, :verbose => verbose)
                 rm_rf(simpdir, :verbose => verbose)
              end
            end

            Dir.chdir("#{repo_target_dir}/SIMP") do
              # Add the SIMP Dependencies
              simp_base_ver = simpver.split('-').first

              if $simp6
                yum_dep_location = File.join(@build_dir,'yum_data','packages')
              else
                simp_dep_src = %(SIMP#{simp_base_ver}_#{baseos}#{baseosver}_#{arch})
                yum_dep_location = File.join(@build_dir,'yum_data',simp_dep_src,'packages')
              end

              yum_dep_rpms = Dir.glob(File.join(yum_dep_location,'*.rpm'))

              unless File.directory?(yum_dep_location)
                fail("Could not find dependency directory at #{yum_dep_location}") unless reposync_active
              end

              if yum_dep_rpms.empty?
                fail("Could not find any dependency RPMs at #{yum_dep_location}") unless reposync_active
              end

              # Add any one-off RPMs that you might want to add to your own build
              # These are *not* checked to make sure that they actually match your
              # environment
              aux_packages = File.join(File.dirname(yum_dep_location),'aux_packages')
              if File.directory?(aux_packages)
                yum_dep_rpms += Dir.glob(File.join(aux_packages,'*.rpm'))
              end

              yum_dep_rpms.each do |rpm|
                rpm_arch = rpm.split('.')[-2]

                unless File.directory?(rpm_arch)
                  mkdir(rpm_arch)
                end

                # Just in case this is a symlink, broken, or some other nonsense.
                target_file = File.join(rpm_arch,File.basename(rpm))
                rm_f(target_file) if File.exist?(target_file)

                cp(rpm,rpm_arch, :verbose => verbose)
              end

              unless File.exist?('repodata/repomd.xml')
                fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
              end
            end

            if reposync_active
                fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
            else
              ln_sf('noarch', arch, :verbose => verbose) if (!File.directory?(arch) && File.directory?('noarch'))
              fail("Could not find architecture '#{arch}' in the SIMP distribution") unless (File.directory?(arch) || File.symlink?(arch))

              # Get everything set up properly...
              Dir.chdir(arch) do
                Dir.glob('../*') do |rpm_dir|
                  # Don't dive into ourselves
                  next if File.basename(rpm_dir) == arch

                  Dir.glob(%(#{rpm_dir}/*.rpm)) do |source_rpm|
                    link_target = File.basename(source_rpm)
                    if File.exist?(source_rpm) && File.exist?(link_target)
                      next if Pathname.new(source_rpm).realpath == Pathname.new(link_target).realpath
                    end

                    ln_sf(source_rpm,link_target, :verbose => verbose)
                  end
                end

                fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
              end
            end
          end

          ### Munge the Repos

          # Create an Updates directory that is properly populated
          updates_readme = "            This directory houses updates to NON-MODULAR RPMs.\n\n            DO NOT put modular RPMs in this directory or you will break your\n            system updates!\n            README\n\n          updates_dir = File.join(dir, 'Updates')\n          mkdir_p(updates_dir)\n\n          Dir.chdir(updates_dir) do\n            File.open('README','w'){|fh| fh.puts(updates_readme) }\n\n            repos = Dir.glob(File.join('..','**','repodata'))\n            modular_repos = Dir.glob(File.join('..','**','repodata','*-modules.*'))\n            non_modular_repos = repos.select{|x| modular_repos.grep(%r{^\#{Regexp.escape(x)}}).empty? }\n            non_modular_repos.map!{|x| File.split(x).first}\n            non_modular_repos.delete_if{|x| x.match(%r{/SimpRepos|/SIMP}) }\n            non_modular_repos.each do |non_modular_repo|\n              Dir.glob(File.join(non_modular_repo, '**', '*.rpm')).each do |rpm|\n                # when non_modular_repo is '..', can still find RPMs we need\n                # to exclude\n                next if rpm.match(%r{/SimpRepos|/SIMP})\n\n                rpm_dest = File.basename(rpm)\n                if File.exist?(rpm) && File.exist?(rpm_dest)\n                  next if (File.realpath(rpm) == File.realpath(rpm_dest))\n                end\n\n                ln_sf(rpm, rpm_dest, :verbose => verbose)\n              end\n            end\n\n            fail(\"Error: Could not run createrepo in \#{Dir.pwd}\") unless system(%(\#{mkrepo} .))\n          end\n\n          # New ISO Layout\n          if reposync_active\n            Dir.chdir(repo_target_dir) do\n              gpgkeysdir = File.join('SIMP','GPGKEYS')\n\n              if File.directory?(gpgkeysdir)\n                cp_r(gpgkeysdir, '.', :verbose => verbose)\n                rm_rf(gpgkeysdir, :verbose => verbose)\n              end\n            end\n          end\n\n\n          # NOTE: repoclosure doesn't work with modular repos, and probably\n          # never can:\n          #\n          #    https://bugzilla.redhat.com/show_bug.cgi?id=1547041\n          #\n          # If an ISO somehow has *NO* modular repos/packages, than running\n          # with SIMP_BUILD_repoclosure=yes is safe.  Otherwise, it will\n          # probably fail even if all the packages depsolve cleanly in real\n          # life\n          if ENV.fetch('SIMP_BUILD_repoclosure', 'no') == 'yes'\n            # Make sure we have all of the necessary RPMs!\n            Rake::Task['pkg:repoclosure'].invoke(File.expand_path(dir))\n          end\n\n          # Do some sane chmod'ing and build ISO\n          system(\"chmod -fR u+rwX,g+rX,o=g \#{dir}\")\n          simp_output_name = \"SIMP-\#{simpver}-\#{baseos}-\#{baseosver}-\#{arch}\"\n          @simp_output_iso = \"\#{simp_output_name}.iso\"\n\n          mkisofs_cmd = [\n            'mkisofs',\n            \"-A SIMP-\#{simpver}\",\n            \"-V SIMP-\#{simpver}\",\n            \"-volset SIMP-\#{simpver}\",\n            '-uid 0',\n            '-gid 0',\n            '-J',\n            '-joliet-long',\n            '-r',\n            '-v',\n            '-T',\n            '-b isolinux/isolinux.bin',\n            '-c boot.cat',\n            '-boot-load-size 4',\n            '-boot-info-table',\n            '-no-emul-boot',\n            '-eltorito-alt-boot',\n            '-e images/efiboot.img',\n            # This is apparently needed twice to get the lines above it to\n            # take. Not sure why.\n            '-no-emul-boot',\n            '-m TRANS.TBL',\n            '-x ./lost+found',\n            \"-o \#{@simp_output_iso}\",\n            dir\n          ].join(' ')\n\n          $stdout.puts \"Running: \#{mkisofs_cmd}\"\n\n          system(mkisofs_cmd)\n        end\n      end # End of tarfiles loop\n\n      # If we got here and didn't generate any ISOs, something went horribly wrong\n      fail('Error: No ISO was built!') unless ( @simp_output_iso && File.exist?(@simp_output_iso) )\n\n      # Embed the validation checksum\n      system(\"implantisomd5 --supported-iso \#{@simp_output_iso}\")\n    end\n\n=begin\n    desc <<-EOM\n    Build the source ISO.\n      Note: The process clobbers the temporary and built files, rebuilds the\n      tarball(s) and packages the source ISO. Therefore it will take a\n      while.\n        * :key - The GPG key to sign the RPMs with. Defaults to 'prod'.\n    EOM\n=end\n    task :src,[:prep, :key] do |t,args|\n      args.with_defaults(:key => 'prod')\n\n      if Dir.glob(\"\#{@dvd_dir}/*.gz\").empty?\n        fail(\"Error: Could not find compiled source tarballs\")\n      end\n\n      Rake::Task['tar:build']\n\n      Dir.chdir(@base_dir) do\n        File.basename(Dir.glob(\"\#{@dvd_dir}/*.tar.gz\").first,'.tar.gz') =~ /SIMP-DVD-[^-]+-(.+)/\n        name = \"SIMP-\#{$1}\"\n        sh %{mkisofs -uid 0 -gid 0 -D -A \#{name} -J -joliet-long -m \".git*\" -m \"./build/tmp\" -m \"./build/SRPMS\" -m \"./build/RPMS\" -m \"./build/build_keys\" -o \#{name}.src.iso .}\n      end\n    end\n  end\nend\n"

#verbose ⇒ Object

# File 'lib/simp/rake/build/iso.rb', line 18

def verbose
  ENV.fetch('SIMP_ISO_verbose','no') == 'yes'
end