Method: Autoproj::Ops::Import#import_selected_packages

Defined in:

lib/autoproj/ops/import.rb

#import_selected_packages(selection, parallel: ws.config.parallel_import_level, recursive: true, retry_count: nil, keep_going: false, install_vcs_packages: Hash.new, non_imported_packages: :checkout, auto_exclude: auto_exclude?, , filter: ->(package) { true }, **import_options) ⇒ Object

Import all packages from the given selection, and their dependencies

# File 'lib/autoproj/ops/import.rb', line 196

def import_selected_packages(selection,
    parallel: ws.config.parallel_import_level,
    recursive: true,
    retry_count: nil,
    keep_going: false,
    install_vcs_packages: Hash.new,
    non_imported_packages: :checkout,
    auto_exclude: auto_exclude?,
    filter: ->(package) { true },
    **import_options)

    unless %i[checkout ignore return].include?(non_imported_packages)
        raise ArgumentError, "invalid value for 'non_imported_packages'. "\
                             "Expected one of :checkout, :ignore or :return "\
                             "but got #{non_imported_packages}"
    end

    # This is used in the ensure block, initialize as early as
    # possible
    executor = Concurrent::FixedThreadPool.new(parallel)
    manifest = ws.manifest

    selected_packages = selection.each_source_package_name.map do |pkg_name|
        manifest.find_package_definition(pkg_name)
    end.to_set

    # The reverse dependencies for the package tree. It is discovered as
    # we go on with the import
    #
    # It only contains strong dependencies. Optional dependencies are
    # not included, as we will use this only to take into account
    # package exclusion (and that does not affect optional dependencies)
    reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }

    completion_queue = Queue.new
    pending_packages = Set.new
    # The set of all packages that are currently selected by +selection+
    all_processed_packages = Set.new
    main_thread_imports = Array.new
    package_queue = selected_packages.to_a.sort_by(&:name)

    failures = Array.new
    missing_vcs = Array.new
    installed_vcs_packages = Set["none", "local"]
    while failures.empty? || keep_going
        # Allow 'filter' to parallelize as well
        if filter.respond_to?(:lookahead)
            package_queue.each { |pkg| filter.lookahead(pkg) }
        end

        # Queue work for all packages in the queue
        package_queue.each do |pkg|
            # Remove packages that have already been processed
            next if all_processed_packages.include?(pkg)

            vcs_installed = installed_vcs_packages.include?(pkg.vcs.type)
            if (non_imported_packages != :checkout) && !pkg.checked_out?
                all_processed_packages << pkg
                if non_imported_packages == :return
                    completion_queue << [pkg, Time.now, false, nil]
                else
                    ws.manifest.ignore_package(pkg.name)
                end
                next
            elsif install_vcs_packages && !vcs_installed
                missing_vcs << pkg
                next
            end
            all_processed_packages << pkg

            unless filter.call(pkg)
                completion_queue << [pkg, Time.now]
                next
            end

            importer = pkg.autobuild.importer
            if !pre_package_import(selection, manifest, pkg.autobuild,
                                   reverse_dependencies)
                next
            elsif !importer
                # The validity of this is checked in
                # pre_package_import
                completion_queue << [pkg, Time.now, false, nil]
                next
            elsif importer.interactive?
                main_thread_imports << pkg
                next
            elsif pkg.checked_out? && import_options[:checkout_only]
                main_thread_imports << pkg
                next
            end

            begin
                pending_packages << pkg
                queue_import_work(
                    executor, completion_queue, pkg,
                    retry_count: retry_count,
                    **import_options.merge(allow_interactive: false)
                )
            rescue Exception
                pending_packages.delete(pkg)
                raise
            end
        end
        package_queue.clear

        if completion_queue.empty? && pending_packages.empty?
            unless missing_vcs.empty?
                installed_vcs_packages.merge(
                    install_vcs_packages_for(
                        *missing_vcs,
                        install_only: import_options[:checkout_only],
                        **install_vcs_packages
                    )
                )
                package_queue.concat(missing_vcs)
                missing_vcs.clear
                next
            end

            # We've nothing to process anymore ... process
            # interactive imports if there are some. Otherwise,
            # we're done
            if main_thread_imports.empty?
                break
            else
                main_thread_imports.delete_if do |pkg|
                    begin
                        if retry_count
                            pkg.autobuild.importer.retry_count = retry_count
                        end
                        result = pkg.autobuild.import(
                            **import_options.merge(allow_interactive: true)
                        )
                    rescue StandardError => e
                    end
                    completion_queue << [pkg,
                                         Time.now, result, e]
                end
            end
        end

        # And wait for one to finish
        pkg, _time, _result, reason = completion_queue.pop
        pending_packages.delete(pkg)
        if reason
            if reason.kind_of?(Autobuild::InteractionRequired)
                main_thread_imports << pkg
            elsif auto_exclude
                manifest.add_exclusion(
                    pkg.name, "#{pkg.name} failed to import with "\
                              "#{reason} and auto_exclude was true"
                )
                selection.filter_excluded_and_ignored_packages(manifest)
            else
                # One importer failed... terminate
                Autoproj.error "import of #{pkg.name} failed"
                Autoproj.error reason.to_s unless reason.kind_of?(Interrupt)
                failures << reason
            end
        else
            new_packages = post_package_import(
                selection, manifest, pkg, reverse_dependencies,
                auto_exclude: auto_exclude
            )
            if new_packages
                # Excluded dependencies might have caused the package to be
                # excluded as well ... do not add any dependency to the
                # processing queue if it is the case
                if manifest.excluded?(pkg.name)
                    selection.filter_excluded_and_ignored_packages(manifest)
                elsif recursive
                    package_queue = new_packages.sort_by(&:name)
                end
            end
        end
    end

    all_processed_packages.delete_if do |processed_pkg|
        ws.manifest.excluded?(processed_pkg.name) ||
            ws.manifest.ignored?(processed_pkg.name)
    end
    [all_processed_packages, failures]
ensure
    if failures && !failures.empty? && !keep_going
        Autoproj.error "waiting for pending import jobs to finish"
    end
    if executor
        executor.shutdown
        executor.wait_for_termination
    end
end