Method: Deb::S3::CLI#clean

Defined in:
lib/deb/s3/cli.rb

#clean ⇒ Object 



607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
 
# File 'lib/deb/s3/cli.rb', line 607

def clean
  configure_s3_client

  begin
    if options[:lock]
      log("Checking for existing lock file")
      log("Locking repository for updates")
      Deb::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
      @lock_acquired = true
    end

    log("Retrieving existing manifests")

    # Enumerate objects under the dists/<codename>/ prefix to find any
    # Packages files and load them....

    req = Deb::S3::Utils.s3.list_objects_v2({
      :bucket => Deb::S3::Utils.bucket,
      :prefix => Deb::S3::Utils.s3_path("dists/#{ options[:codename] }/"),
    })

    manifests = []
    req.contents.each do |object|
      if match = object.key.match(/dists\/([^\/]+)\/([^\/]+)\/binary-([^\/]+)\/Packages$/)
        codename, component, arch = match.captures
        manifests.push(Deb::S3::Manifest.retrieve(codename, component, arch, options[:cache_control], options[:fail_if_exists], options[:skip_package_upload]))
      end
    end

    # Iterate over the packages in each manifest and build a Set of all the
    # referenced URLs (relative to bucket root)...

    refd_urls = Set[]
    manifests.each do |manifest|
      manifest.packages.each do |package|
        refd_urls.add(Deb::S3::Utils.s3_path(package.url_filename(manifest.codename)))
      end
    end

    log("Searching for unreferenced packages")

    # Enumerate objects under the pools/<codename> prefix and delete any that
    # arent referenced by any of the manifests.

    continuation_token = nil
    while true
      req = Deb::S3::Utils.s3.list_objects_v2({
        :bucket => Deb::S3::Utils.bucket,
        :prefix => Deb::S3::Utils.s3_path("pool/#{ options[:codename] }/"),
        :continuation_token => continuation_token,
      })

      req.contents.each do |object|
        if not refd_urls.include?(object.key)
          sublog("Deleting #{ object.key }")

          Deb::S3::Utils.s3.delete_object({
            :bucket => Deb::S3::Utils.bucket,
            :key => object.key,
          })
        end
      end

      if req.is_truncated
        continuation_token = req.next_continuation_token
      else
        break
      end
    end
  ensure
    if options[:lock] && @lock_acquired
      Deb::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
      log("Lock released.")
    end
  end
end