Class: GHTMoreCommitsRetriever

Inherits:
GHTorrent::Command show all
Includes:
GHTorrent::Logging, GHTorrent::Persister, GHTorrent::Retriever, GHTorrent::Settings
Defined in:
lib/ghtorrent/commands/ght_get_more_commits.rb

Constant Summary

Constants included from GHTorrent::Settings

GHTorrent::Settings::CONFIGKEYS, GHTorrent::Settings::DEFAULTS

Constants included from GHTorrent::Persister

GHTorrent::Persister::ADAPTERS

Instance Method Summary collapse

Methods included from GHTorrent::Logging

#debug, #error, #info, #loggerr, #warn

Methods included from GHTorrent::Settings

#config, #merge, #merge_config_values, #override_config, #settings

Methods included from GHTorrent::Utils

included, #read_value, #user_type, #write_value

Methods included from GHTorrent::Persister

#connect, #disconnect

Methods included from GHTorrent::Retriever

#get_event, #get_events, #get_repo_events, #retrieve_commit, #retrieve_commit_comment, #retrieve_commit_comments, #retrieve_commits, #retrieve_fork, #retrieve_forks, #retrieve_issue, #retrieve_issue_comment, #retrieve_issue_comments, #retrieve_issue_event, #retrieve_issue_events, #retrieve_issue_labels, #retrieve_issues, #retrieve_languages, #retrieve_org, #retrieve_org_members, #retrieve_orgs, #retrieve_pull_req_comment, #retrieve_pull_req_comments, #retrieve_pull_req_commits, #retrieve_pull_request, #retrieve_pull_requests, #retrieve_repo, #retrieve_repo_collaborator, #retrieve_repo_collaborators, #retrieve_repo_label, #retrieve_repo_labels, #retrieve_user_byemail, #retrieve_user_byusername, #retrieve_user_follower, #retrieve_user_followers, #retrieve_user_following, #retrieve_watcher, #retrieve_watchers

Methods included from GHTorrent::APIClient

#api_request, #num_pages, #paged_api_request

Methods inherited from GHTorrent::Command

#command_name, #override_config, #process_options, #queue_client, run, #version

Instance Method Details

#go ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/ghtorrent/commands/ght_get_more_commits.rb', line 49

def go

  @ght ||= GHTorrent::Mirror.new(settings)
  user_entry = @ght.transaction{@ght.ensure_user(ARGV[0], false, false)}

  if user_entry.nil?
    Trollop::die "Cannot find user #{owner}"
  end

  user = user_entry[:login]

  repo_entry = @ght.transaction{@ght.ensure_repo(ARGV[0], ARGV[1])}

  if repo_entry.nil?
    Trollop::die "Cannot find repository #{owner}/#{ARGV[1]}"
  end

  repo = repo_entry[:name]

  head = if options[:full] == false
           @ght.get_db.from(:commits).\
                    where(:commits__project_id => repo_entry[:id]).\
                    order(:created_at).\
                    first[:sha]
         else
           nil
         end

  total_commits = 0
  old_head = nil
  while (true)
    begin
      debug("Retrieving more commits for #{user}/#{repo} from head: #{head}")

      @settings = override_config(@settings, :mirror_history_pages_back, 1)
      commits = retrieve_commits(repo, head, user)

      if commits.nil? or commits.empty? or commits.size == 1
        break
      end

      head = commits.last['sha']

      commits.map do |c|
        total_commits += 1

        if options[:num] < total_commits
          info("Already retrieved #{total_commits} commits. Stopping.")
          return
        end

        if Time.parse(c['commit']['author']['date']) < Time.at(options[:upto])
          info("Commit #{c['sha']} older than #{Time.at(options[:upto])}. Stopping.")
          return
        end

        @ght.transaction do
          @ght.ensure_commit(repo, c['sha'], user)
        end
      end
    rescue StandardError => e
      warn("Error processing: #{e}")
      warn(e.backtrace.join("\n"))
      if old_head == head
        info("Commit #{c['sha']} older than #{Time.at(options[:upto])}. Stopping.")
        fail("Cannot retrieve commits from head: #{head}")
      end
      old_head = head
    end
  end
  debug("Processed #{total_commits} commits for #{user}/#{repo}")
end

#persister ⇒ Object

def logger

@ght.logger

end



44
45
46
47
 
# File 'lib/ghtorrent/commands/ght_get_more_commits.rb', line 44

def persister
  @persister ||= connect(:mongo, settings)
  @persister
end

#prepare_options(options) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/ghtorrent/commands/ght_get_more_commits.rb', line 17

def prepare_options(options)
  options.banner <<-BANNER
Retrieves more commits for the provided repository

#{command_name} [options] owner repo

#{command_name} options:
  BANNER

  options.opt :num, 'Number of commits to retrieve',
              :short => 'n', :default => 1024 * 1024 * 1024, :type => :int
  options.opt :full, 'Retrieve all commits, starting from the latest available.
                      If not set, will start from latest stored commit',
              :short => 'f', :default => false, :type => :boolean
  options.opt :upto, 'Get all commits up to the provided timestamp',
              :short => 'x', :default => 0, :type => :int
end

#validate ⇒ Object 



35
36
37
38
 
# File 'lib/ghtorrent/commands/ght_get_more_commits.rb', line 35

def validate
  super
  Trollop::die "Two arguments are required" unless args[0] && !args[0].empty?
end