Class: GHTRepoRetriever

Inherits:
Object
  • Object
show all
Includes:
GHTorrent::Retriever, GHTorrent::Settings
Defined in:
lib/ghtorrent/commands/ght_retrieve_repos.rb

Constant Summary

Constants included from GHTorrent::Settings

GHTorrent::Settings::CONFIGKEYS, GHTorrent::Settings::DEFAULTS

Instance Method Summary collapse

Methods included from GHTorrent::Retriever

#get_event, #get_events, #get_repo_events, #persister, #retrieve_commit, #retrieve_commit_comment, #retrieve_commit_comments, #retrieve_commits, #retrieve_fork, #retrieve_forks, #retrieve_issue, #retrieve_issue_comment, #retrieve_issue_comments, #retrieve_issue_event, #retrieve_issue_events, #retrieve_issue_labels, #retrieve_issues, #retrieve_languages, #retrieve_org, #retrieve_org_members, #retrieve_orgs, #retrieve_pull_req_comment, #retrieve_pull_req_comments, #retrieve_pull_req_commits, #retrieve_pull_request, #retrieve_pull_requests, #retrieve_repo, #retrieve_repo_collaborator, #retrieve_repo_collaborators, #retrieve_repo_label, #retrieve_repo_labels, #retrieve_user_byemail, #retrieve_user_byusername, #retrieve_user_follower, #retrieve_user_followers, #retrieve_user_following, #retrieve_watcher, #retrieve_watchers

Methods included from GHTorrent::Logging

#debug, #error, #info, #loggerr, #warn

Methods included from GHTorrent::Settings

#config, #merge, #merge_config_values, #override_config

Methods included from GHTorrent::Utils

included, #read_value, #user_type, #write_value

Methods included from GHTorrent::APIClient

#api_request, #num_pages, #paged_api_request

Constructor Details

#initialize(config, queue) ⇒ GHTRepoRetriever

Returns a new instance of GHTRepoRetriever.



22
23
24
25
 
# File 'lib/ghtorrent/commands/ght_retrieve_repos.rb', line 22

def initialize(config, queue)
  @config = config
  @queue = queue
end

Instance Method Details

#ght ⇒ Object 



31
32
33
34
 
# File 'lib/ghtorrent/commands/ght_retrieve_repos.rb', line 31

def ght
  @ght ||= TransactedGHTorrent.new(@config)
  @ght
end

#logger ⇒ Object 



27
28
29
 
# File 'lib/ghtorrent/commands/ght_retrieve_repos.rb', line 27

def logger
  ght.logger
end

#run(command) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/ghtorrent/commands/ght_retrieve_repos.rb', line 40

def run(command)

  processor = Proc.new do |msg|
    owner, repo = msg.split(/ /)

    # On rare occasions, 2 instances might try to add the same user
    # at the same time, which might lead to transaction conflicts
    # Give the script one more opportunity before bailing out
    user_entry = nil
    i = 0

    while user_entry.nil? and i < 10 do
      i += 1
      warn("Trying to get user #{owner}, attempt #{i}")
      begin
        user_entry = ght.transaction { ght.ensure_user(owner, false, false) }
      rescue StandardError => e
        warn e.message
      end
    end

    if user_entry.nil?
      warn("Cannot find user #{owner}")
      next
    end

    repo_entry = ght.transaction { ght.ensure_repo(owner, repo,
                                                   recursive = false) }

    if repo_entry.nil?
      warn("Cannot find repository #{owner}/#{repo}")
      next
    end

    debug("Retrieving repo #{owner}/#{repo}")

    retrieval_stages = %w(ensure_commits ensure_forks ensure_pull_requests
                          ensure_issues ensure_watchers ensure_labels) # ensure_project_members

    retrieval_stages.each do |x|
      run_retrieval_stage(ght, owner, repo, x)
    end

    # Repository owner bound data retrieval
    run_retrieval_stage(ght, owner, repo, 'ensure_user_followers', 
                        onlyuser = true)

    if user_entry[:type] == 'ORG'
      run_retrieval_stage(ght, owner, repo, 'ensure_org', onlyuser = true)
    end

    # Cleanup
    ght.dispose
    ght = nil
    GC.start
  end

  command.queue_client(@queue, :before, processor)
end

#run_retrieval_stage(ght, owner, repo, function, only_user = false) ⇒ Object 



100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/ghtorrent/commands/ght_retrieve_repos.rb', line 100

def run_retrieval_stage(ght, owner, repo, function, only_user = false)
  begin
    if only_user
      ght.send(function, owner)
    else
      ght.send(function, owner, repo)
    end
  rescue StandardError => e
    warn("Error processing #{function} for #{owner}/#{repo}")
    warn("Exception message #{$!}")
    warn("Exception trace #{e.backtrace.join("\n")}")
  end
end

#settings ⇒ Object 



36
37
38
 
# File 'lib/ghtorrent/commands/ght_retrieve_repos.rb', line 36

def settings
  @config
end

#stop ⇒ Object 



114
115
116
117
 
# File 'lib/ghtorrent/commands/ght_retrieve_repos.rb', line 114

def stop
  warn('Stop flag set, waiting for operations to finish')
  @stop = true
end