Class: GitHubChangelogGenerator::OctoFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/github_changelog_generator/octo_fetcher.rb

Overview

A Fetcher responsible for all requests to GitHub and all basic manipulation with related data (such as filtering, validating, e.t.c)

Example: fetcher = GitHubChangelogGenerator::OctoFetcher.new(options)

Constant Summary collapse

PER_PAGE_NUMBER = 
100
MAX_THREAD_NUMBER = 
25
MAX_FORBIDDEN_RETRIES = 
100
CHANGELOG_GITHUB_TOKEN = 
"CHANGELOG_GITHUB_TOKEN"
GH_RATE_LIMIT_EXCEEDED_MSG = 
"Warning: Can't finish operation: GitHub API rate limit exceeded, change log may be " \
"missing some issues. You can limit the number of issues fetched using the `--max-issues NUM` argument."
NO_TOKEN_PROVIDED = 
"Warning: No token provided (-t option) and variable $CHANGELOG_GITHUB_TOKEN was not found. " \
"This script can make only 50 requests to GitHub API per hour without token!"

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ OctoFetcher

Returns a new instance of OctoFetcher.

Parameters:

  • options (Hash) (defaults to: {}) —

    Options passed in

Options Hash (options):

  • :user (String) —

    GitHub username

  • :project (String) —

    GitHub project

  • :since (String) —

    Only issues updated at or after this time are returned. This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ. eg. Time.parse(“2016-01-01 10:00:00”).iso8601

  • :http_cache (Boolean) —

    Use ActiveSupport::Cache::FileStore to cache http requests

  • :cache_file (Boolean) —

    If using http_cache, this is the cache file path

  • :cache_log (Boolean) —

    If using http_cache, this is the cache log file path



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 26

def initialize(options = {}) # rubocop:disable Metrics/CyclomaticComplexity
  @options      = options || {}
  @user         = @options[:user]
  @project      = @options[:project]
  @since        = @options[:since]
  @http_cache   = @options[:http_cache]
  @cache_file   = @options.fetch(:cache_file, "/tmp/github-changelog-http-cache") if @http_cache
  @cache_log    = @options.fetch(:cache_log, "/tmp/github-changelog-logger.log") if @http_cache
  init_cache if @http_cache

  @github_token = fetch_github_token

  @request_options               = { per_page: PER_PAGE_NUMBER }
  @github_options                = {}
  @github_options[:access_token] = @github_token unless @github_token.nil?
  @github_options[:api_endpoint] = @options[:github_endpoint] unless @options[:github_endpoint].nil?

  client_type = @options[:github_endpoint].nil? ? Octokit::Client : Octokit::EnterpriseAdminClient
  @client     = client_type.new(@github_options)
end

Instance Method Details

#calculate_pages(client, method, request_options) ⇒ Integer

Returns the number of pages for a API call

Returns:

  • (Integer) —

    number of pages for this API call in total



75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 75

def calculate_pages(client, method, request_options)
  # Makes the first API call so that we can call last_response
  check_github_response do
    client.send(method, user_project, @request_options.merge(request_options))
  end

  last_response = client.last_response

  if (last_pg = last_response.rels[:last])
    querystring_as_hash(last_pg.href)["page"].to_i
  else
    1
  end
end

#fetch_closed_issues_and_pr ⇒ Tuple

This method fetch all closed issues and separate them to pull requests and pure issues (pull request is kind of issue in term of GitHub)

Returns:

  • (Tuple) —

    with (issues [Array <Hash>], pull-requests [Array <Hash>])



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 120

def fetch_closed_issues_and_pr
  print "Fetching closed issues...\r" if @options[:verbose]
  issues = []
  options = {
    state: "closed",
    filter: "all",
    labels: nil
  }
  options[:since] = @since unless @since.nil?

  page_i      = 0
  count_pages = calculate_pages(@client, "issues", options)

  iterate_pages(@client, "issues", options) do |new_issues|
    page_i += PER_PAGE_NUMBER
    print_in_same_line("Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}")
    issues.concat(new_issues)
    break if @options[:max_issues] && issues.length >= @options[:max_issues]
  end
  print_empty_line
  Helper.log.info "Received issues: #{issues.count}"

  issues = issues.map { |h| stringify_keys_deep(h.to_hash) }

  # separate arrays of issues and pull requests:
  issues.partition do |x|
    x["pull_request"].nil?
  end
end

#fetch_closed_pull_requests ⇒ Array <Hash>

Fetch all pull requests. We need them to detect :merged_at parameter

Returns:

  • (Array <Hash>) —

    all pull requests



153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 153

def fetch_closed_pull_requests
  pull_requests = []
  options = { state: "closed" }

  unless @options[:release_branch].nil?
    options[:base] = @options[:release_branch]
  end

  page_i      = 0
  count_pages = calculate_pages(@client, "pull_requests", options)

  iterate_pages(@client, "pull_requests", options) do |new_pr|
    page_i += PER_PAGE_NUMBER
    log_string = "Fetching merged dates... #{page_i}/#{count_pages * PER_PAGE_NUMBER}"
    print_in_same_line(log_string)
    pull_requests.concat(new_pr)
  end
  print_empty_line

  Helper.log.info "Pull Request count: #{pull_requests.count}"
  pull_requests = pull_requests.map { |h| stringify_keys_deep(h.to_hash) }
  pull_requests
end

#fetch_commit(event) ⇒ Hash

Fetch commit for specified event

Returns:

  • (Hash) 


222
223
224
225
226
227
228
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 222

def fetch_commit(event)
  check_github_response do
    commit = @client.commit(user_project, event["commit_id"])
    commit = stringify_keys_deep(commit.to_hash)
    commit
  end
end

#fetch_date_of_tag(tag) ⇒ Time

Fetch tag time from repo

Parameters:

  • tag (Hash) —

    GitHub data item about a Tag

Returns:

  • (Time) —

    time of specified tag



212
213
214
215
216
217
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 212

def fetch_date_of_tag(tag)
  commit_data = check_github_response { @client.commit(user_project, tag["commit"]["sha"]) }
  commit_data = stringify_keys_deep(commit_data.to_hash)

  commit_data["commit"]["committer"]["date"]
end

#fetch_events_async(issues) ⇒ Void

Fetch event for all issues and add them to ‘events’

Parameters:

  • issues (Array)

Returns:

  • (Void) 


181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 181

def fetch_events_async(issues)
  i       = 0
  threads = []

  issues.each_slice(MAX_THREAD_NUMBER) do |issues_slice|
    issues_slice.each do |issue|
      threads << Thread.new do
        issue["events"] = []
        iterate_pages(@client, "issue_events", issue["number"], {}) do |new_event|
          issue["events"].concat(new_event)
        end
        issue["events"] = issue["events"].map { |h| stringify_keys_deep(h.to_hash) }
        print_in_same_line("Fetching events for issues and PR: #{i + 1}/#{issues.count}")
        i += 1
      end
    end
    threads.each(&:join)
    threads = []
  end

  # to clear line from prev print
  print_empty_line

  Helper.log.info "Fetching events for issues and PR: #{i}"
end

#get_all_tags ⇒ Array <Hash>

Fetch all tags from repo

Returns:

  • (Array <Hash>) —

    array of tags



66
67
68
69
70
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 66

def get_all_tags
  print "Fetching tags...\r" if @options[:verbose]

  check_github_response { github_fetch_tags }
end

#github_fetch_tags ⇒ Array <Hash>

Fill input array with tags

Returns:

  • (Array <Hash>) —

    array of tags in repo



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 93

def github_fetch_tags
  tags        = []
  page_i      = 0
  count_pages = calculate_pages(@client, "tags", {})

  iterate_pages(@client, "tags", {}) do |new_tags|
    page_i += PER_PAGE_NUMBER
    print_in_same_line("Fetching tags... #{page_i}/#{count_pages * PER_PAGE_NUMBER}")
    tags.concat(new_tags)
  end
  print_empty_line

  if tags.count == 0
    Helper.log.warn "Warning: Can't find any tags in repo. \
Make sure, that you push tags to remote repo via 'git push --tags'"
  else
    Helper.log.info "Found #{tags.count} tags"
  end
  # tags are a Sawyer::Resource. Convert to hash
  tags = tags.map { |h| stringify_keys_deep(h.to_hash) }
  tags
end

#init_cache ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/github_changelog_generator/octo_fetcher.rb', line 47

def init_cache
  middleware_opts = {
    serializer: Marshal,
    store: ActiveSupport::Cache::FileStore.new(@cache_file),
    logger: Logger.new(@cache_log),
    shared_cache: false
  }
  stack = Faraday::RackBuilder.new do |builder|
    builder.use Faraday::HttpCache, middleware_opts
    builder.use Octokit::Response::RaiseError
    builder.adapter Faraday.default_adapter
    # builder.response :logger
  end
  Octokit.middleware = stack
end