Class: Confiner::Plugins::Gitlab

Inherits:
Confiner::Plugin show all
Defined in:
lib/confiner/plugins/gitlab.rb

Constant Summary collapse

MERGE_REQUEST_TITLE = 
'[QUARANTINE] %s'
QUARANTINE_METADATA = 
%(, quarantine: { issue: '%{issue_url}', type: :investigating })
ONLY_QUARANTINE_METADATA = 
%(, quarantine: { issue: '%{issue_url}', type: :investigating, only: { %{pattern} } })

Constants inherited from Confiner::Plugin

Confiner::Plugin::DEFAULT_PLUGIN_ARGS

Instance Attribute Summary

Attributes inherited from Confiner::Plugin

#examples

Instance Method Summary collapse

Methods inherited from Confiner::Plugin

arguments, #run

Methods included from Logger

#log, log_to, log_to=, #run

Constructor Details

#initialize(options, **args) ⇒ Gitlab

GitLab Confiner Plugin

Examples:

gitlab = Confiner::Plugins::Gitlab.new({ debug: true }, private_token: 'ABC-123', project_id: 'my-group/my-project', target_project: 'my-group/my-project', failure_issue_labels: 'QA,test', failure_issue_prefix: 'Failure in ', timeout: 10, threshold: 3, endpoint: 'https://gitlab.com/api/v4', pwd: 'qa', ref: 'master')
gitlab.quarantine
gitlab.dequarantine

Parameters:

  • args (Hash) —

    the arguments for GitLab

  • :environment (Hash) —

    a customizable set of options

Options Hash (**args):

  • :private_token (String) —

    the private token for GitLab to connect to

  • :project_id (String) —

    the project id (or name) where the pipelines are fetched from

  • :target_project (String) —

    where failure issues will be searched, and where an MR will be filed

  • :failure_issue_labels (String) —

    what labels to search for when searching for the failure issue (comma separated)

  • :failure_issue_prefix (String) —

    what prefix an issue has in GitLab Issues to search for the failure issue

  • :environment (Hash) —

    metadata about the environment the tests are running and which will be confined

  • :timeout (Integer) —

    the timeout that HTTParty will consume (timeout of requests)

  • :threshold (Integer) —

    the failure / pass threshold

  • :endpoint (String) —

    the GitLab API Endpoint (e.g. gitlab.com/api/v4)

  • :pwd (String) —

    the path of the working directory for where the tests are located

  • :ref (String) —

    the default Git ref used when updating

  • :job_pattern (String) —

    the regex pattern to match names of jobs in GitLab (Job = Suite Name)

Raises:

  • (ArgumentError) 


48
49
50
51
52
53
54
55
56
 
# File 'lib/confiner/plugins/gitlab.rb', line 48

def initialize(options, **args)
  super

  ENV['GITLAB_API_HTTPARTY_OPTIONS'] = ENV.fetch('GITLAB_API_HTTPARTY_OPTIONS') { "{read_timeout: #{timeout}}" }

  raise ArgumentError, 'Missing private_token' if private_token.nil?

  @gitlab_client = ::Gitlab.client(private_token: private_token, endpoint: endpoint)
end

Instance Method Details

#dequarantine ⇒ Object

Dequarantine Action - Automatically Dequarantine tests



150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
 
# File 'lib/confiner/plugins/gitlab.rb', line 150

def dequarantine
  log :gitlab, 'Beginning Dequarantine Process', 2

  @examples = get_examples

  dequarantines = @examples.select(&:passed?).map(&:name).uniq.each_with_object([]) do |passed_example, dequarantines|
    passes = @examples.select { _1.name == passed_example && _1.passed? }
    fails = @examples.select { _1.name == passed_example && _1.failed? }

    if passes.size >= threshold
      next log(:dequarantine, "Detected #{fails.size} failures for #{passed_example}, thus, not de-quarantining", 3) unless fails.size.zero?

      dequarantines << passed_example

      example = @examples.find { _1.name == passed_example }

      log :dequarantine, "Dequarantining #{example} (#{passes.size} >= #{threshold})", 3

      begin
        file_contents = get_example_file_contents(example)
        new_contents, changed_line_no, failure_issue = remove_quarantine_metadata(file_contents, example)

        next log(:warn, <<~MESSAGE.tr("\n", ' '), 4) if file_contents == new_contents
          Unable to dequarantine. This is likely due to the quarantine being applied to the outer context.
          See https://gitlab.com/gitlab-org/quality/confiner/-/issues/3
        MESSAGE

        if @options.dry_run
          log :dry_run, 'Skipping creating branch, committing and filing merge request', 4
        else
          branch = create_branch(failure_issue, 'dequarantine', example)
          commit_changes(branch, <<~COMMIT_MESSAGE, example, new_contents)
            Dequarantine end-to-end test

            Dequarantine #{example.name}
          COMMIT_MESSAGE

          create_merge_request('[DEQUARANTINE]',example, branch) do
            markdown_occurrences = []

            passes.each do |pass|
              markdown_occurrences << "1. [#{pass.occurrence[:job]}](#{pass.occurrence[:pipeline_url]})"
            end

            meets_or_exceeds = passes.size > threshold ? 'exceeds' : 'meets'

            <<~MARKDOWN
              ## What does this MR do?

              Dequarantines the test `#{example.name}` (https://gitlab.com/#{target_project}/-/blob/#{ref}/#{example.file}#L#{changed_line_no})

              This quarantined test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have passed
              #{passes.size} times consecutively, which #{meets_or_exceeds} the threshold of #{threshold} times.

              #{markdown_occurrences.join("\n")}

              > #{failure_issue}

              <div align="center">
              (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
              </div>
            MARKDOWN
          end
        end

        log :dequarantine, "Done Dequarantining #{passed_example}", 3
      rescue => e
        log :fatal, "There was an issue dequarantining #{example}. Error was #{e.message}\n#{e.backtrace}"
      end
    end
  end

  if dequarantines.any?
    log :dequarantine, "Found #{dequarantines.size} candidates to be dequarantined", 3
  else
    log :dequarantine, 'Found no candidates to be dequarantined', 3
  end

  log :gitlab, 'Done with Dequarantine Process', 2
end

#quarantine ⇒ Object

Quarantine Action - Automatically Quarantine tests



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/confiner/plugins/gitlab.rb', line 59

def quarantine
  log :gitlab, 'Beginning Quarantine Process', 2

  if environment&.any?
    raise ArgumentError, 'Specify both environment[name] and environment[pattern]' unless environment['name'] && environment['pattern']
  end

  # store the examples from the pipelines
  @examples = get_examples

  quarantines = @examples.select(&:failed?).map(&:name).uniq.each_with_object([]) do |failed_example, quarantines|
    # count the number of failures consecutively for this example

    fails = @examples.select { _1.name == failed_example && _1.failed? }
    if fails.size >= threshold
      quarantines << failed_example

      example = @examples.find { _1.name == failed_example }

      log :quarantine, "Quarantining #{failed_example} (#{fails.size} >= #{threshold})", 3

      # check to see if there is a merge request first
      # if there is no merge request...
      #   - Check for an existing issue
      #   - Check for an existing Quarantine MR
      #   - Add a quarantine tag: `it 'should be quarantined', quarantine: { issue: 'https://issue', type: :investigating }`
      #   - File the merge request

      begin
        # begin the quarantining process
        failure_issue = get_failure_issue_for_example(example)
        file_contents = get_example_file_contents(example)
        new_contents, changed_line_no = add_quarantine_metadata(file_contents, example, failure_issue)

        log(:debug, new_contents, 4) if @options.debug

        if @options.dry_run
          log :dry_run, 'Skipping creating branch, committing and filing merge request', 4
        else
          branch = create_branch(failure_issue, 'quarantine', example)
          commit_changes(branch, <<~COMMIT_MESSAGE, example, new_contents)
            Quarantine end-to-end test

            Quarantine #{example.name}
          COMMIT_MESSAGE

          create_merge_request('[QUARANTINE]', example, branch) do
            markdown_occurrences = []

            fails.each do |fail|
              markdown_occurrences << "1. [#{fail.occurrence[:job]}](#{fail.occurrence[:pipeline_url]})"
            end

            meets_or_exceeds = fails.size > threshold ? 'exceeds' : 'meets'

            <<~MARKDOWN
              ## What does this MR do?

              Quarantines the test `#{example.name}` (https://gitlab.com/#{target_project}/-/blob/#{ref}/#{example.file}#L#{changed_line_no})

              This test has been found by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) to have failed
              #{fails.size} times, which #{meets_or_exceeds} the threshold of #{threshold} times.

              #{markdown_occurrences.join("\n")}

              > #{failure_issue['web_url']}

              <div align="center">
              (This MR was automatically generated by [Confiner](https://gitlab.com/gitlab-org/quality/confiner) at #{Time.now.utc})
              </div>
            MARKDOWN
          end
        end

        log :quarantine, "Done Quarantining #{failed_example}", 3
      rescue => e
        log :fatal, "There was an issue quarantining #{example}. Error was #{e.message}\n#{e.backtrace}"
      end
    end
  end

  if quarantines.any?
    log :quarantine, "Found #{quarantines.size} candidates to be quarantined", 3
  else
    log :quarantine, 'Found no candidates to be quarantined', 3
  end

  log :gitlab, 'Done with Quarantine Process', 2
end