Skip to content

Commit 2a4304a

Browse files
smcgivernsmcgivern
committed
Merge branch 'rc/32308-rspec-retry-hack' into 'master'
Detect and keep track of flaky specs See merge request !13021
2 parents 69eb4be + 40bbc9d commit 2a4304a

File tree

13 files changed

+706
-106
lines changed

13 files changed

+706
-106
lines changed

‎.gitlab-ci.yml

+125-87
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ variables:
2727
GET_SOURCES_ATTEMPTS: "3"
2828
KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
2929
KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
30+
FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/${CI_PROJECT_NAME}/report-master.json
3031

3132
before_script:
3233
- bundle --version
@@ -45,16 +46,17 @@ stages:
4546
tags:
4647
- gitlab-org
4748

48-
.knapsack-state: &knapsack-state
49+
.tests-metadata-state: &tests-metadata-state
4950
services: []
5051
variables:
5152
SETUP_DB: "false"
5253
USE_BUNDLE_INSTALL: "false"
53-
KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
54+
TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
5455
artifacts:
5556
expire_in: 31d
5657
paths:
5758
- knapsack/
59+
- rspec_flaky/
5860

5961
.use-pg: &use-pg
6062
services:
@@ -86,7 +88,7 @@ stages:
8688
except:
8789
- /(^docs[\/-].*|.*-docs$)/
8890

89-
.rspec-knapsack: &rspec-knapsack
91+
.rspec-metadata: &rspec-metadata
9092
<<: *dedicated-runner
9193
<<: *pull-cache
9294
stage: test
@@ -96,8 +98,13 @@ stages:
9698
- export CI_NODE_TOTAL=${JOB_NAME[-1]}
9799
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
98100
- export KNAPSACK_GENERATE_REPORT=true
101+
- export ALL_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/${CI_PROJECT_NAME}/all_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
102+
- export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/${CI_PROJECT_NAME}/new_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
103+
- export FLAKY_RSPEC_GENERATE_REPORT=true
99104
- export CACHE_CLASSES=true
100105
- cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
106+
- cp ${FLAKY_RSPEC_SUITE_REPORT_PATH} ${ALL_FLAKY_RSPEC_REPORT_PATH}
107+
- '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}'
101108
- scripts/gitaly-test-spawn
102109
- knapsack rspec "--color --format documentation"
103110
artifacts:
@@ -106,20 +113,21 @@ stages:
106113
paths:
107114
- coverage/
108115
- knapsack/
116+
- rspec_flaky/
109117
- tmp/capybara/
110118

111-
.rspec-knapsack-pg: &rspec-knapsack-pg
112-
<<: *rspec-knapsack
119+
.rspec-metadata-pg: &rspec-metadata-pg
120+
<<: *rspec-metadata
113121
<<: *use-pg
114122
<<: *except-docs
115123

116-
.rspec-knapsack-mysql: &rspec-knapsack-mysql
117-
<<: *rspec-knapsack
124+
.rspec-metadata-mysql: &rspec-metadata-mysql
125+
<<: *rspec-metadata
118126
<<: *use-mysql
119127
<<: *only-if-want-mysql
120128
<<: *except-docs
121129

122-
.spinach-knapsack: &spinach-knapsack
130+
.spinach-metadata: &spinach-metadata
123131
<<: *dedicated-runner
124132
<<: *pull-cache
125133
stage: test
@@ -140,13 +148,13 @@ stages:
140148
- knapsack/
141149
- tmp/capybara/
142150

143-
.spinach-knapsack-pg: &spinach-knapsack-pg
144-
<<: *spinach-knapsack
151+
.spinach-metadata-pg: &spinach-metadata-pg
152+
<<: *spinach-metadata
145153
<<: *use-pg
146154
<<: *except-docs
147155

148-
.spinach-knapsack-mysql: &spinach-knapsack-mysql
149-
<<: *spinach-knapsack
156+
.spinach-metadata-mysql: &spinach-metadata-mysql
157+
<<: *spinach-metadata
150158
<<: *use-mysql
151159
<<: *only-if-want-mysql
152160
<<: *except-docs
@@ -176,40 +184,70 @@ build-package:
176184
- //@gitlab-org/gitlab-ce
177185
- //@gitlab-org/gitlab-ee
178186

179-
# Prepare and merge knapsack tests
180-
knapsack:
181-
<<: *knapsack-state
187+
# Retrieve knapsack and rspec_flaky reports
188+
retrieve-tests-metadata:
189+
<<: *tests-metadata-state
182190
<<: *dedicated-runner
183191
<<: *except-docs
184192
stage: prepare
185193
cache:
186-
key: knapsack
187-
paths:
188-
- knapsack/
194+
key: tests_metadata
189195
policy: pull
190196
script:
191197
- mkdir -p knapsack/${CI_PROJECT_NAME}/
192-
- wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
193-
- wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
198+
- wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
199+
- wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
194200
- '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
195201
- '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'
202+
- mkdir -p rspec_flaky/${CI_PROJECT_NAME}/
203+
- wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
204+
- '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
196205

197-
update-knapsack:
198-
<<: *knapsack-state
206+
update-tests-metadata:
207+
<<: *tests-metadata-state
199208
<<: *dedicated-runner
200209
<<: *only-canonical-masters
201210
stage: post-test
202211
cache:
203-
key: knapsack
212+
key: tests_metadata
204213
paths:
205214
- knapsack/
215+
- rspec_flaky/
206216
policy: push
207217
script:
208218
- retry gem install fog-aws mime-types
209219
- scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
210220
- scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach-pg_node_*.json
211-
- '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
221+
- scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/${CI_PROJECT_NAME}/all_node_*.json
222+
- '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
223+
- '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
212224
- rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
225+
- rm -f rspec_flaky/${CI_PROJECT_NAME}/all_node_*.json
226+
227+
flaky-examples-check:
228+
<<: *dedicated-runner
229+
image: ruby:2.3-alpine
230+
services: []
231+
before_script: []
232+
cache: {}
233+
variables:
234+
SETUP_DB: "false"
235+
USE_BUNDLE_INSTALL: "false"
236+
NEW_FLAKY_SPECS_REPORT: rspec_flaky/${CI_PROJECT_NAME}/new_rspec_flaky_examples.json
237+
stage: post-test
238+
allow_failure: yes
239+
only:
240+
- branches
241+
except:
242+
- master
243+
artifacts:
244+
expire_in: 30d
245+
paths:
246+
- rspec_flaky/
247+
script:
248+
- '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}'
249+
- scripts/merge-reports $NEW_FLAKY_SPECS_REPORT rspec_flaky/${CI_PROJECT_NAME}/new_node_*.json
250+
- scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT
213251

214252
setup-test-env:
215253
<<: *use-pg
@@ -232,69 +270,69 @@ setup-test-env:
232270
- public/assets
233271
- tmp/tests
234272

235-
rspec-pg 0 25: *rspec-knapsack-pg
236-
rspec-pg 1 25: *rspec-knapsack-pg
237-
rspec-pg 2 25: *rspec-knapsack-pg
238-
rspec-pg 3 25: *rspec-knapsack-pg
239-
rspec-pg 4 25: *rspec-knapsack-pg
240-
rspec-pg 5 25: *rspec-knapsack-pg
241-
rspec-pg 6 25: *rspec-knapsack-pg
242-
rspec-pg 7 25: *rspec-knapsack-pg
243-
rspec-pg 8 25: *rspec-knapsack-pg
244-
rspec-pg 9 25: *rspec-knapsack-pg
245-
rspec-pg 10 25: *rspec-knapsack-pg
246-
rspec-pg 11 25: *rspec-knapsack-pg
247-
rspec-pg 12 25: *rspec-knapsack-pg
248-
rspec-pg 13 25: *rspec-knapsack-pg
249-
rspec-pg 14 25: *rspec-knapsack-pg
250-
rspec-pg 15 25: *rspec-knapsack-pg
251-
rspec-pg 16 25: *rspec-knapsack-pg
252-
rspec-pg 17 25: *rspec-knapsack-pg
253-
rspec-pg 18 25: *rspec-knapsack-pg
254-
rspec-pg 19 25: *rspec-knapsack-pg
255-
rspec-pg 20 25: *rspec-knapsack-pg
256-
rspec-pg 21 25: *rspec-knapsack-pg
257-
rspec-pg 22 25: *rspec-knapsack-pg
258-
rspec-pg 23 25: *rspec-knapsack-pg
259-
rspec-pg 24 25: *rspec-knapsack-pg
260-
261-
rspec-mysql 0 25: *rspec-knapsack-mysql
262-
rspec-mysql 1 25: *rspec-knapsack-mysql
263-
rspec-mysql 2 25: *rspec-knapsack-mysql
264-
rspec-mysql 3 25: *rspec-knapsack-mysql
265-
rspec-mysql 4 25: *rspec-knapsack-mysql
266-
rspec-mysql 5 25: *rspec-knapsack-mysql
267-
rspec-mysql 6 25: *rspec-knapsack-mysql
268-
rspec-mysql 7 25: *rspec-knapsack-mysql
269-
rspec-mysql 8 25: *rspec-knapsack-mysql
270-
rspec-mysql 9 25: *rspec-knapsack-mysql
271-
rspec-mysql 10 25: *rspec-knapsack-mysql
272-
rspec-mysql 11 25: *rspec-knapsack-mysql
273-
rspec-mysql 12 25: *rspec-knapsack-mysql
274-
rspec-mysql 13 25: *rspec-knapsack-mysql
275-
rspec-mysql 14 25: *rspec-knapsack-mysql
276-
rspec-mysql 15 25: *rspec-knapsack-mysql
277-
rspec-mysql 16 25: *rspec-knapsack-mysql
278-
rspec-mysql 17 25: *rspec-knapsack-mysql
279-
rspec-mysql 18 25: *rspec-knapsack-mysql
280-
rspec-mysql 19 25: *rspec-knapsack-mysql
281-
rspec-mysql 20 25: *rspec-knapsack-mysql
282-
rspec-mysql 21 25: *rspec-knapsack-mysql
283-
rspec-mysql 22 25: *rspec-knapsack-mysql
284-
rspec-mysql 23 25: *rspec-knapsack-mysql
285-
rspec-mysql 24 25: *rspec-knapsack-mysql
286-
287-
spinach-pg 0 5: *spinach-knapsack-pg
288-
spinach-pg 1 5: *spinach-knapsack-pg
289-
spinach-pg 2 5: *spinach-knapsack-pg
290-
spinach-pg 3 5: *spinach-knapsack-pg
291-
spinach-pg 4 5: *spinach-knapsack-pg
292-
293-
spinach-mysql 0 5: *spinach-knapsack-mysql
294-
spinach-mysql 1 5: *spinach-knapsack-mysql
295-
spinach-mysql 2 5: *spinach-knapsack-mysql
296-
spinach-mysql 3 5: *spinach-knapsack-mysql
297-
spinach-mysql 4 5: *spinach-knapsack-mysql
273+
rspec-pg 0 25: *rspec-metadata-pg
274+
rspec-pg 1 25: *rspec-metadata-pg
275+
rspec-pg 2 25: *rspec-metadata-pg
276+
rspec-pg 3 25: *rspec-metadata-pg
277+
rspec-pg 4 25: *rspec-metadata-pg
278+
rspec-pg 5 25: *rspec-metadata-pg
279+
rspec-pg 6 25: *rspec-metadata-pg
280+
rspec-pg 7 25: *rspec-metadata-pg
281+
rspec-pg 8 25: *rspec-metadata-pg
282+
rspec-pg 9 25: *rspec-metadata-pg
283+
rspec-pg 10 25: *rspec-metadata-pg
284+
rspec-pg 11 25: *rspec-metadata-pg
285+
rspec-pg 12 25: *rspec-metadata-pg
286+
rspec-pg 13 25: *rspec-metadata-pg
287+
rspec-pg 14 25: *rspec-metadata-pg
288+
rspec-pg 15 25: *rspec-metadata-pg
289+
rspec-pg 16 25: *rspec-metadata-pg
290+
rspec-pg 17 25: *rspec-metadata-pg
291+
rspec-pg 18 25: *rspec-metadata-pg
292+
rspec-pg 19 25: *rspec-metadata-pg
293+
rspec-pg 20 25: *rspec-metadata-pg
294+
rspec-pg 21 25: *rspec-metadata-pg
295+
rspec-pg 22 25: *rspec-metadata-pg
296+
rspec-pg 23 25: *rspec-metadata-pg
297+
rspec-pg 24 25: *rspec-metadata-pg
298+
299+
rspec-mysql 0 25: *rspec-metadata-mysql
300+
rspec-mysql 1 25: *rspec-metadata-mysql
301+
rspec-mysql 2 25: *rspec-metadata-mysql
302+
rspec-mysql 3 25: *rspec-metadata-mysql
303+
rspec-mysql 4 25: *rspec-metadata-mysql
304+
rspec-mysql 5 25: *rspec-metadata-mysql
305+
rspec-mysql 6 25: *rspec-metadata-mysql
306+
rspec-mysql 7 25: *rspec-metadata-mysql
307+
rspec-mysql 8 25: *rspec-metadata-mysql
308+
rspec-mysql 9 25: *rspec-metadata-mysql
309+
rspec-mysql 10 25: *rspec-metadata-mysql
310+
rspec-mysql 11 25: *rspec-metadata-mysql
311+
rspec-mysql 12 25: *rspec-metadata-mysql
312+
rspec-mysql 13 25: *rspec-metadata-mysql
313+
rspec-mysql 14 25: *rspec-metadata-mysql
314+
rspec-mysql 15 25: *rspec-metadata-mysql
315+
rspec-mysql 16 25: *rspec-metadata-mysql
316+
rspec-mysql 17 25: *rspec-metadata-mysql
317+
rspec-mysql 18 25: *rspec-metadata-mysql
318+
rspec-mysql 19 25: *rspec-metadata-mysql
319+
rspec-mysql 20 25: *rspec-metadata-mysql
320+
rspec-mysql 21 25: *rspec-metadata-mysql
321+
rspec-mysql 22 25: *rspec-metadata-mysql
322+
rspec-mysql 23 25: *rspec-metadata-mysql
323+
rspec-mysql 24 25: *rspec-metadata-mysql
324+
325+
spinach-pg 0 5: *spinach-metadata-pg
326+
spinach-pg 1 5: *spinach-metadata-pg
327+
spinach-pg 2 5: *spinach-metadata-pg
328+
spinach-pg 3 5: *spinach-metadata-pg
329+
spinach-pg 4 5: *spinach-metadata-pg
330+
331+
spinach-mysql 0 5: *spinach-metadata-mysql
332+
spinach-mysql 1 5: *spinach-metadata-mysql
333+
spinach-mysql 2 5: *spinach-metadata-mysql
334+
spinach-mysql 3 5: *spinach-metadata-mysql
335+
spinach-mysql 4 5: *spinach-metadata-mysql
298336

299337
# Static analysis jobs
300338
.ruby-static-analysis: &ruby-static-analysis

‎.rubocop.yml

+2-3
Original file line numberDiff line numberDiff line change
@@ -1045,16 +1045,15 @@ RSpec/BeforeAfterAll:
10451045
RSpec/DescribeClass:
10461046
Enabled: false
10471047

1048-
# Use `described_class` for tested class / module.
1048+
# Checks that the second argument to `describe` specifies a method.
10491049
RSpec/DescribeMethod:
10501050
Enabled: false
10511051

10521052
# Avoid describing symbols.
10531053
RSpec/DescribeSymbol:
10541054
Enabled: true
10551055

1056-
# Checks that the second argument to top level describe is the tested method
1057-
# name.
1056+
# Checks that tests use `described_class`.
10581057
RSpec/DescribedClass:
10591058
Enabled: true
10601059

‎doc/development/testing.md

+20-3
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,9 @@ trade-off:
157157

158158
- Unit tests are usually cheap, and you should consider them like the basement
159159
of your house: you need them to be confident that your code is behaving
160-
correctly. However if you run only unit tests without integration / system tests, you might [miss] the [big] [picture]!
161-
- Integration tests are a bit more expensive, but don't abuse them. A feature test
160+
correctly. However if you run only unit tests without integration / system
161+
tests, you might [miss] the [big] [picture]!
162+
- Integration tests are a bit more expensive, but don't abuse them. A system test
162163
is often better than an integration test that is stubbing a lot of internals.
163164
- System tests are expensive (compared to unit tests), even more if they require
164165
a JavaScript driver. Make sure to follow the guidelines in the [Speed](#test-speed)
@@ -195,11 +196,27 @@ Please consult the [dedicated "Frontend testing" guide](./fe_guide/testing.md).
195196
- Try to match the ordering of tests to the ordering within the class.
196197
- Try to follow the [Four-Phase Test][four-phase-test] pattern, using newlines
197198
to separate phases.
198-
- Try to use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'`
199+
- Use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'`
200+
- Don't assert against the absolute value of a sequence-generated attribute (see
201+
[Gotchas](gotchas.md#dont-assert-against-the-absolute-value-of-a-sequence-generated-attribute)).
202+
- Don't supply the `:each` argument to hooks since it's the default.
199203
- On `before` and `after` hooks, prefer it scoped to `:context` over `:all`
200204

201205
[four-phase-test]: https://robots.thoughtbot.com/four-phase-test
202206

207+
### Automatic retries and flaky tests detection
208+
209+
On our CI, we use [rspec-retry] to automatically retry a failing example a few
210+
times (see [`spec/spec_helper.rb`] for the precise retries count).
211+
212+
We also use a home-made `RspecFlaky::Listener` listener which records flaky
213+
examples in a JSON report file on `master` (`retrieve-tests-metadata` and `update-tests-metadata` jobs), and warns when a new flaky example
214+
is detected in any other branch (`flaky-examples-check` job). In the future, the
215+
`flaky-examples-check` job will not be allowed to fail.
216+
217+
[rspec-retry]: https://github.com/NoRedInk/rspec-retry
218+
[`spec/spec_helper.rb`]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/spec_helper.rb
219+
203220
### `let` variables
204221

205222
GitLab's RSpec suite has made extensive use of `let` variables to reduce

0 commit comments

Comments
 (0)