Fix races in timer trigger tests.
Timer trigger tests are vulnerable to a race where the timer trigger can
enqueue new builds while we are checking the number of builds queued.
This can happen during intermediate build queue checks or in
assertFinalState.
To fix this race swap in a layout that does not use the timer trigger
which will prevent further build queueing once the timer triggered
builds have done their thing.
Change-Id: Ic9b64e34c6a39275b57b1299b47d9b1d0e0e5c79
diff --git a/tests/fixtures/layout-idle.yaml b/tests/fixtures/layout-idle.yaml
index e4574fa..0870788 100644
--- a/tests/fixtures/layout-idle.yaml
+++ b/tests/fixtures/layout-idle.yaml
@@ -8,5 +8,5 @@
projects:
- name: org/project
periodic:
- - project-test1
- - project-test2
+ - project-bitrot-stable-old
+ - project-bitrot-stable-older
diff --git a/tests/fixtures/layout-no-timer.yaml b/tests/fixtures/layout-no-timer.yaml
new file mode 100644
index 0000000..9436821
--- /dev/null
+++ b/tests/fixtures/layout-no-timer.yaml
@@ -0,0 +1,14 @@
+pipelines:
+ - name: periodic
+ manager: IndependentPipelineManager
+ # Trigger is required, set it to one that is a noop
+ # during tests that check the timer trigger.
+ trigger:
+ gerrit:
+ - event: patchset-created
+
+projects:
+ - name: org/project
+ periodic:
+ - project-bitrot-stable-old
+ - project-bitrot-stable-older
diff --git a/tests/fixtures/layout-timer-smtp.yaml b/tests/fixtures/layout-timer-smtp.yaml
index ac59df4..b5a6ce0 100644
--- a/tests/fixtures/layout-timer-smtp.yaml
+++ b/tests/fixtures/layout-timer-smtp.yaml
@@ -3,7 +3,7 @@
manager: IndependentPipelineManager
trigger:
timer:
- - time: '* * * * * */10'
+ - time: '* * * * * */1'
success:
smtp:
to: alternative_me@example.com
diff --git a/tests/fixtures/layout-timer.yaml b/tests/fixtures/layout-timer.yaml
index 9e0f66b..4904f87 100644
--- a/tests/fixtures/layout-timer.yaml
+++ b/tests/fixtures/layout-timer.yaml
@@ -15,7 +15,7 @@
manager: IndependentPipelineManager
trigger:
timer:
- - time: '* * * * * */10'
+ - time: '* * * * * */1'
projects:
- name: org/project
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 4ef1790..d060b00 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -3175,25 +3175,25 @@
self.sched.reconfigure(self.config)
self.registerJobs()
- start = time.time()
- failed = True
- while ((time.time() - start) < 30):
- if len(self.builds) == 2:
- failed = False
- break
- else:
- time.sleep(1)
-
- if failed:
- raise Exception("Expected jobs never ran")
-
+ # The pipeline triggers every second, so we should have seen
+ # several by now.
+ time.sleep(5)
self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 2)
+
port = self.webapp.server.socket.getsockname()[1]
f = urllib.urlopen("http://localhost:%s/status.json" % port)
data = f.read()
self.worker.hold_jobs_in_build = False
+ # Stop queuing timer triggered jobs so that the assertions
+ # below don't race against more jobs being queued.
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-no-timer.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
self.worker.release()
self.waitUntilSettled()
@@ -3217,29 +3217,33 @@
def test_idle(self):
"Test that frequent periodic jobs work"
self.worker.hold_jobs_in_build = True
- self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-idle.yaml')
- self.sched.reconfigure(self.config)
- self.registerJobs()
- # The pipeline triggers every second, so we should have seen
- # several by now.
- time.sleep(5)
- self.waitUntilSettled()
- self.assertEqual(len(self.builds), 2)
- self.worker.release('.*')
- self.waitUntilSettled()
- self.assertEqual(len(self.builds), 0)
- self.assertEqual(len(self.history), 2)
+ for x in range(1, 3):
+ # Test that timer triggers periodic jobs even across
+ # layout config reloads.
+ # Start timer trigger
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-idle.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
- time.sleep(5)
- self.waitUntilSettled()
- self.assertEqual(len(self.builds), 2)
- self.assertEqual(len(self.history), 2)
- self.worker.release('.*')
- self.waitUntilSettled()
- self.assertEqual(len(self.builds), 0)
- self.assertEqual(len(self.history), 4)
+ # The pipeline triggers every second, so we should have seen
+ # several by now.
+ time.sleep(5)
+ self.waitUntilSettled()
+
+ # Stop queuing timer triggered jobs so that the assertions
+ # below don't race against more jobs being queued.
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-no-timer.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
+
+ self.assertEqual(len(self.builds), 2)
+ self.worker.release('.*')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 0)
+ self.assertEqual(len(self.history), x * 2)
def test_check_smtp_pool(self):
self.config.set('zuul', 'layout_config',
@@ -3274,25 +3278,22 @@
def test_timer_smtp(self):
"Test that a periodic job is triggered"
+ self.worker.hold_jobs_in_build = True
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-timer-smtp.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
- start = time.time()
- failed = True
- while ((time.time() - start) < 30):
- if len(self.history) == 2:
- failed = False
- break
- else:
- time.sleep(1)
-
- if failed:
- raise Exception("Expected jobs never ran")
-
+ # The pipeline triggers every second, so we should have seen
+ # several by now.
+ time.sleep(5)
self.waitUntilSettled()
+ self.assertEqual(len(self.builds), 2)
+ self.worker.release('.*')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 2)
+
self.assertEqual(self.getJobFromHistory(
'project-bitrot-stable-old').result, 'SUCCESS')
self.assertEqual(self.getJobFromHistory(
@@ -3311,6 +3312,15 @@
self.assertIn('Subject: Periodic check for org/project succeeded',
self.smtp_messages[0]['headers'])
+ # Stop queuing timer triggered jobs and let any that may have
+ # queued through so that end of test assertions pass.
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-no-timer.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
+ self.worker.release('.*')
+ self.waitUntilSettled()
+
def test_client_enqueue(self):
"Test that the RPC client can enqueue a change"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')