Remove nodesets from builds canceled during reconfiguration
We observed errant behavior in the configuration covered by
test_reconfigure_window_shrink in production when a reconfiguration
shrunk the active window to less than the current value when
jobs had already completed.
Correct the underlying issue by removing the nodeset associated with
a build from the buildset when the reconfiguration routine cancels it.
Then, if we later launch the same job for some reason, we will obtain
a new nodeset.
If the build is running at the time it's canceled, we will still need
the scheduler to return the nodeset to nodepool. Since it currently
relies on the value in the buildset to find the nodeset, attach the
nodeset to the build directly, so that even if we have removed the
nodeset from the buildset, the scheduler will still have a pointer
to the nodeset when the build completes.
Having said all that, we really don't want to waste resources by
shrinking the window on reconfiguration. A future change is likely
to correct that and very likely invalidate the test just added. The
only other time a build is likely to be canceled during reconfiguration
yet used again later is if a job is removed, then added back to a
project while changes are in the queue. So that we continue to have
a test which covers this case, add a second test based on that scenario.
Both of these tests fail without the included fix.
Change-Id: If61b34e0f1464cb69d9d0b9053e05f1af996a67b
diff --git a/tests/fixtures/layouts/reconfigure-remove-add.yaml b/tests/fixtures/layouts/reconfigure-remove-add.yaml
new file mode 100644
index 0000000..c9bccd3
--- /dev/null
+++ b/tests/fixtures/layouts/reconfigure-remove-add.yaml
@@ -0,0 +1,41 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ start:
+ gerrit:
+ Verified: 0
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+
+- job:
+ name: base
+ parent: null
+ nodeset:
+ nodes:
+ - label: ubuntu-xenial
+ name: controller
+
+- job:
+ name: job1
+ run: playbooks/job1.yaml
+
+- job:
+ name: job2
+ run: playbooks/job2.yaml
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - job1
+ - job2
diff --git a/tests/fixtures/layouts/reconfigure-remove-add2.yaml b/tests/fixtures/layouts/reconfigure-remove-add2.yaml
new file mode 100644
index 0000000..33c169e
--- /dev/null
+++ b/tests/fixtures/layouts/reconfigure-remove-add2.yaml
@@ -0,0 +1,40 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ start:
+ gerrit:
+ Verified: 0
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+
+- job:
+ name: base
+ parent: null
+ nodeset:
+ nodes:
+ - label: ubuntu-xenial
+ name: controller
+
+- job:
+ name: job1
+ run: playbooks/job1.yaml
+
+- job:
+ name: job2
+ run: playbooks/job2.yaml
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - job1
diff --git a/tests/fixtures/layouts/reconfigure-window.yaml b/tests/fixtures/layouts/reconfigure-window.yaml
new file mode 100644
index 0000000..c9bccd3
--- /dev/null
+++ b/tests/fixtures/layouts/reconfigure-window.yaml
@@ -0,0 +1,41 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ start:
+ gerrit:
+ Verified: 0
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+
+- job:
+ name: base
+ parent: null
+ nodeset:
+ nodes:
+ - label: ubuntu-xenial
+ name: controller
+
+- job:
+ name: job1
+ run: playbooks/job1.yaml
+
+- job:
+ name: job2
+ run: playbooks/job2.yaml
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - job1
+ - job2
diff --git a/tests/fixtures/layouts/reconfigure-window2.yaml b/tests/fixtures/layouts/reconfigure-window2.yaml
new file mode 100644
index 0000000..8949f7d
--- /dev/null
+++ b/tests/fixtures/layouts/reconfigure-window2.yaml
@@ -0,0 +1,47 @@
+- pipeline:
+ name: gate
+ manager: dependent
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ start:
+ gerrit:
+ Verified: 0
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+ window: 1
+ window-floor: 1
+ window-increase-type: linear
+ window-increase-factor: 1
+ window-decrease-type: linear
+ window-decrease-factor: 1
+
+- job:
+ name: base
+ parent: null
+ nodeset:
+ nodes:
+ - label: ubuntu-xenial
+ name: controller
+
+- job:
+ name: job1
+ run: playbooks/job1.yaml
+
+- job:
+ name: job2
+ run: playbooks/job2.yaml
+
+- project:
+ name: org/project
+ gate:
+ jobs:
+ - job1
+ - job2
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 53a20ff..264883c 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -3853,6 +3853,91 @@
self.assertEqual(queue.window_floor, 1)
self.assertEqual(C.data['status'], 'MERGED')
+ @simple_layout('layouts/reconfigure-window.yaml')
+ def test_reconfigure_window_shrink(self):
+ # Test the active window shrinking during reconfiguration
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ B.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ queue = tenant.layout.pipelines['gate'].queues[0]
+ self.assertEqual(queue.window, 20)
+ self.assertTrue(len(self.builds), 4)
+
+ self.executor_server.release('job1')
+ self.waitUntilSettled()
+ self.commitConfigUpdate('org/common-config',
+ 'layouts/reconfigure-window2.yaml')
+ self.sched.reconfigure(self.config)
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ queue = tenant.layout.pipelines['gate'].queues[0]
+ self.assertEqual(queue.window, 1)
+ # B is now outside the window, but builds haven't been canceled
+ self.assertTrue(len(self.builds), 4)
+
+ self.sched.reconfigure(self.config)
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ queue = tenant.layout.pipelines['gate'].queues[0]
+ self.assertEqual(queue.window, 1)
+ # B's builds have been canceled now
+ self.assertTrue(len(self.builds), 2)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ # This will run new builds for B
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='job1', result='SUCCESS', changes='1,1'),
+ dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
+ dict(name='job2', result='SUCCESS', changes='1,1'),
+ dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
+ dict(name='job1', result='SUCCESS', changes='1,1 2,1'),
+ dict(name='job2', result='SUCCESS', changes='1,1 2,1'),
+ ], ordered=False)
+
+ @simple_layout('layouts/reconfigure-remove-add.yaml')
+ def test_reconfigure_remove_add(self):
+ # Test removing, then adding a job while in queue
+ self.executor_server.hold_jobs_in_build = True
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertTrue(len(self.builds), 2)
+ self.executor_server.release('job2')
+ self.assertTrue(len(self.builds), 1)
+
+ # Remove job2
+ self.commitConfigUpdate('org/common-config',
+ 'layouts/reconfigure-remove-add2.yaml')
+ self.sched.reconfigure(self.config)
+ self.assertTrue(len(self.builds), 1)
+
+ # Add job2 back
+ self.commitConfigUpdate('org/common-config',
+ 'layouts/reconfigure-remove-add.yaml')
+ self.sched.reconfigure(self.config)
+ self.assertTrue(len(self.builds), 2)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ # This will run new builds for B
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='job2', result='SUCCESS', changes='1,1'),
+ dict(name='job1', result='SUCCESS', changes='1,1'),
+ dict(name='job2', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
def test_worker_update_metadata(self):
"Test if a worker can send back metadata about itself"
self.executor_server.hold_jobs_in_build = True