Estimate job runtimes internally
Rather than relying on the workers to supply estimated job runtimes,
record the last 10 successful run times and use those to estimate
the run time of each job. This means that workers (which may be
highly distributed and lack access to a substantial job history)
no longer need to provide these values, and the central scheduler,
which is better placed to do so since italready sees all job run
times, will.
Failure times and a scoreboard of results are kept for each job
as well for potential future use in evaluating likelihood of
job success.
Change-Id: If0955e15a3da9eb842dbee02a4750a177a092d3e
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index fe7c7cc..15d33c8 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -34,7 +34,6 @@
import zuul.reporter.smtp
from tests.base import (
- BaseTestCase,
ZuulTestCase,
repack_repo,
)
@@ -44,40 +43,6 @@
'%(levelname)-8s %(message)s')
-class TestSchedulerConfigParsing(BaseTestCase):
-
- def test_parse_skip_if(self):
- job_yaml = """
-jobs:
- - name: job_name
- skip-if:
- - project: ^project_name$
- branch: ^stable/icehouse$
- all-files-match-any:
- - ^filename$
- - project: ^project2_name$
- all-files-match-any:
- - ^filename2$
- """.strip()
- data = yaml.load(job_yaml)
- config_job = data.get('jobs')[0]
- sched = zuul.scheduler.Scheduler({})
- cm = zuul.change_matcher
- expected = cm.MatchAny([
- cm.MatchAll([
- cm.ProjectMatcher('^project_name$'),
- cm.BranchMatcher('^stable/icehouse$'),
- cm.MatchAllFiles([cm.FileMatcher('^filename$')]),
- ]),
- cm.MatchAll([
- cm.ProjectMatcher('^project2_name$'),
- cm.MatchAllFiles([cm.FileMatcher('^filename2$')]),
- ]),
- ])
- matcher = sched._parseSkipIf(config_job)
- self.assertEqual(expected, matcher)
-
-
class TestScheduler(ZuulTestCase):
def test_jobs_launched(self):
@@ -495,6 +460,46 @@
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
+ def _test_time_database(self, iteration):
+ self.worker.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ time.sleep(2)
+
+ data = json.loads(self.sched.formatStatusJSON())
+ found_job = None
+ for pipeline in data['pipelines']:
+ if pipeline['name'] != 'gate':
+ continue
+ for queue in pipeline['change_queues']:
+ for head in queue['heads']:
+ for item in head:
+ for job in item['jobs']:
+ if job['name'] == 'project-merge':
+ found_job = job
+ break
+
+ self.assertIsNotNone(found_job)
+ if iteration == 1:
+ self.assertIsNotNone(found_job['estimated_time'])
+ self.assertIsNone(found_job['remaining_time'])
+ else:
+ self.assertIsNotNone(found_job['estimated_time'])
+ self.assertTrue(found_job['estimated_time'] >= 2)
+ self.assertIsNotNone(found_job['remaining_time'])
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ def test_time_database(self):
+ "Test the time database"
+
+ self._test_time_database(1)
+ self._test_time_database(2)
+
def test_two_failed_changes_at_head(self):
"Test that changes are reparented correctly if 2 fail at head"
@@ -600,6 +605,36 @@
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
+ def test_parse_skip_if(self):
+ job_yaml = """
+jobs:
+ - name: job_name
+ skip-if:
+ - project: ^project_name$
+ branch: ^stable/icehouse$
+ all-files-match-any:
+ - ^filename$
+ - project: ^project2_name$
+ all-files-match-any:
+ - ^filename2$
+ """.strip()
+ data = yaml.load(job_yaml)
+ config_job = data.get('jobs')[0]
+ cm = zuul.change_matcher
+ expected = cm.MatchAny([
+ cm.MatchAll([
+ cm.ProjectMatcher('^project_name$'),
+ cm.BranchMatcher('^stable/icehouse$'),
+ cm.MatchAllFiles([cm.FileMatcher('^filename$')]),
+ ]),
+ cm.MatchAll([
+ cm.ProjectMatcher('^project2_name$'),
+ cm.MatchAllFiles([cm.FileMatcher('^filename2$')]),
+ ]),
+ ])
+ matcher = self.sched._parseSkipIf(config_job)
+ self.assertEqual(expected, matcher)
+
def test_patch_order(self):
"Test that dependent patches are tested in the right order"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')