Gitiles
Code Review Sign In
gerrit.cesnet.cz / github / openstack-infra / zuul / 589dfcaf3f1f577df6a70c284a2ee13983100476 / . / tests / unit / test_scheduler.py
blob: e1d00f6dcdfd50833a2dc131b968a42c18060da6 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import gc
import json
import textwrap
import os
import re
import shutil
import time
from unittest import skip
import git
from six.moves import urllib
import testtools
import zuul.change_matcher
import zuul.scheduler
import zuul.rpcclient
import zuul.model
from tests.base import (
ZuulTestCase,
repack_repo,
)
class TestScheduler(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
def test_jobs_executed(self):
"Test that jobs are executed and a change is merged"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(self.getJobFromHistory('project-test1').node,
'image1')
self.assertIsNone(self.getJobFromHistory('project-test2').node)
# TODOv3(jeblair): we may want to report stats by tenant (also?).
self.assertReportedStat('gerrit.event.comment-added', value='1|c')
self.assertReportedStat('zuul.pipeline.gate.current_changes',
value='1|g')
self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
kind='ms')
self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
value='1|c')
self.assertReportedStat('zuul.pipeline.gate.resident_time', kind='ms')
self.assertReportedStat('zuul.pipeline.gate.total_changes',
value='1|c')
self.assertReportedStat(
'zuul.pipeline.gate.org.project.resident_time', kind='ms')
self.assertReportedStat(
'zuul.pipeline.gate.org.project.total_changes', value='1|c')
for build in self.builds:
self.assertEqual(build.parameters['ZUUL_VOTING'], '1')
def test_initial_pipeline_gauges(self):
"Test that each pipeline reported its length on start"
self.assertReportedStat('zuul.pipeline.gate.current_changes',
value='0|g')
self.assertReportedStat('zuul.pipeline.check.current_changes',
value='0|g')
def test_job_branch(self):
"Test the correct variant of a job runs on a branch"
self.create_branch('org/project', 'stable')
A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2,
"A should report start and success")
self.assertIn('gate', A.messages[1],
"A should transit gate")
self.assertEqual(self.getJobFromHistory('project-test1').node,
'image2')
def test_parallel_changes(self):
"Test that changes are tested in parallel and merged in series"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 1)
self.assertEqual(self.builds[0].name, 'project-merge')
self.assertTrue(self.builds[0].hasChanges(A))
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertTrue(self.builds[0].hasChanges(A))
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertTrue(self.builds[1].hasChanges(A))
self.assertEqual(self.builds[2].name, 'project-merge')
self.assertTrue(self.builds[2].hasChanges(A, B))
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 5)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertTrue(self.builds[0].hasChanges(A))
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertTrue(self.builds[1].hasChanges(A))
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertTrue(self.builds[2].hasChanges(A, B))
self.assertEqual(self.builds[3].name, 'project-test2')
self.assertTrue(self.builds[3].hasChanges(A, B))
self.assertEqual(self.builds[4].name, 'project-merge')
self.assertTrue(self.builds[4].hasChanges(A, B, C))
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 6)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertTrue(self.builds[0].hasChanges(A))
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertTrue(self.builds[1].hasChanges(A))
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertTrue(self.builds[2].hasChanges(A, B))
self.assertEqual(self.builds[3].name, 'project-test2')
self.assertTrue(self.builds[3].hasChanges(A, B))
self.assertEqual(self.builds[4].name, 'project-test1')
self.assertTrue(self.builds[4].hasChanges(A, B, C))
self.assertEqual(self.builds[5].name, 'project-test2')
self.assertTrue(self.builds[5].hasChanges(A, B, C))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 9)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
def test_failed_changes(self):
"Test that a change behind a failed change is retested"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertBuilds([dict(name='project-merge', changes='1,1')])
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# A/project-merge is complete
self.assertBuilds([
dict(name='project-test1', changes='1,1'),
dict(name='project-test2', changes='1,1'),
dict(name='project-merge', changes='1,1 2,1'),
])
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# A/project-merge is complete
# B/project-merge is complete
self.assertBuilds([
dict(name='project-test1', changes='1,1'),
dict(name='project-test2', changes='1,1'),
dict(name='project-test1', changes='1,1 2,1'),
dict(name='project-test2', changes='1,1 2,1'),
])
# Release project-test1 for A which will fail. This will
# abort both running B jobs and reexecute project-merge for B.
self.builds[0].release()
self.waitUntilSettled()
self.orderedRelease()
self.assertHistory([
dict(name='project-merge', result='SUCCESS', changes='1,1'),
dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'),
dict(name='project-test1', result='FAILURE', changes='1,1'),
dict(name='project-test1', result='ABORTED', changes='1,1 2,1'),
dict(name='project-test2', result='ABORTED', changes='1,1 2,1'),
dict(name='project-test2', result='SUCCESS', changes='1,1'),
dict(name='project-merge', result='SUCCESS', changes='2,1'),
dict(name='project-test1', result='SUCCESS', changes='2,1'),
dict(name='project-test2', result='SUCCESS', changes='2,1'),
], ordered=False)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
def test_independent_queues(self):
"Test that changes end up in the right queues"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
# There should be one merge job at the head of each queue running
self.assertBuilds([
dict(name='project-merge', changes='1,1'),
dict(name='project-merge', changes='2,1'),
])
# Release the current merge builds
self.builds[0].release()
self.waitUntilSettled()
self.builds[0].release()
self.waitUntilSettled()
# Release the merge job for project2 which is behind project1
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# All the test builds should be running:
self.assertBuilds([
dict(name='project-test1', changes='1,1'),
dict(name='project-test2', changes='1,1'),
dict(name='project-test1', changes='2,1'),
dict(name='project-test2', changes='2,1'),
dict(name='project1-project2-integration', changes='2,1'),
dict(name='project-test1', changes='2,1 3,1'),
dict(name='project-test2', changes='2,1 3,1'),
dict(name='project1-project2-integration', changes='2,1 3,1'),
])
self.orderedRelease()
self.assertHistory([
dict(name='project-merge', result='SUCCESS', changes='1,1'),
dict(name='project-merge', result='SUCCESS', changes='2,1'),
dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'),
dict(name='project-test1', result='SUCCESS', changes='1,1'),
dict(name='project-test2', result='SUCCESS', changes='1,1'),
dict(name='project-test1', result='SUCCESS', changes='2,1'),
dict(name='project-test2', result='SUCCESS', changes='2,1'),
dict(
name='project1-project2-integration',
result='SUCCESS',
changes='2,1'),
dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'),
dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'),
dict(name='project1-project2-integration',
result='SUCCESS',
changes='2,1 3,1'),
])
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
def test_failed_change_at_head(self):
"Test that if a change at the head fails, jobs behind it are canceled"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertBuilds([
dict(name='project-merge', changes='1,1'),
])
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertBuilds([
dict(name='project-test1', changes='1,1'),
dict(name='project-test2', changes='1,1'),
dict(name='project-test1', changes='1,1 2,1'),
dict(name='project-test2', changes='1,1 2,1'),
dict(name='project-test1', changes='1,1 2,1 3,1'),
dict(name='project-test2', changes='1,1 2,1 3,1'),
])
self.release(self.builds[0])
self.waitUntilSettled()
# project-test2, project-merge for B
self.assertBuilds([
dict(name='project-test2', changes='1,1'),
dict(name='project-merge', changes='2,1'),
])
# Unordered history comparison because the aborts can finish
# in any order.
self.assertHistory([
dict(name='project-merge', result='SUCCESS',
changes='1,1'),
dict(name='project-merge', result='SUCCESS',
changes='1,1 2,1'),
dict(name='project-merge', result='SUCCESS',
changes='1,1 2,1 3,1'),
dict(name='project-test1', result='FAILURE',
changes='1,1'),
dict(name='project-test1', result='ABORTED',
changes='1,1 2,1'),
dict(name='project-test2', result='ABORTED',
changes='1,1 2,1'),
dict(name='project-test1', result='ABORTED',
changes='1,1 2,1 3,1'),
dict(name='project-test2', result='ABORTED',
changes='1,1 2,1 3,1'),
], ordered=False)
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.orderedRelease()
self.assertBuilds([])
self.assertHistory([
dict(name='project-merge', result='SUCCESS',
changes='1,1'),
dict(name='project-merge', result='SUCCESS',
changes='1,1 2,1'),
dict(name='project-merge', result='SUCCESS',
changes='1,1 2,1 3,1'),
dict(name='project-test1', result='FAILURE',
changes='1,1'),
dict(name='project-test1', result='ABORTED',
changes='1,1 2,1'),
dict(name='project-test2', result='ABORTED',
changes='1,1 2,1'),
dict(name='project-test1', result='ABORTED',
changes='1,1 2,1 3,1'),
dict(name='project-test2', result='ABORTED',
changes='1,1 2,1 3,1'),
dict(name='project-merge', result='SUCCESS',
changes='2,1'),
dict(name='project-merge', result='SUCCESS',
changes='2,1 3,1'),
dict(name='project-test2', result='SUCCESS',
changes='1,1'),
dict(name='project-test1', result='SUCCESS',
changes='2,1'),
dict(name='project-test2', result='SUCCESS',
changes='2,1'),
dict(name='project-test1', result='SUCCESS',
changes='2,1 3,1'),
dict(name='project-test2', result='SUCCESS',
changes='2,1 3,1'),
], ordered=False)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
def test_failed_change_in_middle(self):
"Test a failed change in the middle of the queue"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', B)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 6)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertEqual(self.builds[3].name, 'project-test2')
self.assertEqual(self.builds[4].name, 'project-test1')
self.assertEqual(self.builds[5].name, 'project-test2')
self.release(self.builds[2])
self.waitUntilSettled()
# project-test1 and project-test2 for A
# project-test2 for B
# project-merge for C (without B)
self.assertEqual(len(self.builds), 4)
self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2)
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# project-test1 and project-test2 for A
# project-test2 for B
# project-test1 and project-test2 for C
self.assertEqual(len(self.builds), 5)
tenant = self.sched.abide.tenants.get('tenant-one')
items = tenant.layout.pipelines['gate'].getAllItems()
builds = items[0].current_build_set.getBuilds()
self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
self.assertEqual(self.countJobResults(builds, None), 2)
builds = items[1].current_build_set.getBuilds()
self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
self.assertEqual(self.countJobResults(builds, 'FAILURE'), 1)
self.assertEqual(self.countJobResults(builds, None), 1)
builds = items[2].current_build_set.getBuilds()
self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1)
self.assertEqual(self.countJobResults(builds, None), 2)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 12)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
def test_failed_change_at_head_with_queue(self):
"Test that if a change at the head fails, queued jobs are canceled"
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
queue = self.gearman_server.getQueue()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(queue), 1)
self.assertEqual(queue[0].name, 'executor:execute')
job_args = json.loads(queue[0].arguments)
self.assertEqual(job_args['job'], 'project-merge')
self.assertEqual(job_args['items'][0]['number'], '%d' % A.number)
self.gearman_server.release('.*-merge')
self.waitUntilSettled()
self.gearman_server.release('.*-merge')
self.waitUntilSettled()
self.gearman_server.release('.*-merge')
self.waitUntilSettled()
queue = self.gearman_server.getQueue()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(queue), 6)
self.assertEqual(
json.loads(queue[0].arguments)['job'], 'project-test1')
self.assertEqual(
json.loads(queue[1].arguments)['job'], 'project-test2')
self.assertEqual(
json.loads(queue[2].arguments)['job'], 'project-test1')
self.assertEqual(
json.loads(queue[3].arguments)['job'], 'project-test2')
self.assertEqual(
json.loads(queue[4].arguments)['job'], 'project-test1')
self.assertEqual(
json.loads(queue[5].arguments)['job'], 'project-test2')
self.release(queue[0])
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
queue = self.gearman_server.getQueue()
self.assertEqual(len(queue), 2) # project-test2, project-merge for B
self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 0)
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 11)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
@skip("Disabled for early v3 development")
def _test_time_database(self, iteration):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
time.sleep(2)
data = json.loads(self.sched.formatStatusJSON())
found_job = None
for pipeline in data['pipelines']:
if pipeline['name'] != 'gate':
continue
for queue in pipeline['change_queues']:
for head in queue['heads']:
for item in head:
for job in item['jobs']:
if job['name'] == 'project-merge':
found_job = job
break
self.assertIsNotNone(found_job)
if iteration == 1:
self.assertIsNotNone(found_job['estimated_time'])
self.assertIsNone(found_job['remaining_time'])
else:
self.assertIsNotNone(found_job['estimated_time'])
self.assertTrue(found_job['estimated_time'] >= 2)
self.assertIsNotNone(found_job['remaining_time'])
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
@skip("Disabled for early v3 development")
def test_time_database(self):
"Test the time database"
self._test_time_database(1)
self._test_time_database(2)
def test_two_failed_changes_at_head(self):
"Test that changes are reparented correctly if 2 fail at head"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', A)
self.executor_server.failJob('project-test1', B)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 6)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertEqual(self.builds[3].name, 'project-test2')
self.assertEqual(self.builds[4].name, 'project-test1')
self.assertEqual(self.builds[5].name, 'project-test2')
self.assertTrue(self.builds[0].hasChanges(A))
self.assertTrue(self.builds[2].hasChanges(A))
self.assertTrue(self.builds[2].hasChanges(B))
self.assertTrue(self.builds[4].hasChanges(A))
self.assertTrue(self.builds[4].hasChanges(B))
self.assertTrue(self.builds[4].hasChanges(C))
# Fail change B first
self.release(self.builds[2])
self.waitUntilSettled()
# restart of C after B failure
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 5)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project-test2')
self.assertEqual(self.builds[3].name, 'project-test1')
self.assertEqual(self.builds[4].name, 'project-test2')
self.assertTrue(self.builds[1].hasChanges(A))
self.assertTrue(self.builds[2].hasChanges(A))
self.assertTrue(self.builds[2].hasChanges(B))
self.assertTrue(self.builds[4].hasChanges(A))
self.assertFalse(self.builds[4].hasChanges(B))
self.assertTrue(self.builds[4].hasChanges(C))
# Finish running all passing jobs for change A
self.release(self.builds[1])
self.waitUntilSettled()
# Fail and report change A
self.release(self.builds[0])
self.waitUntilSettled()
# restart of B,C after A failure
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 4)
self.assertEqual(self.builds[0].name, 'project-test1') # B
self.assertEqual(self.builds[1].name, 'project-test2') # B
self.assertEqual(self.builds[2].name, 'project-test1') # C
self.assertEqual(self.builds[3].name, 'project-test2') # C
self.assertFalse(self.builds[1].hasChanges(A))
self.assertTrue(self.builds[1].hasChanges(B))
self.assertFalse(self.builds[1].hasChanges(C))
self.assertFalse(self.builds[2].hasChanges(A))
# After A failed and B and C restarted, B should be back in
# C's tests because it has not failed yet.
self.assertTrue(self.builds[2].hasChanges(B))
self.assertTrue(self.builds[2].hasChanges(C))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 21)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
def test_patch_order(self):
"Test that dependent patches are tested in the right order"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
M2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M2')
M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
M2.setMerged()
M1.setMerged()
# C -> B -> A -> M1 -> M2
# M2 is here to make sure it is never queried. If it is, it
# means zuul is walking down the entire history of merged
# changes.
C.setDependsOn(B, 1)
B.setDependsOn(A, 1)
A.setDependsOn(M1, 1)
M1.setDependsOn(M2, 1)
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'NEW')
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(M2.queried, 0)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
def test_needed_changes_enqueue(self):
"Test that a needed change is enqueued ahead"
# A Given a git tree like this, if we enqueue
# / \ change C, we should walk up and down the tree
# B G and enqueue changes in the order ABCDEFG.
# /|\ This is also the order that you would get if
# *C E F you enqueued changes in the order ABCDEFG, so
# / the ordering is stable across re-enqueue events.
# D
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
B.setDependsOn(A, 1)
C.setDependsOn(B, 1)
D.setDependsOn(C, 1)
E.setDependsOn(B, 1)
F.setDependsOn(B, 1)
G.setDependsOn(A, 1)
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
D.addApproval('code-review', 2)
E.addApproval('code-review', 2)
F.addApproval('code-review', 2)
G.addApproval('code-review', 2)
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(D.data['status'], 'NEW')
self.assertEqual(E.data['status'], 'NEW')
self.assertEqual(F.data['status'], 'NEW')
self.assertEqual(G.data['status'], 'NEW')
# We're about to add approvals to changes without adding the
# triggering events to Zuul, so that we can be sure that it is
# enqueing the changes based on dependencies, not because of
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
for connection in self.connections.connections.values():
connection.maintainCache([])
self.executor_server.hold_jobs_in_build = True
A.addApproval('approved', 1)
B.addApproval('approved', 1)
D.addApproval('approved', 1)
E.addApproval('approved', 1)
F.addApproval('approved', 1)
G.addApproval('approved', 1)
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
for x in range(8):
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(D.data['status'], 'MERGED')
self.assertEqual(E.data['status'], 'MERGED')
self.assertEqual(F.data['status'], 'MERGED')
self.assertEqual(G.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertEqual(D.reported, 2)
self.assertEqual(E.reported, 2)
self.assertEqual(F.reported, 2)
self.assertEqual(G.reported, 2)
self.assertEqual(self.history[6].changes,
'1,1 2,1 3,1 4,1 5,1 6,1 7,1')
def test_source_cache(self):
"Test that the source cache operates correctly"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
X = self.fake_gerrit.addFakeChange('org/project', 'master', 'X')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
M1.setMerged()
B.setDependsOn(A, 1)
A.setDependsOn(M1, 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(X.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
for build in self.builds:
if build.parameters['ZUUL_PIPELINE'] == 'check':
build.release()
self.waitUntilSettled()
for build in self.builds:
if build.parameters['ZUUL_PIPELINE'] == 'check':
build.release()
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.log.debug("len %s" % self.fake_gerrit._change_cache.keys())
# there should still be changes in the cache
self.assertNotEqual(len(self.fake_gerrit._change_cache.keys()), 0)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(A.queried, 2) # Initial and isMerged
self.assertEqual(B.queried, 3) # Initial A, refresh from B, isMerged
def test_can_merge(self):
"Test whether a change is ready to merge"
# TODO: move to test_gerrit (this is a unit test!)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
tenant = self.sched.abide.tenants.get('tenant-one')
source = tenant.layout.pipelines['gate'].source
# TODO(pabelanger): As we add more source / trigger APIs we should make
# it easier for users to create events for testing.
event = zuul.model.TriggerEvent()
event.trigger_name = 'gerrit'
event.change_number = '1'
event.patch_number = '2'
a = source.getChange(event)
mgr = tenant.layout.pipelines['gate'].manager
self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
A.addApproval('code-review', 2)
a = source.getChange(event, refresh=True)
self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds()))
A.addApproval('approved', 1)
a = source.getChange(event, refresh=True)
self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
def test_project_merge_conflict(self):
"Test that gate merge conflicts are handled properly"
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project',
'master', 'A',
files={'conflict': 'foo'})
B = self.fake_gerrit.addFakeChange('org/project',
'master', 'B',
files={'conflict': 'bar'})
C = self.fake_gerrit.addFakeChange('org/project',
'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
self.assertEqual(C.reported, 1)
self.gearman_server.release('project-merge')
self.waitUntilSettled()
self.gearman_server.release('project-merge')
self.waitUntilSettled()
self.gearman_server.release('project-merge')
self.waitUntilSettled()
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertHistory([
dict(name='project-merge', result='SUCCESS', changes='1,1'),
dict(name='project-test1', result='SUCCESS', changes='1,1'),
dict(name='project-test2', result='SUCCESS', changes='1,1'),
dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'),
dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'),
dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'),
], ordered=False)
def test_delayed_merge_conflict(self):
"Test that delayed check merge conflicts are handled properly"
# Hold jobs in the gearman queue so that we can test whether
# the executor returns a merge failure after the scheduler has
# successfully merged.
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project',
'master', 'A',
files={'conflict': 'foo'})
B = self.fake_gerrit.addFakeChange('org/project',
'master', 'B',
files={'conflict': 'bar'})
C = self.fake_gerrit.addFakeChange('org/project',
'master', 'C')
C.setDependsOn(B, 1)
# A enters the gate queue; B and C enter the check queue
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 0) # Check does not report start
self.assertEqual(C.reported, 0) # Check does not report start
# A merges while B and C are queued in check
# Release A project-merge
queue = self.gearman_server.getQueue()
self.release(queue[0])
self.waitUntilSettled()
# Release A project-test*
# gate has higher precedence, so A's test jobs are added in
# front of the merge jobs for B and C
queue = self.gearman_server.getQueue()
self.release(queue[0])
self.release(queue[1])
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 0)
self.assertEqual(C.reported, 0)
self.assertHistory([
dict(name='project-merge', result='SUCCESS', changes='1,1'),
dict(name='project-test1', result='SUCCESS', changes='1,1'),
dict(name='project-test2', result='SUCCESS', changes='1,1'),
], ordered=False)
# B and C report merge conflicts
# Release B project-merge
queue = self.gearman_server.getQueue()
self.release(queue[0])
self.waitUntilSettled()
# Release C
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 1)
self.assertEqual(C.reported, 1)
self.assertHistory([
dict(name='project-merge', result='SUCCESS', changes='1,1'),
dict(name='project-test1', result='SUCCESS', changes='1,1'),
dict(name='project-test2', result='SUCCESS', changes='1,1'),
dict(name='project-merge', result='MERGER_FAILURE', changes='2,1'),
dict(name='project-merge', result='MERGER_FAILURE',
changes='2,1 3,1'),
], ordered=False)
def test_post(self):
"Test that post jobs run"
e = {
"type": "ref-updated",
"submitter": {
"name": "User Name",
},
"refUpdate": {
"oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
"newRev": "d479a0bfcb34da57a31adb2a595c0cf687812543",
"refName": "master",
"project": "org/project",
}
}
self.fake_gerrit.addEvent(e)
self.waitUntilSettled()
job_names = [x.name for x in self.history]
self.assertEqual(len(self.history), 1)
self.assertIn('project-post', job_names)
def test_post_ignore_deletes(self):
"Test that deleting refs does not trigger post jobs"
e = {
"type": "ref-updated",
"submitter": {
"name": "User Name",
},
"refUpdate": {
"oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
"newRev": "0000000000000000000000000000000000000000",
"refName": "master",
"project": "org/project",
}
}
self.fake_gerrit.addEvent(e)
self.waitUntilSettled()
job_names = [x.name for x in self.history]
self.assertEqual(len(self.history), 0)
self.assertNotIn('project-post', job_names)
def test_post_ignore_deletes_negative(self):
"Test that deleting refs does trigger post jobs"
self.updateConfigLayout('layout-dont-ignore-ref-deletes')
self.sched.reconfigure(self.config)
e = {
"type": "ref-updated",
"submitter": {
"name": "User Name",
},
"refUpdate": {
"oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
"newRev": "0000000000000000000000000000000000000000",
"refName": "master",
"project": "org/project",
}
}
self.fake_gerrit.addEvent(e)
self.waitUntilSettled()
job_names = [x.name for x in self.history]
self.assertEqual(len(self.history), 1)
self.assertIn('project-post', job_names)
@skip("Disabled for early v3 development")
def test_build_configuration_branch_interaction(self):
"Test that switching between branches works"
self.test_build_configuration()
self.test_build_configuration_branch()
# C has been merged, undo that
path = os.path.join(self.upstream_root, "org/project")
repo = git.Repo(path)
repo.heads.master.commit = repo.commit('init')
self.test_build_configuration()
def test_dependent_changes_dequeue(self):
"Test that dependent patches are not needlessly tested"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1')
M1.setMerged()
# C -> B -> A -> M1
C.setDependsOn(B, 1)
B.setDependsOn(A, 1)
A.setDependsOn(M1, 1)
self.executor_server.failJob('project-merge', A)
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 2)
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(C.reported, 2)
self.assertEqual(len(self.history), 1)
def test_failing_dependent_changes(self):
"Test that failing dependent patches are taken out of stream"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
D.addApproval('code-review', 2)
E.addApproval('code-review', 2)
# E, D -> C -> B, A
D.setDependsOn(C, 1)
C.setDependsOn(B, 1)
self.executor_server.failJob('project-test1', B)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(D.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(E.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
for build in self.builds:
if build.parameters['ZUUL_CHANGE'] != '1':
build.release()
self.waitUntilSettled()
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertIn('Build succeeded', A.messages[1])
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 2)
self.assertIn('Build failed', B.messages[1])
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(C.reported, 2)
self.assertIn('depends on a change', C.messages[1])
self.assertEqual(D.data['status'], 'NEW')
self.assertEqual(D.reported, 2)
self.assertIn('depends on a change', D.messages[1])
self.assertEqual(E.data['status'], 'MERGED')
self.assertEqual(E.reported, 2)
self.assertIn('Build succeeded', E.messages[1])
self.assertEqual(len(self.history), 18)
def test_head_is_dequeued_once(self):
"Test that if a change at the head fails it is dequeued only once"
# If it's dequeued more than once, we should see extra
# aborted jobs.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', A)
self.executor_server.failJob('project-test2', A)
self.executor_server.failJob('project1-project2-integration', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 1)
self.assertEqual(self.builds[0].name, 'project-merge')
self.assertTrue(self.builds[0].hasChanges(A))
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 9)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project1-project2-integration')
self.assertEqual(self.builds[3].name, 'project-test1')
self.assertEqual(self.builds[4].name, 'project-test2')
self.assertEqual(self.builds[5].name, 'project1-project2-integration')
self.assertEqual(self.builds[6].name, 'project-test1')
self.assertEqual(self.builds[7].name, 'project-test2')
self.assertEqual(self.builds[8].name, 'project1-project2-integration')
self.release(self.builds[0])
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3) # test2,integration, merge for B
self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 6)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 20)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
def test_nonvoting_job(self):
"Test that non-voting jobs don't vote."
A = self.fake_gerrit.addFakeChange('org/nonvoting-project',
'master', 'A')
A.addApproval('code-review', 2)
self.executor_server.failJob('nonvoting-project-test2', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(
self.getJobFromHistory('nonvoting-project-merge').result,
'SUCCESS')
self.assertEqual(
self.getJobFromHistory('nonvoting-project-test1').result,
'SUCCESS')
self.assertEqual(
self.getJobFromHistory('nonvoting-project-test2').result,
'FAILURE')
for build in self.builds:
self.assertEqual(build.parameters['ZUUL_VOTING'], '0')
def test_check_queue_success(self):
"Test successful check queue jobs."
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
def test_check_queue_failure(self):
"Test failed check queue jobs."
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.executor_server.failJob('project-test2', A)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'FAILURE')
def test_dependent_behind_dequeue(self):
# This particular test does a large amount of merges and needs a little
# more time to complete
self.wait_timeout = 120
"test that dependent changes behind dequeued changes work"
# This complicated test is a reproduction of a real life bug
self.sched.reconfigure(self.config)
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
F = self.fake_gerrit.addFakeChange('org/project3', 'master', 'F')
D.setDependsOn(C, 1)
E.setDependsOn(D, 1)
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
D.addApproval('code-review', 2)
E.addApproval('code-review', 2)
F.addApproval('code-review', 2)
A.fail_merge = True
# Change object re-use in the gerrit trigger is hidden if
# changes are added in quick succession; waiting makes it more
# like real life.
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(D.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(E.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(F.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# all jobs running
# Grab pointers to the jobs we want to release before
# releasing any, because list indexes may change as
# the jobs complete.
a, b, c = self.builds[:3]
a.release()
b.release()
c.release()
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(D.data['status'], 'MERGED')
self.assertEqual(E.data['status'], 'MERGED')
self.assertEqual(F.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
self.assertEqual(D.reported, 2)
self.assertEqual(E.reported, 2)
self.assertEqual(F.reported, 2)
self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 15)
self.assertEqual(len(self.history), 44)
def test_merger_repack(self):
"Test that the merger works after a repack"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEmptyQueues()
self.build_history = []
path = os.path.join(self.merger_src_root, "org/project")
if os.path.exists(path):
repack_repo(path)
path = os.path.join(self.executor_src_root, "org/project")
if os.path.exists(path):
repack_repo(path)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
def test_merger_repack_large_change(self):
"Test that the merger works with large changes after a repack"
# https://bugs.executepad.net/zuul/+bug/1078946
# This test assumes the repo is already cloned; make sure it is
tenant = self.sched.abide.tenants.get('tenant-one')
trusted, project = tenant.getProject('org/project1')
url = self.fake_gerrit.getGitUrl(project)
self.merge_server.merger.addProject('org/project1', url)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
A.addPatchset(large=True)
path = os.path.join(self.upstream_root, "org/project1")
repack_repo(path)
path = os.path.join(self.merger_src_root, "org/project1")
if os.path.exists(path):
repack_repo(path)
path = os.path.join(self.executor_src_root, "org/project1")
if os.path.exists(path):
repack_repo(path)
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
def test_new_patchset_dequeues_old(self):
"Test that a new patchset causes the old to be dequeued"
# D -> C (depends on B) -> B (depends on A) -> A -> M
self.executor_server.hold_jobs_in_build = True
M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
M.setMerged()
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
D.addApproval('code-review', 2)
C.setDependsOn(B, 1)
B.setDependsOn(A, 1)
A.setDependsOn(M, 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.fake_gerrit.addEvent(D.addApproval('approved', 1))
self.waitUntilSettled()
B.addPatchset()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 2)
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(C.reported, 2)
self.assertEqual(D.data['status'], 'MERGED')
self.assertEqual(D.reported, 2)
self.assertEqual(len(self.history), 9) # 3 each for A, B, D.
def test_new_patchset_check(self):
"Test a new patchset in check"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
tenant = self.sched.abide.tenants.get('tenant-one')
check_pipeline = tenant.layout.pipelines['check']
# Add two git-dependent changes
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# A live item, and a non-live/live pair
items = check_pipeline.getAllItems()
self.assertEqual(len(items), 3)
self.assertEqual(items[0].change.number, '1')
self.assertEqual(items[0].change.patchset, '1')
self.assertFalse(items[0].live)
self.assertEqual(items[1].change.number, '2')
self.assertEqual(items[1].change.patchset, '1')
self.assertTrue(items[1].live)
self.assertEqual(items[2].change.number, '1')
self.assertEqual(items[2].change.patchset, '1')
self.assertTrue(items[2].live)
# Add a new patchset to A
A.addPatchset()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
# The live copy of A,1 should be gone, but the non-live and B
# should continue, and we should have a new A,2
items = check_pipeline.getAllItems()
self.assertEqual(len(items), 3)
self.assertEqual(items[0].change.number, '1')
self.assertEqual(items[0].change.patchset, '1')
self.assertFalse(items[0].live)
self.assertEqual(items[1].change.number, '2')
self.assertEqual(items[1].change.patchset, '1')
self.assertTrue(items[1].live)
self.assertEqual(items[2].change.number, '1')
self.assertEqual(items[2].change.patchset, '2')
self.assertTrue(items[2].live)
# Add a new patchset to B
B.addPatchset()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
# The live copy of B,1 should be gone, and it's non-live copy of A,1
# but we should have a new B,2 (still based on A,1)
items = check_pipeline.getAllItems()
self.assertEqual(len(items), 3)
self.assertEqual(items[0].change.number, '1')
self.assertEqual(items[0].change.patchset, '2')
self.assertTrue(items[0].live)
self.assertEqual(items[1].change.number, '1')
self.assertEqual(items[1].change.patchset, '1')
self.assertFalse(items[1].live)
self.assertEqual(items[2].change.number, '2')
self.assertEqual(items[2].change.patchset, '2')
self.assertTrue(items[2].live)
self.builds[0].release()
self.waitUntilSettled()
self.builds[0].release()
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
self.assertEqual(self.history[0].result, 'ABORTED')
self.assertEqual(self.history[0].changes, '1,1')
self.assertEqual(self.history[1].result, 'ABORTED')
self.assertEqual(self.history[1].changes, '1,1 2,1')
self.assertEqual(self.history[2].result, 'SUCCESS')
self.assertEqual(self.history[2].changes, '1,2')
self.assertEqual(self.history[3].result, 'SUCCESS')
self.assertEqual(self.history[3].changes, '1,1 2,2')
def test_abandoned_gate(self):
"Test that an abandoned change is dequeued from gate"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 1, "One job being built (on hold)")
self.assertEqual(self.builds[0].name, 'project-merge')
self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertBuilds([])
self.assertHistory([
dict(name='project-merge', result='ABORTED', changes='1,1')],
ordered=False)
self.assertEqual(A.reported, 1,
"Abandoned gate change should report only start")
def test_abandoned_check(self):
"Test that an abandoned change is dequeued from check"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
tenant = self.sched.abide.tenants.get('tenant-one')
check_pipeline = tenant.layout.pipelines['check']
# Add two git-dependent changes
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# A live item, and a non-live/live pair
items = check_pipeline.getAllItems()
self.assertEqual(len(items), 3)
self.assertEqual(items[0].change.number, '1')
self.assertFalse(items[0].live)
self.assertEqual(items[1].change.number, '2')
self.assertTrue(items[1].live)
self.assertEqual(items[2].change.number, '1')
self.assertTrue(items[2].live)
# Abandon A
self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
self.waitUntilSettled()
# The live copy of A should be gone, but the non-live and B
# should continue
items = check_pipeline.getAllItems()
self.assertEqual(len(items), 2)
self.assertEqual(items[0].change.number, '1')
self.assertFalse(items[0].live)
self.assertEqual(items[1].change.number, '2')
self.assertTrue(items[1].live)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.history), 4)
self.assertEqual(self.history[0].result, 'ABORTED',
'Build should have been aborted')
self.assertEqual(A.reported, 0, "Abandoned change should not report")
self.assertEqual(B.reported, 1, "Change should report")
def test_abandoned_not_timer(self):
"Test that an abandoned change does not cancel timer jobs"
self.executor_server.hold_jobs_in_build = True
# Start timer trigger - also org/project
self.updateConfigLayout('layout-idle')
self.sched.reconfigure(self.config)
# The pipeline triggers every second, so we should have seen
# several by now.
time.sleep(5)
self.waitUntilSettled()
# Stop queuing timer triggered jobs so that the assertions
# below don't race against more jobs being queued.
# Must be in same repo, so overwrite config with another one
self.commitLayoutUpdate('layout-idle', 'layout-no-timer')
self.sched.reconfigure(self.config)
self.assertEqual(len(self.builds), 2, "Two timer jobs")
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3, "One change plus two timer jobs")
self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2, "Two timer jobs remain")
self.executor_server.release()
self.waitUntilSettled()
def test_zuul_url_return(self):
"Test if ZUUL_URL is returning when zuul_url is set in zuul.conf"
self.assertTrue(self.sched.config.has_option('merger', 'zuul_url'))
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 1)
for build in self.builds:
self.assertTrue('ZUUL_URL' in build.parameters)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
def test_new_patchset_dequeues_old_on_head(self):
"Test that a new patchset causes the old to be dequeued (at head)"
# D -> C (depends on B) -> B (depends on A) -> A -> M
self.executor_server.hold_jobs_in_build = True
M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M')
M.setMerged()
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
D.addApproval('code-review', 2)
C.setDependsOn(B, 1)
B.setDependsOn(A, 1)
A.setDependsOn(M, 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.fake_gerrit.addEvent(D.addApproval('approved', 1))
self.waitUntilSettled()
A.addPatchset()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 2)
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(C.reported, 2)
self.assertEqual(D.data['status'], 'MERGED')
self.assertEqual(D.reported, 2)
self.assertEqual(len(self.history), 7)
def test_new_patchset_dequeues_old_without_dependents(self):
"Test that a new patchset causes only the old to be dequeued"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
B.addPatchset()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 2)
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(C.reported, 2)
self.assertEqual(len(self.history), 9)
def test_new_patchset_dequeues_old_independent_queue(self):
"Test that a new patchset causes the old to be dequeued (independent)"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
B.addPatchset()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 1)
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(C.reported, 1)
self.assertEqual(len(self.history), 10)
self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1)
def test_noop_job(self):
"Test that the internal noop job works"
A = self.fake_gerrit.addFakeChange('org/noop-project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(len(self.gearman_server.getQueue()), 0)
self.assertTrue(self.sched._areAllBuildsComplete())
self.assertEqual(len(self.history), 0)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
def test_no_job_project(self):
"Test that reports with no jobs don't get sent"
A = self.fake_gerrit.addFakeChange('org/no-jobs-project',
'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Change wasn't reported to
self.assertEqual(A.reported, False)
# Check queue is empty afterwards
tenant = self.sched.abide.tenants.get('tenant-one')
check_pipeline = tenant.layout.pipelines['check']
items = check_pipeline.getAllItems()
self.assertEqual(len(items), 0)
self.assertEqual(len(self.history), 0)
def test_zuul_refs(self):
"Test that zuul refs exist and have the right changes"
self.executor_server.hold_jobs_in_build = True
M1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'M1')
M1.setMerged()
M2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'M2')
M2.setMerged()
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
D.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.fake_gerrit.addEvent(D.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
a_zref = b_zref = c_zref = d_zref = None
a_build = b_build = c_build = d_build = None
for x in self.builds:
if x.parameters['ZUUL_CHANGE'] == '3':
a_zref = x.parameters['ZUUL_REF']
a_build = x
elif x.parameters['ZUUL_CHANGE'] == '4':
b_zref = x.parameters['ZUUL_REF']
b_build = x
elif x.parameters['ZUUL_CHANGE'] == '5':
c_zref = x.parameters['ZUUL_REF']
c_build = x
elif x.parameters['ZUUL_CHANGE'] == '6':
d_zref = x.parameters['ZUUL_REF']
d_build = x
if a_build and b_build and c_build and d_build:
break
# There are... four... refs.
self.assertIsNotNone(a_zref)
self.assertIsNotNone(b_zref)
self.assertIsNotNone(c_zref)
self.assertIsNotNone(d_zref)
# And they should all be different
refs = set([a_zref, b_zref, c_zref, d_zref])
self.assertEqual(len(refs), 4)
# should have a, not b, and should not be in project2
self.assertTrue(a_build.hasChanges(A))
self.assertFalse(a_build.hasChanges(B, M2))
# should have a and b, and should not be in project2
self.assertTrue(b_build.hasChanges(A, B))
self.assertFalse(b_build.hasChanges(M2))
# should have a and b in 1, c in 2
self.assertTrue(c_build.hasChanges(A, B, C))
self.assertFalse(c_build.hasChanges(D))
# should have a and b in 1, c and d in 2
self.assertTrue(d_build.hasChanges(A, B, C, D))
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(C.reported, 2)
self.assertEqual(D.data['status'], 'MERGED')
self.assertEqual(D.reported, 2)
def test_rerun_on_error(self):
"Test that if a worker fails to run a job, it is run again"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.builds[0].requeue = True
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(self.countJobResults(self.history, None), 1)
self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3)
def test_statsd(self):
"Test each of the statsd methods used in the scheduler"
import extras
statsd = extras.try_import('statsd.statsd')
statsd.incr('test-incr')
statsd.timing('test-timing', 3)
statsd.gauge('test-gauge', 12)
self.assertReportedStat('test-incr', '1|c')
self.assertReportedStat('test-timing', '3|ms')
self.assertReportedStat('test-gauge', '12|g')
def test_stuck_job_cleanup(self):
"Test that pending jobs are cleaned up if removed from layout"
# We want to hold the project-merge job that the fake change enqueues
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
# The assertion is that we have one job in the queue, project-merge
self.assertEqual(len(self.gearman_server.getQueue()), 1)
self.commitLayoutUpdate('common-config', 'layout-no-jobs')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.gearman_server.release('gate-noop')
self.waitUntilSettled()
# asserting that project-merge is removed from queue
self.assertEqual(len(self.gearman_server.getQueue()), 0)
self.assertTrue(self.sched._areAllBuildsComplete())
self.assertEqual(len(self.history), 1)
self.assertEqual(self.history[0].name, 'gate-noop')
self.assertEqual(self.history[0].result, 'SUCCESS')
def test_file_head(self):
# This is a regression test for an observed bug. A change
# with a file named "HEAD" in the root directory of the repo
# was processed by a merger. It then was unable to reset the
# repo because of:
# GitCommandError: 'git reset --hard HEAD' returned
# with exit code 128
# stderr: 'fatal: ambiguous argument 'HEAD': both revision
# and filename
# Use '--' to separate filenames from revisions'
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addPatchset({'HEAD': ''})
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertIn('Build succeeded', A.messages[0])
self.assertIn('Build succeeded', B.messages[0])
def test_file_jobs(self):
"Test that file jobs run only when appropriate"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addPatchset({'pip-requires': 'foo'})
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
testfile_jobs = [x for x in self.history
if x.name == 'project-testfile']
self.assertEqual(len(testfile_jobs), 1)
self.assertEqual(testfile_jobs[0].changes, '1,2')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
def _test_irrelevant_files_jobs(self, should_skip):
"Test that jobs with irrelevant-files filter run only when appropriate"
self.updateConfigLayout('layout-irrelevant-files')
self.sched.reconfigure(self.config)
if should_skip:
files = {'ignoreme': 'ignored\n'}
else:
files = {'respectme': 'please!\n'}
change = self.fake_gerrit.addFakeChange('org/project',
'master',
'test irrelevant-files',
files=files)
self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
tested_change_ids = [x.changes[0] for x in self.history
if x.name == 'project-test-irrelevant-files']
if should_skip:
self.assertEqual([], tested_change_ids)
else:
self.assertIn(change.data['number'], tested_change_ids)
def test_irrelevant_files_match_skips_job(self):
self._test_irrelevant_files_jobs(should_skip=True)
def test_irrelevant_files_no_match_runs_job(self):
self._test_irrelevant_files_jobs(should_skip=False)
def test_inherited_jobs_keep_matchers(self):
self.updateConfigLayout('layout-inheritance')
self.sched.reconfigure(self.config)
files = {'ignoreme': 'ignored\n'}
change = self.fake_gerrit.addFakeChange('org/project',
'master',
'test irrelevant-files',
files=files)
self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
run_jobs = set([build.name for build in self.history])
self.assertEqual(set(['project-test-nomatch-starts-empty',
'project-test-nomatch-starts-full']), run_jobs)
@skip("Disabled for early v3 development")
def test_test_config(self):
"Test that we can test the config"
self.sched.testConfig(self.config.get('zuul', 'tenant_config'),
self.connections)
def test_queue_names(self):
"Test shared change queue names"
tenant = self.sched.abide.tenants.get('tenant-one')
source = tenant.layout.pipelines['gate'].source
project1 = source.getProject('org/project1')
project2 = source.getProject('org/project2')
q1 = tenant.layout.pipelines['gate'].getQueue(project1)
q2 = tenant.layout.pipelines['gate'].getQueue(project2)
self.assertEqual(q1.name, 'integrated')
self.assertEqual(q2.name, 'integrated')
def test_queue_precedence(self):
"Test that queue precedence works"
self.gearman_server.hold_jobs_in_queue = True
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
# Run one build at a time to ensure non-race order:
self.orderedRelease()
self.executor_server.hold_jobs_in_build = False
self.waitUntilSettled()
self.log.debug(self.history)
self.assertEqual(self.history[0].pipeline, 'gate')
self.assertEqual(self.history[1].pipeline, 'check')
self.assertEqual(self.history[2].pipeline, 'gate')
self.assertEqual(self.history[3].pipeline, 'gate')
self.assertEqual(self.history[4].pipeline, 'check')
self.assertEqual(self.history[5].pipeline, 'check')
def test_json_status(self):
"Test that we can retrieve JSON status info"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('project-merge')
self.waitUntilSettled()
port = self.webapp.server.socket.getsockname()[1]
req = urllib.request.Request(
"http://localhost:%s/tenant-one/status" % port)
f = urllib.request.urlopen(req)
headers = f.info()
self.assertIn('Content-Length', headers)
self.assertIn('Content-Type', headers)
self.assertIsNotNone(re.match('^application/json(; charset=UTF-8)?$',
headers['Content-Type']))
self.assertIn('Access-Control-Allow-Origin', headers)
self.assertIn('Cache-Control', headers)
self.assertIn('Last-Modified', headers)
self.assertIn('Expires', headers)
data = f.read()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
data = json.loads(data)
status_jobs = []
for p in data['pipelines']:
for q in p['change_queues']:
if p['name'] in ['gate', 'conflict']:
self.assertEqual(q['window'], 20)
else:
self.assertEqual(q['window'], 0)
for head in q['heads']:
for change in head:
self.assertTrue(change['active'])
self.assertEqual(change['id'], '1,1')
for job in change['jobs']:
status_jobs.append(job)
self.assertEqual('project-merge', status_jobs[0]['name'])
self.assertEqual('https://server/job/project-merge/0/',
status_jobs[0]['url'])
self.assertEqual('https://server/job/project-merge/0/',
status_jobs[0]['report_url'])
self.assertEqual('project-test1', status_jobs[1]['name'])
self.assertEqual('https://server/job/project-test1/0/',
status_jobs[1]['url'])
self.assertEqual('https://server/job/project-test1/0/',
status_jobs[1]['report_url'])
self.assertEqual('project-test2', status_jobs[2]['name'])
self.assertEqual('https://server/job/project-test2/0/',
status_jobs[2]['url'])
self.assertEqual('https://server/job/project-test2/0/',
status_jobs[2]['report_url'])
def test_semaphore_one(self):
"Test semaphores with max=1 (mutex)"
self.updateConfigLayout('layout-semaphore')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'semaphore-one-test1')
self.assertEqual(self.builds[2].name, 'project-test1')
self.executor_server.release('semaphore-one-test1')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test1')
self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.executor_server.release('semaphore-one-test2')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test1')
self.assertEqual(self.builds[2].name, 'semaphore-one-test1')
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.executor_server.release('semaphore-one-test1')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test1')
self.assertEqual(self.builds[2].name, 'semaphore-one-test2')
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.executor_server.release('semaphore-one-test2')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test1')
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
def test_semaphore_two(self):
"Test semaphores with max>1"
self.updateConfigLayout('layout-semaphore')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
self.assertFalse('test-semaphore-two' in
tenant.semaphore_handler.semaphores)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.builds), 4)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'semaphore-two-test1')
self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
self.assertEqual(self.builds[3].name, 'project-test1')
self.assertTrue('test-semaphore-two' in
tenant.semaphore_handler.semaphores)
self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
'test-semaphore-two', [])), 2)
self.executor_server.release('semaphore-two-test1')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 4)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'semaphore-two-test2')
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertEqual(self.builds[3].name, 'semaphore-two-test1')
self.assertTrue('test-semaphore-two' in
tenant.semaphore_handler.semaphores)
self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
'test-semaphore-two', [])), 2)
self.executor_server.release('semaphore-two-test2')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 4)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test1')
self.assertEqual(self.builds[2].name, 'semaphore-two-test1')
self.assertEqual(self.builds[3].name, 'semaphore-two-test2')
self.assertTrue('test-semaphore-two' in
tenant.semaphore_handler.semaphores)
self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
'test-semaphore-two', [])), 2)
self.executor_server.release('semaphore-two-test1')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test1')
self.assertEqual(self.builds[2].name, 'semaphore-two-test2')
self.assertTrue('test-semaphore-two' in
tenant.semaphore_handler.semaphores)
self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
'test-semaphore-two', [])), 1)
self.executor_server.release('semaphore-two-test2')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test1')
self.assertFalse('test-semaphore-two' in
tenant.semaphore_handler.semaphores)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
def test_semaphore_abandon(self):
"Test abandon with job semaphores"
self.updateConfigLayout('layout-semaphore')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
self.executor_server.hold_jobs_in_build = True
tenant = self.sched.abide.tenants.get('openstack')
check_pipeline = tenant.layout.pipelines['check']
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
self.waitUntilSettled()
# The check pipeline should be empty
items = check_pipeline.getAllItems()
self.assertEqual(len(items), 0)
# The semaphore should be released
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
def test_semaphore_reconfigure(self):
"Test reconfigure with job semaphores"
self.updateConfigLayout('layout-semaphore')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
# reconfigure without layout change
self.sched.reconfigure(self.config)
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
# semaphore still must be held
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.updateConfigLayout('layout-semaphore-reconfiguration')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
self.executor_server.release('project-test1')
self.waitUntilSettled()
# There should be no builds anymore
self.assertEqual(len(self.builds), 0)
# The semaphore should be released
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
def test_live_reconfiguration(self):
"Test that live reconfiguration works"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
@skip("Disabled for early v3 development")
def test_live_reconfiguration_merge_conflict(self):
# A real-world bug: a change in a gate queue has a merge
# conflict and a job is added to its project while it's
# sitting in the queue. The job gets added to the change and
# enqueued and the change gets stuck.
self.worker.registerFunction('build:project-test3')
self.executor_server.hold_jobs_in_build = True
# This change is fine. It's here to stop the queue long
# enough for the next change to be subject to the
# reconfiguration, as well as to provide a conflict for the
# next change. This change will succeed and merge.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addPatchset(['conflict'])
A.addApproval('code-review', 2)
# This change will be in merge conflict. During the
# reconfiguration, we will add a job. We want to make sure
# that doesn't cause it to get stuck.
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
B.addPatchset(['conflict'])
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
# No jobs have run yet
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 1)
self.assertEqual(len(self.history), 0)
# Add the "project-test3" job.
self.updateConfigLayout(
'tests/fixtures/layout-live-reconfiguration-add-job.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 2)
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test3').result,
'SUCCESS')
self.assertEqual(len(self.history), 4)
@skip("Disabled for early v3 development")
def test_live_reconfiguration_failed_root(self):
# An extrapolation of test_live_reconfiguration_merge_conflict
# that tests a job added to a job tree with a failed root does
# not run.
self.worker.registerFunction('build:project-test3')
self.executor_server.hold_jobs_in_build = True
# This change is fine. It's here to stop the queue long
# enough for the next change to be subject to the
# reconfiguration. This change will succeed and merge.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addPatchset(['conflict'])
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
self.executor_server.failJob('project-merge', B)
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# Both -merge jobs have run, but no others.
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 1)
self.assertEqual(self.history[0].result, 'SUCCESS')
self.assertEqual(self.history[0].name, 'project-merge')
self.assertEqual(self.history[1].result, 'FAILURE')
self.assertEqual(self.history[1].name, 'project-merge')
self.assertEqual(len(self.history), 2)
# Add the "project-test3" job.
self.updateConfigLayout(
'tests/fixtures/layout-live-reconfiguration-add-job.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 2)
self.assertEqual(self.history[0].result, 'SUCCESS')
self.assertEqual(self.history[0].name, 'project-merge')
self.assertEqual(self.history[1].result, 'FAILURE')
self.assertEqual(self.history[1].name, 'project-merge')
self.assertEqual(self.history[2].result, 'SUCCESS')
self.assertEqual(self.history[3].result, 'SUCCESS')
self.assertEqual(self.history[4].result, 'SUCCESS')
self.assertEqual(len(self.history), 5)
@skip("Disabled for early v3 development")
def test_live_reconfiguration_failed_job(self):
# Test that a change with a removed failing job does not
# disrupt reconfiguration. If a change has a failed job and
# that job is removed during a reconfiguration, we observed a
# bug where the code to re-set build statuses would run on
# that build and raise an exception because the job no longer
# existed.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
# This change will fail and later be removed by the reconfiguration.
self.executor_server.failJob('project-test1', A)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('project-test1')
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 0)
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'FAILURE')
self.assertEqual(len(self.history), 2)
# Remove the test1 job.
self.updateConfigLayout(
'tests/fixtures/layout-live-reconfiguration-failed-job.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-testfile').result,
'SUCCESS')
self.assertEqual(len(self.history), 4)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertIn('Build succeeded', A.messages[0])
# Ensure the removed job was not included in the report.
self.assertNotIn('project-test1', A.messages[0])
@skip("Disabled for early v3 development")
def test_live_reconfiguration_shared_queue(self):
# Test that a change with a failing job which was removed from
# this project but otherwise still exists in the system does
# not disrupt reconfiguration.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
self.executor_server.failJob('project1-project2-integration', A)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('project1-project2-integration')
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 0)
self.assertEqual(self.getJobFromHistory('project1-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory(
'project1-project2-integration').result, 'FAILURE')
self.assertEqual(len(self.history), 2)
# Remove the integration job.
self.updateConfigLayout(
'tests/fixtures/layout-live-reconfiguration-shared-queue.yaml')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project1-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project1-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project1-test2').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory(
'project1-project2-integration').result, 'FAILURE')
self.assertEqual(len(self.history), 4)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertIn('Build succeeded', A.messages[0])
# Ensure the removed job was not included in the report.
self.assertNotIn('project1-project2-integration', A.messages[0])
@skip("Disabled for early v3 development")
def test_double_live_reconfiguration_shared_queue(self):
# This was a real-world regression. A change is added to
# gate; a reconfigure happens, a second change which depends
# on the first is added, and a second reconfiguration happens.
# Ensure that both changes merge.
# A failure may indicate incorrect caching or cleaning up of
# references during a reconfiguration.
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
B.setDependsOn(A, 1)
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
# Add the parent change.
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# Reconfigure (with only one change in the pipeline).
self.sched.reconfigure(self.config)
self.waitUntilSettled()
# Add the child change.
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# Reconfigure (with both in the pipeline).
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.history), 8)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
def test_live_reconfiguration_del_project(self):
# Test project deletion from layout
# while changes are enqueued
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 5)
# This layout defines only org/project, not org/project1
self.commitLayoutUpdate('common-config',
'layout-live-reconfiguration-del-project')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
# Builds for C aborted, builds for A succeed,
# and have change B applied ahead
job_c = self.getJobFromHistory('project-test1')
self.assertEqual(job_c.changes, '3,1')
self.assertEqual(job_c.result, 'ABORTED')
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(
self.getJobFromHistory('project-test1', 'org/project').changes,
'2,1 1,1')
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 0)
self.assertEqual(C.reported, 0)
tenant = self.sched.abide.tenants.get('tenant-one')
self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
self.assertIn('Build succeeded', A.messages[0])
@skip("Disabled for early v3 development")
def test_delayed_repo_init(self):
self.updateConfigLayout(
'tests/fixtures/layout-delayed-repo-init.yaml')
self.sched.reconfigure(self.config)
self.init_repo("org/new-project")
A = self.fake_gerrit.addFakeChange('org/new-project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
def test_repo_deleted(self):
self.updateConfigLayout('layout-repo-deleted', ['org/delete-project'])
self.sched.reconfigure(self.config)
self.init_repo("org/delete-project")
A = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
# Delete org/new-project zuul repo. Should be recloned.
p = 'org/delete-project'
if os.path.exists(os.path.join(self.merger_src_root, p)):
shutil.rmtree(os.path.join(self.merger_src_root, p))
if os.path.exists(os.path.join(self.executor_src_root, p)):
shutil.rmtree(os.path.join(self.executor_src_root, p))
B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B')
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
def test_tags(self):
"Test job tags"
self.updateConfigLayout('layout-tags')
self.sched.reconfigure(self.config)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.history), 8)
results = {self.getJobFromHistory('merge',
project='org/project1').uuid: 'extratag merge',
self.getJobFromHistory('merge',
project='org/project2').uuid: 'merge'}
for build in self.history:
self.assertEqual(results.get(build.uuid, ''),
build.parameters['vars']['zuul'].get('tags'))
def test_timer(self):
"Test that a periodic job is triggered"
self.executor_server.hold_jobs_in_build = True
self.updateConfigLayout('layout-timer')
self.sched.reconfigure(self.config)
# The pipeline triggers every second, so we should have seen
# several by now.
time.sleep(5)
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2)
port = self.webapp.server.socket.getsockname()[1]
req = urllib.request.Request(
"http://localhost:%s/openstack/status" % port)
f = urllib.request.urlopen(req)
data = f.read()
self.executor_server.hold_jobs_in_build = False
# Stop queuing timer triggered jobs so that the assertions
# below don't race against more jobs being queued.
self.commitLayoutUpdate('layout-timer', 'layout-no-timer')
self.sched.reconfigure(self.config)
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory(
'project-bitrot-stable-old').result, 'SUCCESS')
self.assertEqual(self.getJobFromHistory(
'project-bitrot-stable-older').result, 'SUCCESS')
data = json.loads(data)
status_jobs = set()
for p in data['pipelines']:
for q in p['change_queues']:
for head in q['heads']:
for change in head:
for job in change['jobs']:
status_jobs.add(job['name'])
self.assertIn('project-bitrot-stable-old', status_jobs)
self.assertIn('project-bitrot-stable-older', status_jobs)
def test_idle(self):
"Test that frequent periodic jobs work"
self.executor_server.hold_jobs_in_build = True
self.updateConfigLayout('layout-idle')
for x in range(1, 3):
# Test that timer triggers periodic jobs even across
# layout config reloads.
# Start timer trigger
self.sched.reconfigure(self.config)
self.waitUntilSettled()
# The pipeline triggers every second, so we should have seen
# several by now.
time.sleep(5)
# Stop queuing timer triggered jobs so that the assertions
# below don't race against more jobs being queued.
before = self.commitLayoutUpdate('layout-idle', 'layout-no-timer')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2,
'Timer builds iteration #%d' % x)
self.executor_server.release('.*')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), x * 2)
# Revert back to layout-idle
repo = git.Repo(os.path.join(self.test_root,
'upstream',
'layout-idle'))
repo.git.reset('--hard', before)
def test_check_smtp_pool(self):
self.updateConfigLayout('layout-smtp')
self.sched.reconfigure(self.config)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.waitUntilSettled()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(self.smtp_messages), 2)
# A.messages only holds what FakeGerrit places in it. Thus we
# work on the knowledge of what the first message should be as
# it is only configured to go to SMTP.
self.assertEqual('zuul@example.com',
self.smtp_messages[0]['from_email'])
self.assertEqual(['you@example.com'],
self.smtp_messages[0]['to_email'])
self.assertEqual('Starting check jobs.',
self.smtp_messages[0]['body'])
self.assertEqual('zuul_from@example.com',
self.smtp_messages[1]['from_email'])
self.assertEqual(['alternative_me@example.com'],
self.smtp_messages[1]['to_email'])
self.assertEqual(A.messages[0],
self.smtp_messages[1]['body'])
def test_timer_smtp(self):
"Test that a periodic job is triggered"
self.executor_server.hold_jobs_in_build = True
self.updateConfigLayout('layout-timer-smtp')
self.sched.reconfigure(self.config)
# The pipeline triggers every second, so we should have seen
# several by now.
time.sleep(5)
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2)
self.executor_server.release('.*')
self.waitUntilSettled()
self.assertEqual(len(self.history), 2)
self.assertEqual(self.getJobFromHistory(
'project-bitrot-stable-old').result, 'SUCCESS')
self.assertEqual(self.getJobFromHistory(
'project-bitrot-stable-older').result, 'SUCCESS')
self.assertEqual(len(self.smtp_messages), 1)
# A.messages only holds what FakeGerrit places in it. Thus we
# work on the knowledge of what the first message should be as
# it is only configured to go to SMTP.
self.assertEqual('zuul_from@example.com',
self.smtp_messages[0]['from_email'])
self.assertEqual(['alternative_me@example.com'],
self.smtp_messages[0]['to_email'])
self.assertIn('Subject: Periodic check for org/project succeeded',
self.smtp_messages[0]['headers'])
# Stop queuing timer triggered jobs and let any that may have
# queued through so that end of test assertions pass.
self.commitLayoutUpdate('layout-timer-smtp', 'layout-no-timer')
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.executor_server.release('.*')
self.waitUntilSettled()
@skip("Disabled for early v3 development")
def test_timer_sshkey(self):
"Test that a periodic job can setup SSH key authentication"
self.worker.hold_jobs_in_build = True
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-timer.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
# The pipeline triggers every second, so we should have seen
# several by now.
time.sleep(5)
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2)
ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit")
self.assertTrue(os.path.isfile(ssh_wrapper))
with open(ssh_wrapper) as f:
ssh_wrapper_content = f.read()
self.assertIn("fake_id_rsa", ssh_wrapper_content)
# In the unit tests Merger runs in the same process,
# so we see its' environment variables
self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper)
self.worker.release('.*')
self.waitUntilSettled()
self.assertEqual(len(self.history), 2)
self.assertEqual(self.getJobFromHistory(
'project-bitrot-stable-old').result, 'SUCCESS')
self.assertEqual(self.getJobFromHistory(
'project-bitrot-stable-older').result, 'SUCCESS')
# Stop queuing timer triggered jobs and let any that may have
# queued through so that end of test assertions pass.
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-no-timer.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
self.waitUntilSettled()
self.worker.release('.*')
self.waitUntilSettled()
def test_client_enqueue_change(self):
"Test that the RPC client can enqueue a change"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
A.addApproval('approved', 1)
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
r = client.enqueue(tenant='tenant-one',
pipeline='gate',
project='org/project',
trigger='gerrit',
change='1,1')
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(r, True)
def test_client_enqueue_ref(self):
"Test that the RPC client can enqueue a ref"
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
r = client.enqueue_ref(
tenant='tenant-one',
pipeline='post',
project='org/project',
trigger='gerrit',
ref='master',
oldrev='90f173846e3af9154517b88543ffbd1691f31366',
newrev='d479a0bfcb34da57a31adb2a595c0cf687812543')
self.waitUntilSettled()
job_names = [x.name for x in self.history]
self.assertEqual(len(self.history), 1)
self.assertIn('project-post', job_names)
self.assertEqual(r, True)
def test_client_enqueue_negative(self):
"Test that the RPC client returns errors"
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
"Invalid tenant"):
r = client.enqueue(tenant='tenant-foo',
pipeline='gate',
project='org/project',
trigger='gerrit',
change='1,1')
client.shutdown()
self.assertEqual(r, False)
with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
"Invalid project"):
r = client.enqueue(tenant='tenant-one',
pipeline='gate',
project='project-does-not-exist',
trigger='gerrit',
change='1,1')
client.shutdown()
self.assertEqual(r, False)
with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
"Invalid pipeline"):
r = client.enqueue(tenant='tenant-one',
pipeline='pipeline-does-not-exist',
project='org/project',
trigger='gerrit',
change='1,1')
client.shutdown()
self.assertEqual(r, False)
with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
"Invalid trigger"):
r = client.enqueue(tenant='tenant-one',
pipeline='gate',
project='org/project',
trigger='trigger-does-not-exist',
change='1,1')
client.shutdown()
self.assertEqual(r, False)
with testtools.ExpectedException(zuul.rpcclient.RPCFailure,
"Invalid change"):
r = client.enqueue(tenant='tenant-one',
pipeline='gate',
project='org/project',
trigger='gerrit',
change='1,1')
client.shutdown()
self.assertEqual(r, False)
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
self.assertEqual(len(self.builds), 0)
def test_client_promote(self):
"Test that the RPC client can promote a change"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('tenant-one')
items = tenant.layout.pipelines['gate'].getAllItems()
enqueue_times = {}
for item in items:
enqueue_times[str(item.change)] = item.enqueue_time
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
r = client.promote(tenant='tenant-one',
pipeline='gate',
change_ids=['2,1', '3,1'])
# ensure that enqueue times are durable
items = tenant.layout.pipelines['gate'].getAllItems()
for item in items:
self.assertEqual(
enqueue_times[str(item.change)], item.enqueue_time)
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 6)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertEqual(self.builds[3].name, 'project-test2')
self.assertEqual(self.builds[4].name, 'project-test1')
self.assertEqual(self.builds[5].name, 'project-test2')
self.assertTrue(self.builds[0].hasChanges(B))
self.assertFalse(self.builds[0].hasChanges(A))
self.assertFalse(self.builds[0].hasChanges(C))
self.assertTrue(self.builds[2].hasChanges(B))
self.assertTrue(self.builds[2].hasChanges(C))
self.assertFalse(self.builds[2].hasChanges(A))
self.assertTrue(self.builds[4].hasChanges(B))
self.assertTrue(self.builds[4].hasChanges(C))
self.assertTrue(self.builds[4].hasChanges(A))
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(C.reported, 2)
client.shutdown()
self.assertEqual(r, True)
def test_client_promote_dependent(self):
"Test that the RPC client can promote a dependent change"
# C (depends on B) -> B -> A ; then promote C to get:
# A -> C (depends on B) -> B
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
C.setDependsOn(B, 1)
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
r = client.promote(tenant='tenant-one',
pipeline='gate',
change_ids=['3,1'])
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 6)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertEqual(self.builds[3].name, 'project-test2')
self.assertEqual(self.builds[4].name, 'project-test1')
self.assertEqual(self.builds[5].name, 'project-test2')
self.assertTrue(self.builds[0].hasChanges(B))
self.assertFalse(self.builds[0].hasChanges(A))
self.assertFalse(self.builds[0].hasChanges(C))
self.assertTrue(self.builds[2].hasChanges(B))
self.assertTrue(self.builds[2].hasChanges(C))
self.assertFalse(self.builds[2].hasChanges(A))
self.assertTrue(self.builds[4].hasChanges(B))
self.assertTrue(self.builds[4].hasChanges(C))
self.assertTrue(self.builds[4].hasChanges(A))
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(C.reported, 2)
client.shutdown()
self.assertEqual(r, True)
def test_client_promote_negative(self):
"Test that the RPC client returns errors for promotion"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
r = client.promote(tenant='tenant-one',
pipeline='nonexistent',
change_ids=['2,1', '3,1'])
client.shutdown()
self.assertEqual(r, False)
with testtools.ExpectedException(zuul.rpcclient.RPCFailure):
r = client.promote(tenant='tenant-one',
pipeline='gate',
change_ids=['4,1'])
client.shutdown()
self.assertEqual(r, False)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
def test_queue_rate_limiting(self):
"Test that DependentPipelines are rate limited with dep across window"
self.updateConfigLayout('layout-rate-limit')
self.sched.reconfigure(self.config)
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
C.setDependsOn(B, 1)
self.executor_server.failJob('project-test1', A)
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
# Only A and B will have their merge jobs queued because
# window is 2.
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-merge')
self.assertEqual(self.builds[1].name, 'project-merge')
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# Only A and B will have their test jobs queued because
# window is 2.
self.assertEqual(len(self.builds), 4)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertEqual(self.builds[3].name, 'project-test2')
self.executor_server.release('project-.*')
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
queue = tenant.layout.pipelines['gate'].queues[0]
# A failed so window is reduced by 1 to 1.
self.assertEqual(queue.window, 1)
self.assertEqual(queue.window_floor, 1)
self.assertEqual(A.data['status'], 'NEW')
# Gate is reset and only B's merge job is queued because
# window shrunk to 1.
self.assertEqual(len(self.builds), 1)
self.assertEqual(self.builds[0].name, 'project-merge')
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# Only B's test jobs are queued because window is still 1.
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.executor_server.release('project-.*')
self.waitUntilSettled()
# B was successfully merged so window is increased to 2.
self.assertEqual(queue.window, 2)
self.assertEqual(queue.window_floor, 1)
self.assertEqual(B.data['status'], 'MERGED')
# Only C is left and its merge job is queued.
self.assertEqual(len(self.builds), 1)
self.assertEqual(self.builds[0].name, 'project-merge')
self.executor_server.release('.*-merge')
self.waitUntilSettled()
# After successful merge job the test jobs for C are queued.
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.executor_server.release('project-.*')
self.waitUntilSettled()
# C successfully merged so window is bumped to 3.
self.assertEqual(queue.window, 3)
self.assertEqual(queue.window_floor, 1)
self.assertEqual(C.data['status'], 'MERGED')
def test_queue_rate_limiting_dependent(self):
"Test that DependentPipelines are rate limited with dep in window"
self.updateConfigLayout('layout-rate-limit')
self.sched.reconfigure(self.config)
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
B.setDependsOn(A, 1)
self.executor_server.failJob('project-test1', A)
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
# Only A and B will have their merge jobs queued because
# window is 2.
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-merge')
self.assertEqual(self.builds[1].name, 'project-merge')
self.orderedRelease(2)
# Only A and B will have their test jobs queued because
# window is 2.
self.assertEqual(len(self.builds), 4)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertEqual(self.builds[2].name, 'project-test1')
self.assertEqual(self.builds[3].name, 'project-test2')
self.executor_server.release('project-.*')
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('openstack')
queue = tenant.layout.pipelines['gate'].queues[0]
# A failed so window is reduced by 1 to 1.
self.assertEqual(queue.window, 1)
self.assertEqual(queue.window_floor, 1)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
# Gate is reset and only C's merge job is queued because
# window shrunk to 1 and A and B were dequeued.
self.assertEqual(len(self.builds), 1)
self.assertEqual(self.builds[0].name, 'project-merge')
self.orderedRelease(1)
# Only C's test jobs are queued because window is still 1.
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-test1')
self.assertEqual(self.builds[1].name, 'project-test2')
self.executor_server.release('project-.*')
self.waitUntilSettled()
# C was successfully merged so window is increased to 2.
self.assertEqual(queue.window, 2)
self.assertEqual(queue.window_floor, 1)
self.assertEqual(C.data['status'], 'MERGED')
@skip("Disabled for early v3 development")
def test_worker_update_metadata(self):
"Test if a worker can send back metadata about itself"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(len(self.executor.builds), 1)
self.log.debug('Current builds:')
self.log.debug(self.executor.builds)
start = time.time()
while True:
if time.time() - start > 10:
raise Exception("Timeout waiting for gearman server to report "
+ "back to the client")
build = self.executor.builds.values()[0]
if build.worker.name == "My Worker":
break
else:
time.sleep(0)
self.log.debug(build)
self.assertEqual("My Worker", build.worker.name)
self.assertEqual("localhost", build.worker.hostname)
self.assertEqual(['127.0.0.1', '192.168.1.1'], build.worker.ips)
self.assertEqual("zuul.example.org", build.worker.fqdn)
self.assertEqual("FakeBuilder", build.worker.program)
self.assertEqual("v1.1", build.worker.version)
self.assertEqual({'something': 'else'}, build.worker.extra)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
def test_footer_message(self):
"Test a pipeline's footer message is correctly added to the report."
self.updateConfigLayout('layout-footer-message')
self.sched.reconfigure(self.config)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(2, len(self.smtp_messages))
failure_msg = """\
Build failed. For information on how to proceed, see \
http://wiki.example.org/Test_Failures"""
footer_msg = """\
For CI problems and help debugging, contact ci@example.org"""
self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg))
self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg))
self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg))
self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg))
@skip("Disabled for early v3 development")
def test_merge_failure_reporters(self):
"""Check that the config is set up correctly"""
self.updateConfigLayout(
'tests/fixtures/layout-merge-failure.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
self.assertEqual(
"Merge Failed.\n\nThis change or one of its cross-repo "
"dependencies was unable to be automatically merged with the "
"current state of its repository. Please rebase the change and "
"upload a new patchset.",
self.sched.layout.pipelines['check'].merge_failure_message)
self.assertEqual(
"The merge failed! For more information...",
self.sched.layout.pipelines['gate'].merge_failure_message)
self.assertEqual(
len(self.sched.layout.pipelines['check'].merge_failure_actions), 1)
self.assertEqual(
len(self.sched.layout.pipelines['gate'].merge_failure_actions), 2)
self.assertTrue(isinstance(
self.sched.layout.pipelines['check'].merge_failure_actions[0],
zuul.reporter.gerrit.GerritReporter))
self.assertTrue(
(
isinstance(self.sched.layout.pipelines['gate'].
merge_failure_actions[0],
zuul.reporter.smtp.SMTPReporter) and
isinstance(self.sched.layout.pipelines['gate'].
merge_failure_actions[1],
zuul.reporter.gerrit.GerritReporter)
) or (
isinstance(self.sched.layout.pipelines['gate'].
merge_failure_actions[0],
zuul.reporter.gerrit.GerritReporter) and
isinstance(self.sched.layout.pipelines['gate'].
merge_failure_actions[1],
zuul.reporter.smtp.SMTPReporter)
)
)
@skip("Disabled for early v3 development")
def test_merge_failure_reports(self):
"""Check that when a change fails to merge the correct message is sent
to the correct reporter"""
self.updateConfigLayout(
'tests/fixtures/layout-merge-failure.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
# Check a test failure isn't reported to SMTP
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.executor_server.failJob('project-test1', A)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(3, len(self.history)) # 3 jobs
self.assertEqual(0, len(self.smtp_messages))
# Check a merge failure is reported to SMTP
# B should be merged, but C will conflict with B
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
B.addPatchset(['conflict'])
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
C.addPatchset(['conflict'])
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(6, len(self.history)) # A and B jobs
self.assertEqual(1, len(self.smtp_messages))
self.assertEqual('The merge failed! For more information...',
self.smtp_messages[0]['body'])
@skip("Disabled for early v3 development")
def test_default_merge_failure_reports(self):
"""Check that the default merge failure reports are correct."""
# A should report success, B should report merge failure.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addPatchset(['conflict'])
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
B.addPatchset(['conflict'])
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(3, len(self.history)) # A jobs
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'NEW')
self.assertIn('Build succeeded', A.messages[1])
self.assertIn('Merge Failed', B.messages[1])
self.assertIn('automatically merged', B.messages[1])
self.assertNotIn('logs.example.com', B.messages[1])
self.assertNotIn('SKIPPED', B.messages[1])
def test_client_get_running_jobs(self):
"Test that the RPC client can get a list of running jobs"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
# Wait for gearman server to send the initial workData back to zuul
start = time.time()
while True:
if time.time() - start > 10:
raise Exception("Timeout waiting for gearman server to report "
+ "back to the client")
build = self.executor_client.builds.values()[0]
if build.worker.name == "My Worker":
break
else:
time.sleep(0)
running_items = client.get_running_jobs()
self.assertEqual(1, len(running_items))
running_item = running_items[0]
self.assertEqual([], running_item['failing_reasons'])
self.assertEqual([], running_item['items_behind'])
self.assertEqual('https://hostname/1', running_item['url'])
self.assertEqual(None, running_item['item_ahead'])
self.assertEqual('org/project', running_item['project'])
self.assertEqual(None, running_item['remaining_time'])
self.assertEqual(True, running_item['active'])
self.assertEqual('1,1', running_item['id'])
self.assertEqual(3, len(running_item['jobs']))
for job in running_item['jobs']:
if job['name'] == 'project-merge':
self.assertEqual('project-merge', job['name'])
self.assertEqual('gate', job['pipeline'])
self.assertEqual(False, job['retry'])
self.assertEqual('https://server/job/project-merge/0/',
job['url'])
self.assertEqual(7, len(job['worker']))
self.assertEqual(False, job['canceled'])
self.assertEqual(True, job['voting'])
self.assertEqual(None, job['result'])
self.assertEqual('gate', job['pipeline'])
break
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
running_items = client.get_running_jobs()
self.assertEqual(0, len(running_items))
def test_nonvoting_pipeline(self):
"Test that a nonvoting pipeline (experimental) can still report"
A = self.fake_gerrit.addFakeChange('org/experimental-project',
'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-merge').result,
'SUCCESS')
self.assertEqual(
self.getJobFromHistory('experimental-project-test').result,
'SUCCESS')
self.assertEqual(A.reported, 1)
def test_crd_gate(self):
"Test cross-repo dependencies"
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
AM2 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM2')
AM1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'AM1')
AM2.setMerged()
AM1.setMerged()
BM2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM2')
BM1 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'BM1')
BM2.setMerged()
BM1.setMerged()
# A -> AM1 -> AM2
# B -> BM1 -> BM2
# A Depends-On: B
# M2 is here to make sure it is never queried. If it is, it
# means zuul is walking down the entire history of merged
# changes.
B.setDependsOn(BM1, 1)
BM1.setDependsOn(BM2, 1)
A.setDependsOn(AM1, 1)
AM1.setDependsOn(AM2, 1)
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
for connection in self.connections.connections.values():
connection.maintainCache([])
self.executor_server.hold_jobs_in_build = True
B.addApproval('approved', 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(AM2.queried, 0)
self.assertEqual(BM2.queried, 0)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
changes = self.getJobFromHistory(
'project-merge', 'org/project1').changes
self.assertEqual(changes, '2,1 1,1')
def test_crd_branch(self):
"Test cross-repo dependencies in multiple branches"
self.create_branch('org/project2', 'mp')
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'mp', 'C')
C.data['id'] = B.data['id']
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
# A Depends-On: B+C
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
self.executor_server.hold_jobs_in_build = True
B.addApproval('approved', 1)
C.addApproval('approved', 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
changes = self.getJobFromHistory(
'project-merge', 'org/project1').changes
self.assertEqual(changes, '2,1 3,1 1,1')
def test_crd_multiline(self):
"Test multiple depends-on lines in commit"
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
# A Depends-On: B+C
A.data['commitMessage'] = '%s\n\nDepends-On: %s\nDepends-On: %s\n' % (
A.subject, B.data['id'], C.data['id'])
self.executor_server.hold_jobs_in_build = True
B.addApproval('approved', 1)
C.addApproval('approved', 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(C.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
changes = self.getJobFromHistory(
'project-merge', 'org/project1').changes
self.assertEqual(changes, '2,1 3,1 1,1')
def test_crd_unshared_gate(self):
"Test cross-repo dependencies in unshared gate queues"
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
# A and B do not share a queue, make sure that A is unable to
# enqueue B (and therefore, A is unable to be enqueued).
B.addApproval('approved', 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(A.reported, 0)
self.assertEqual(B.reported, 0)
self.assertEqual(len(self.history), 0)
# Enqueue and merge B alone.
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
# Now that B is merged, A should be able to be enqueued and
# merged.
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
def test_crd_gate_reverse(self):
"Test reverse cross-repo dependencies"
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.executor_server.hold_jobs_in_build = True
A.addApproval('approved', 1)
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
changes = self.getJobFromHistory(
'project-merge', 'org/project1').changes
self.assertEqual(changes, '2,1 1,1')
def test_crd_cycle(self):
"Test cross-repo dependency cycles"
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
# A -> B -> A (via commit-depends)
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
B.subject, A.data['id'])
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.reported, 0)
self.assertEqual(B.reported, 0)
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
def test_crd_gate_unknown(self):
"Test unknown projects in dependent pipeline"
self.init_repo("org/unknown")
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
B.addApproval('approved', 1)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
# Unknown projects cannot share a queue with any other
# since they don't have common jobs with any other (they have no jobs).
# Changes which depend on unknown project changes
# should not be processed in dependent pipeline
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(A.reported, 0)
self.assertEqual(B.reported, 0)
self.assertEqual(len(self.history), 0)
# Simulate change B being gated outside this layout
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
B.setMerged()
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
# Now that B is merged, A should be able to be enqueued and
# merged.
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 0)
def test_crd_check(self):
"Test cross-repo dependencies in independent pipelines"
self.executor_server.hold_jobs_in_build = True
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
queue = self.gearman_server.getQueue()
ref = self.getParameter(queue[-1], 'ZUUL_REF')
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
path = os.path.join(self.builds[0].jobdir.src_root, "org/project1")
repo = git.Repo(path)
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
repo_messages.reverse()
correct_messages = [
'initial commit', 'add content from fixture', 'A-1']
self.assertEqual(repo_messages, correct_messages)
path = os.path.join(self.builds[0].jobdir.src_root, "org/project2")
repo = git.Repo(path)
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
repo_messages.reverse()
correct_messages = [
'initial commit', 'add content from fixture', 'B-1']
self.assertEqual(repo_messages, correct_messages)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 0)
self.assertEqual(self.history[0].changes, '2,1 1,1')
tenant = self.sched.abide.tenants.get('tenant-one')
self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
def test_crd_check_git_depends(self):
"Test single-repo dependencies in independent pipelines"
self.gearman_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
# Add two git-dependent changes and make sure they both report
# success.
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.orderedRelease()
self.gearman_server.hold_jobs_in_build = False
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
self.assertEqual(self.history[0].changes, '1,1')
self.assertEqual(self.history[-1].changes, '1,1 2,1')
tenant = self.sched.abide.tenants.get('tenant-one')
self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
self.assertIn('Build succeeded', A.messages[0])
self.assertIn('Build succeeded', B.messages[0])
def test_crd_check_duplicate(self):
"Test duplicate check in independent pipelines"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
tenant = self.sched.abide.tenants.get('tenant-one')
check_pipeline = tenant.layout.pipelines['check']
# Add two git-dependent changes...
B.setDependsOn(A, 1)
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(check_pipeline.getAllItems()), 2)
# ...make sure the live one is not duplicated...
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(check_pipeline.getAllItems()), 2)
# ...but the non-live one is able to be.
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(len(check_pipeline.getAllItems()), 3)
# Release jobs in order to avoid races with change A jobs
# finishing before change B jobs.
self.orderedRelease()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
self.assertEqual(self.history[0].changes, '1,1 2,1')
self.assertEqual(self.history[1].changes, '1,1')
self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
self.assertIn('Build succeeded', A.messages[0])
self.assertIn('Build succeeded', B.messages[0])
def _test_crd_check_reconfiguration(self, project1, project2):
"Test cross-repo dependencies re-enqueued in independent pipelines"
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange(project1, 'master', 'A')
B = self.fake_gerrit.addFakeChange(project2, 'master', 'B')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.sched.reconfigure(self.config)
# Make sure the items still share a change queue, and the
# first one is not live.
tenant = self.sched.abide.tenants.get('tenant-one')
self.assertEqual(len(tenant.layout.pipelines['check'].queues), 1)
queue = tenant.layout.pipelines['check'].queues[0]
first_item = queue.queue[0]
for item in queue.queue:
self.assertEqual(item.queue, first_item.queue)
self.assertFalse(first_item.live)
self.assertTrue(queue.queue[1].live)
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 0)
self.assertEqual(self.history[0].changes, '2,1 1,1')
self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0)
def test_crd_check_reconfiguration(self):
self._test_crd_check_reconfiguration('org/project1', 'org/project2')
def test_crd_undefined_project(self):
"""Test that undefined projects in dependencies are handled for
independent pipelines"""
# It's a hack for fake gerrit,
# as it implies repo creation upon the creation of any change
self.init_repo("org/unknown")
self._test_crd_check_reconfiguration('org/project1', 'org/unknown')
def test_crd_check_ignore_dependencies(self):
"Test cross-repo dependencies can be ignored"
self.updateConfigLayout('layout-ignore-dependencies')
self.sched.reconfigure(self.config)
self.gearman_server.hold_jobs_in_queue = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
# C git-depends on B
C.setDependsOn(B, 1)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Make sure none of the items share a change queue, and all
# are live.
tenant = self.sched.abide.tenants.get('openstack')
check_pipeline = tenant.layout.pipelines['check']
self.assertEqual(len(check_pipeline.queues), 3)
self.assertEqual(len(check_pipeline.getAllItems()), 3)
for item in check_pipeline.getAllItems():
self.assertTrue(item.live)
self.gearman_server.hold_jobs_in_queue = False
self.gearman_server.release()
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(C.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
self.assertEqual(C.reported, 1)
# Each job should have tested exactly one change
for job in self.history:
self.assertEqual(len(job.changes.split()), 1)
def test_crd_check_transitive(self):
"Test transitive cross-repo dependencies"
# Specifically, if A -> B -> C, and C gets a new patchset and
# A gets a new patchset, ensure the test of A,2 includes B,1
# and C,2 (not C,1 which would indicate stale data in the
# cache for B).
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
# B Depends-On: C
B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
B.subject, C.data['id'])
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(self.history[-1].changes, '3,1 2,1 1,1')
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(self.history[-1].changes, '3,1 2,1')
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(self.history[-1].changes, '3,1')
C.addPatchset()
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.assertEqual(self.history[-1].changes, '3,2')
A.addPatchset()
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2))
self.waitUntilSettled()
self.assertEqual(self.history[-1].changes, '3,2 2,1 1,2')
def test_crd_check_unknown(self):
"Test unknown projects in independent pipeline"
self.init_repo("org/unknown")
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/unknown', 'master', 'D')
# A Depends-On: B
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
# Make sure zuul has seen an event on B.
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 1)
self.assertEqual(B.data['status'], 'NEW')
self.assertEqual(B.reported, 0)
def test_crd_cycle_join(self):
"Test an updated change creates a cycle"
A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Create B->A
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
B.subject, A.data['id'])
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Update A to add A->B (a cycle).
A.addPatchset()
A.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
A.subject, B.data['id'])
# Normally we would submit the patchset-created event for
# processing here, however, we have no way of noting whether
# the dependency cycle detection correctly raised an
# exception, so instead, we reach into the source driver and
# call the method that would ultimately be called by the event
# processing.
tenant = self.sched.abide.tenants.get('tenant-one')
source = tenant.layout.pipelines['gate'].source
# TODO(pabelanger): As we add more source / trigger APIs we should make
# it easier for users to create events for testing.
event = zuul.model.TriggerEvent()
event.trigger_name = 'gerrit'
event.change_number = '1'
event.patch_number = '2'
with testtools.ExpectedException(
Exception, "Dependency cycle detected"):
source.getChange(event, True)
self.log.debug("Got expected dependency cycle exception")
# Now if we update B to remove the depends-on, everything
# should be okay. B; A->B
B.addPatchset()
B.data['commitMessage'] = '%s\n' % (B.subject,)
source.getChange(event, True)
event.change_number = '2'
source.getChange(event, True)
def test_disable_at(self):
"Test a pipeline will only report to the disabled trigger when failing"
self.updateConfigLayout('layout-disabled-at')
self.sched.reconfigure(self.config)
tenant = self.sched.abide.tenants.get('openstack')
self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
self.assertEqual(
0, tenant.layout.pipelines['check']._consecutive_failures)
self.assertFalse(tenant.layout.pipelines['check']._disabled)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E')
F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F')
G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G')
H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H')
I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I')
J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J')
K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K')
self.executor_server.failJob('project-test1', A)
self.executor_server.failJob('project-test1', B)
# Let C pass, resetting the counter
self.executor_server.failJob('project-test1', D)
self.executor_server.failJob('project-test1', E)
self.executor_server.failJob('project-test1', F)
self.executor_server.failJob('project-test1', G)
self.executor_server.failJob('project-test1', H)
# I also passes but should only report to the disabled reporters
self.executor_server.failJob('project-test1', J)
self.executor_server.failJob('project-test1', K)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(
2, tenant.layout.pipelines['check']._consecutive_failures)
self.assertFalse(tenant.layout.pipelines['check']._disabled)
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(
0, tenant.layout.pipelines['check']._consecutive_failures)
self.assertFalse(tenant.layout.pipelines['check']._disabled)
self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# We should be disabled now
self.assertEqual(
3, tenant.layout.pipelines['check']._consecutive_failures)
self.assertTrue(tenant.layout.pipelines['check']._disabled)
# We need to wait between each of these patches to make sure the
# smtp messages come back in an expected order
self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# The first 6 (ABCDEF) jobs should have reported back to gerrt thus
# leaving a message on each change
self.assertEqual(1, len(A.messages))
self.assertIn('Build failed.', A.messages[0])
self.assertEqual(1, len(B.messages))
self.assertIn('Build failed.', B.messages[0])
self.assertEqual(1, len(C.messages))
self.assertIn('Build succeeded.', C.messages[0])
self.assertEqual(1, len(D.messages))
self.assertIn('Build failed.', D.messages[0])
self.assertEqual(1, len(E.messages))
self.assertIn('Build failed.', E.messages[0])
self.assertEqual(1, len(F.messages))
self.assertIn('Build failed.', F.messages[0])
# The last 3 (GHI) would have only reported via smtp.
self.assertEqual(3, len(self.smtp_messages))
self.assertEqual(0, len(G.messages))
self.assertIn('Build failed.', self.smtp_messages[0]['body'])
self.assertIn(
'project-test1 https://server/job', self.smtp_messages[0]['body'])
self.assertEqual(0, len(H.messages))
self.assertIn('Build failed.', self.smtp_messages[1]['body'])
self.assertIn(
'project-test1 https://server/job', self.smtp_messages[1]['body'])
self.assertEqual(0, len(I.messages))
self.assertIn('Build succeeded.', self.smtp_messages[2]['body'])
self.assertIn(
'project-test1 https://server/job', self.smtp_messages[2]['body'])
# Now reload the configuration (simulate a HUP) to check the pipeline
# comes out of disabled
self.sched.reconfigure(self.config)
tenant = self.sched.abide.tenants.get('openstack')
self.assertEqual(3, tenant.layout.pipelines['check'].disable_at)
self.assertEqual(
0, tenant.layout.pipelines['check']._consecutive_failures)
self.assertFalse(tenant.layout.pipelines['check']._disabled)
self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(
2, tenant.layout.pipelines['check']._consecutive_failures)
self.assertFalse(tenant.layout.pipelines['check']._disabled)
# J and K went back to gerrit
self.assertEqual(1, len(J.messages))
self.assertIn('Build failed.', J.messages[0])
self.assertEqual(1, len(K.messages))
self.assertIn('Build failed.', K.messages[0])
# No more messages reported via smtp
self.assertEqual(3, len(self.smtp_messages))
def test_rerun_on_abort(self):
"Test that if a execute server fails to run a job, it is run again"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 2)
self.builds[0].requeue = True
self.executor_server.release('.*-test*')
self.waitUntilSettled()
for x in range(3):
self.assertEqual(len(self.builds), 1,
'len of builds at x=%d is wrong' % x)
self.builds[0].requeue = True
self.executor_server.release('.*-test1')
self.waitUntilSettled()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.history), 6)
self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2)
self.assertEqual(A.reported, 1)
self.assertIn('RETRY_LIMIT', A.messages[0])
def test_zookeeper_disconnect(self):
"Test that jobs are executed after a zookeeper disconnect"
self.fake_nodepool.paused = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.zk.client.stop()
self.zk.client.start()
self.fake_nodepool.paused = False
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
def test_nodepool_failure(self):
"Test that jobs are reported after a nodepool failure"
self.fake_nodepool.paused = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
req = self.fake_nodepool.getNodeRequests()[0]
self.fake_nodepool.addFailRequest(req)
self.fake_nodepool.paused = False
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(A.reported, 2)
self.assertIn('project-merge : NODE_FAILURE', A.messages[1])
self.assertIn('project-test1 : SKIPPED', A.messages[1])
self.assertIn('project-test2 : SKIPPED', A.messages[1])
class TestExecutor(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
def assertFinalState(self):
# In this test, we expect to shut down in a non-final state,
# so skip these checks.
pass
def assertCleanShutdown(self):
self.log.debug("Assert clean shutdown")
# After shutdown, make sure no jobs are running
self.assertEqual({}, self.executor_server.job_workers)
# Make sure that git.Repo objects have been garbage collected.
repos = []
gc.collect()
for obj in gc.get_objects():
if isinstance(obj, git.Repo):
self.log.debug("Leaked git repo object: %s" % repr(obj))
repos.append(obj)
self.assertEqual(len(repos), 0)
def test_executor_shutdown(self):
"Test that the executor can shut down with jobs running"
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
class TestDependencyGraph(ZuulTestCase):
tenant_config_file = 'config/dependency-graph/main.yaml'
def test_dependeny_graph_dispatch_jobs_once(self):
"Test a job in a dependency graph is queued only once"
# Job dependencies, starting with A
# A
# / \
# B C
# / \ / \
# D F E
# |
# G
self.executor_server.hold_jobs_in_build = True
change = self.fake_gerrit.addFakeChange(
'org/project', 'master', 'change')
change.addApproval('code-review', 2)
self.fake_gerrit.addEvent(change.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['A'])
self.executor_server.release('A')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
self.executor_server.release('B')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
self.executor_server.release('D')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['C'])
self.executor_server.release('C')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F'])
self.executor_server.release('F')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G'])
self.executor_server.release('G')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['E'])
self.executor_server.release('E')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 7)
self.assertEqual(change.data['status'], 'MERGED')
self.assertEqual(change.reported, 2)
def test_jobs_launched_only_if_all_dependencies_are_successful(self):
"Test that a job waits till all dependencies are successful"
# Job dependencies, starting with A
# A
# / \
# B C*
# / \ / \
# D F E
# |
# G
self.executor_server.hold_jobs_in_build = True
change = self.fake_gerrit.addFakeChange(
'org/project', 'master', 'change')
change.addApproval('code-review', 2)
self.executor_server.failJob('C', change)
self.fake_gerrit.addEvent(change.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['A'])
self.executor_server.release('A')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C'])
self.executor_server.release('B')
self.waitUntilSettled()
self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D'])
self.executor_server.release('D')
self.waitUntilSettled()
self.assertEqual([b.name for b in self.builds], ['C'])
self.executor_server.release('C')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(len(self.history), 4)
self.assertEqual(change.data['status'], 'NEW')
self.assertEqual(change.reported, 2)
class TestDuplicatePipeline(ZuulTestCase):
tenant_config_file = 'config/duplicate-pipeline/main.yaml'
def test_duplicate_pipelines(self):
"Test that a change matching multiple pipelines works"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getChangeRestoredEvent())
self.waitUntilSettled()
self.assertHistory([
dict(name='project-test1', result='SUCCESS', changes='1,1',
pipeline='dup1'),
dict(name='project-test1', result='SUCCESS', changes='1,1',
pipeline='dup2'),
], ordered=False)
self.assertEqual(len(A.messages), 2)
if 'dup1' in A.messages[0]:
self.assertIn('dup1', A.messages[0])
self.assertNotIn('dup2', A.messages[0])
self.assertIn('project-test1', A.messages[0])
self.assertIn('dup2', A.messages[1])
self.assertNotIn('dup1', A.messages[1])
self.assertIn('project-test1', A.messages[1])
else:
self.assertIn('dup1', A.messages[1])
self.assertNotIn('dup2', A.messages[1])
self.assertIn('project-test1', A.messages[1])
self.assertIn('dup2', A.messages[0])
self.assertNotIn('dup1', A.messages[0])
self.assertIn('project-test1', A.messages[0])
class TestSchedulerOneJobProject(ZuulTestCase):
tenant_config_file = 'config/one-job-project/main.yaml'
def test_one_job_project(self):
"Test that queueing works with one job"
A = self.fake_gerrit.addFakeChange('org/one-job-project',
'master', 'A')
B = self.fake_gerrit.addFakeChange('org/one-job-project',
'master', 'B')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
class TestSchedulerTemplatedProject(ZuulTestCase):
tenant_config_file = 'config/templated-project/main.yaml'
def test_job_from_templates_executed(self):
"Test whether a job generated via a template can be executed"
A = self.fake_gerrit.addFakeChange(
'org/templated-project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
def test_layered_templates(self):
"Test whether a job generated via a template can be executed"
A = self.fake_gerrit.addFakeChange(
'org/layered-project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test2').result,
'SUCCESS')
self.assertEqual(self.getJobFromHistory('layered-project-test3'
).result, 'SUCCESS')
self.assertEqual(self.getJobFromHistory('layered-project-test4'
).result, 'SUCCESS')
self.assertEqual(self.getJobFromHistory('layered-project-foo-test5'
).result, 'SUCCESS')
self.assertEqual(self.getJobFromHistory('project-test6').result,
'SUCCESS')
class TestSchedulerSuccessURL(ZuulTestCase):
tenant_config_file = 'config/success-url/main.yaml'
def test_success_url(self):
"Ensure bad build params are ignored"
self.sched.reconfigure(self.config)
self.init_repo('org/docs')
A = self.fake_gerrit.addFakeChange('org/docs', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# Both builds ran: docs-draft-test + docs-draft-test2
self.assertEqual(len(self.history), 2)
# Grab build id
for build in self.history:
if build.name == 'docs-draft-test':
uuid = build.uuid[:7]
break
# Two msgs: 'Starting...' + results
self.assertEqual(len(self.smtp_messages), 2)
body = self.smtp_messages[1]['body'].splitlines()
self.assertEqual('Build succeeded.', body[0])
self.assertIn(
'- docs-draft-test http://docs-draft.example.org/1/1/1/check/'
'docs-draft-test/%s/publish-docs/' % uuid,
body[2])
# NOTE: This default URL is currently hard-coded in executor/server.py
self.assertIn(
'- docs-draft-test2 https://server/job',
body[3])
class TestSchedulerMerges(ZuulTestCase):
tenant_config_file = 'config/merges/main.yaml'
def _test_project_merge_mode(self, mode):
self.executor_server.keep_jobdir = False
project = 'org/project-%s' % mode
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
B = self.fake_gerrit.addFakeChange(project, 'master', 'B')
C = self.fake_gerrit.addFakeChange(project, 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
build = self.builds[-1]
ref = self.getParameter(build, 'ZUUL_REF')
path = os.path.join(build.jobdir.src_root, project)
repo = git.Repo(path)
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
repo_messages.reverse()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
return repo_messages
def _test_merge(self, mode):
us_path = os.path.join(
self.upstream_root, 'org/project-%s' % mode)
expected_messages = [
'initial commit',
'add content from fixture',
# the intermediate commits order is nondeterministic
"Merge commit 'refs/changes/1/2/1' of %s into HEAD" % us_path,
"Merge commit 'refs/changes/1/3/1' of %s into HEAD" % us_path,
]
result = self._test_project_merge_mode(mode)
self.assertEqual(result[:2], expected_messages[:2])
self.assertEqual(result[-2:], expected_messages[-2:])
def test_project_merge_mode_merge(self):
self._test_merge('merge')
def test_project_merge_mode_merge_resolve(self):
self._test_merge('merge-resolve')
def test_project_merge_mode_cherrypick(self):
expected_messages = [
'initial commit',
'add content from fixture',
'A-1',
'B-1',
'C-1']
result = self._test_project_merge_mode('cherry-pick')
self.assertEqual(result, expected_messages)
def test_merge_branch(self):
"Test that the right commits are on alternate branches"
self.create_branch('org/project-merge-branches', 'mp')
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange(
'org/project-merge-branches', 'mp', 'A')
B = self.fake_gerrit.addFakeChange(
'org/project-merge-branches', 'mp', 'B')
C = self.fake_gerrit.addFakeChange(
'org/project-merge-branches', 'mp', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
self.executor_server.release('.*-merge')
self.waitUntilSettled()
build = self.builds[-1]
self.assertEqual(self.getParameter(build, 'ZUUL_BRANCH'), 'mp')
ref = self.getParameter(build, 'ZUUL_REF')
path = os.path.join(
build.jobdir.src_root, 'org/project-merge-branches')
repo = git.Repo(path)
repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
repo_messages.reverse()
correct_messages = [
'initial commit',
'add content from fixture',
'mp commit',
'A-1', 'B-1', 'C-1']
self.assertEqual(repo_messages, correct_messages)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
def test_merge_multi_branch(self):
"Test that dependent changes on multiple branches are merged"
self.create_branch('org/project-merge-branches', 'mp')
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange(
'org/project-merge-branches', 'master', 'A')
B = self.fake_gerrit.addFakeChange(
'org/project-merge-branches', 'mp', 'B')
C = self.fake_gerrit.addFakeChange(
'org/project-merge-branches', 'master', 'C')
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
job_A = None
for job in self.builds:
if 'project-merge' in job.name:
job_A = job
ref_A = self.getParameter(job_A, 'ZUUL_REF')
commit_A = self.getParameter(job_A, 'ZUUL_COMMIT')
self.log.debug("Got Zuul ref for change A: %s" % ref_A)
self.log.debug("Got Zuul commit for change A: %s" % commit_A)
path = os.path.join(
job_A.jobdir.src_root, "org/project-merge-branches")
repo = git.Repo(path)
repo_messages = [c.message.strip()
for c in repo.iter_commits(ref_A)]
repo_messages.reverse()
correct_messages = [
'initial commit', 'add content from fixture', 'A-1']
self.assertEqual(repo_messages, correct_messages)
self.executor_server.release('.*-merge')
self.waitUntilSettled()
job_B = None
for job in self.builds:
if 'project-merge' in job.name:
job_B = job
ref_B = self.getParameter(job_B, 'ZUUL_REF')
commit_B = self.getParameter(job_B, 'ZUUL_COMMIT')
self.log.debug("Got Zuul ref for change B: %s" % ref_B)
self.log.debug("Got Zuul commit for change B: %s" % commit_B)
path = os.path.join(
job_B.jobdir.src_root, "org/project-merge-branches")
repo = git.Repo(path)
repo_messages = [c.message.strip()
for c in repo.iter_commits(ref_B)]
repo_messages.reverse()
correct_messages = [
'initial commit', 'add content from fixture', 'mp commit', 'B-1']
self.assertEqual(repo_messages, correct_messages)
self.executor_server.release('.*-merge')
self.waitUntilSettled()
job_C = None
for job in self.builds:
if 'project-merge' in job.name:
job_C = job
ref_C = self.getParameter(job_C, 'ZUUL_REF')
commit_C = self.getParameter(job_C, 'ZUUL_COMMIT')
self.log.debug("Got Zuul ref for change C: %s" % ref_C)
self.log.debug("Got Zuul commit for change C: %s" % commit_C)
path = os.path.join(
job_C.jobdir.src_root, "org/project-merge-branches")
repo = git.Repo(path)
repo_messages = [c.message.strip()
for c in repo.iter_commits(ref_C)]
repo_messages.reverse()
correct_messages = [
'initial commit', 'add content from fixture',
'A-1', 'C-1']
# Ensure the right commits are in the history for this ref
self.assertEqual(repo_messages, correct_messages)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
class TestSemaphoreMultiTenant(ZuulTestCase):
tenant_config_file = 'config/multi-tenant-semaphore/main.yaml'
def test_semaphore_tenant_isolation(self):
"Test semaphores in multiple tenants"
self.waitUntilSettled()
tenant_one = self.sched.abide.tenants.get('tenant-one')
tenant_two = self.sched.abide.tenants.get('tenant-two')
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D')
E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E')
self.assertFalse('test-semaphore' in
tenant_one.semaphore_handler.semaphores)
self.assertFalse('test-semaphore' in
tenant_two.semaphore_handler.semaphores)
# add patches to project1 of tenant-one
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# one build of project1-test1 must run
# semaphore of tenant-one must be acquired once
# semaphore of tenant-two must not be acquired
self.assertEqual(len(self.builds), 1)
self.assertEqual(self.builds[0].name, 'project1-test1')
self.assertTrue('test-semaphore' in
tenant_one.semaphore_handler.semaphores)
self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
'test-semaphore', [])), 1)
self.assertFalse('test-semaphore' in
tenant_two.semaphore_handler.semaphores)
# add patches to project2 of tenant-two
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# one build of project1-test1 must run
# two builds of project2-test1 must run
# semaphore of tenant-one must be acquired once
# semaphore of tenant-two must be acquired twice
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project1-test1')
self.assertEqual(self.builds[1].name, 'project2-test1')
self.assertEqual(self.builds[2].name, 'project2-test1')
self.assertTrue('test-semaphore' in
tenant_one.semaphore_handler.semaphores)
self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
'test-semaphore', [])), 1)
self.assertTrue('test-semaphore' in
tenant_two.semaphore_handler.semaphores)
self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
'test-semaphore', [])), 2)
self.executor_server.release('project1-test1')
self.waitUntilSettled()
# one build of project1-test1 must run
# two builds of project2-test1 must run
# semaphore of tenant-one must be acquired once
# semaphore of tenant-two must be acquired twice
self.assertEqual(len(self.builds), 3)
self.assertEqual(self.builds[0].name, 'project2-test1')
self.assertEqual(self.builds[1].name, 'project2-test1')
self.assertEqual(self.builds[2].name, 'project1-test1')
self.assertTrue('test-semaphore' in
tenant_one.semaphore_handler.semaphores)
self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
'test-semaphore', [])), 1)
self.assertTrue('test-semaphore' in
tenant_two.semaphore_handler.semaphores)
self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
'test-semaphore', [])), 2)
self.executor_server.release('project2-test1')
self.waitUntilSettled()
# one build of project1-test1 must run
# one build of project2-test1 must run
# semaphore of tenant-one must be acquired once
# semaphore of tenant-two must be acquired once
self.assertEqual(len(self.builds), 2)
self.assertTrue('test-semaphore' in
tenant_one.semaphore_handler.semaphores)
self.assertEqual(len(tenant_one.semaphore_handler.semaphores.get(
'test-semaphore', [])), 1)
self.assertTrue('test-semaphore' in
tenant_two.semaphore_handler.semaphores)
self.assertEqual(len(tenant_two.semaphore_handler.semaphores.get(
'test-semaphore', [])), 1)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
# no build must run
# semaphore of tenant-one must not be acquired
# semaphore of tenant-two must not be acquired
self.assertEqual(len(self.builds), 0)
self.assertFalse('test-semaphore' in
tenant_one.semaphore_handler.semaphores)
self.assertFalse('test-semaphore' in
tenant_two.semaphore_handler.semaphores)
self.assertEqual(A.reported, 1)
self.assertEqual(B.reported, 1)
class TestSemaphoreInRepo(ZuulTestCase):
tenant_config_file = 'config/in-repo/main.yaml'
def test_semaphore_in_repo(self):
"Test semaphores in repo config"
# This tests dynamic semaphore handling in project repos. The semaphore
# max value should not be evaluated dynamically but must be updated
# after the change lands.
self.waitUntilSettled()
tenant = self.sched.abide.tenants.get('tenant-one')
in_repo_conf = textwrap.dedent(
"""
- job:
name: project-test2
semaphore: test-semaphore
- project:
name: org/project
tenant-one-gate:
jobs:
- project-test2
# the max value in dynamic layout must be ignored
- semaphore:
name: test-semaphore
max: 2
""")
in_repo_playbook = textwrap.dedent(
"""
- hosts: all
tasks: []
""")
file_dict = {'.zuul.yaml': in_repo_conf,
'playbooks/project-test2.yaml': in_repo_playbook}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
files=file_dict)
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
B.setDependsOn(A, 1)
C.setDependsOn(A, 1)
self.executor_server.hold_jobs_in_build = True
A.addApproval('code-review', 2)
B.addApproval('code-review', 2)
C.addApproval('code-review', 2)
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.fake_gerrit.addEvent(C.addApproval('approved', 1))
self.waitUntilSettled()
# check that the layout in a queue item still has max value of 1
# for test-semaphore
pipeline = tenant.layout.pipelines.get('tenant-one-gate')
queue = None
for queue_candidate in pipeline.queues:
if queue_candidate.name == 'org/project':
queue = queue_candidate
break
queue_item = queue.queue[0]
item_dynamic_layout = queue_item.current_build_set.layout
dynamic_test_semaphore = \
item_dynamic_layout.semaphores.get('test-semaphore')
self.assertEqual(dynamic_test_semaphore.max, 1)
# one build must be in queue, one semaphores acquired
self.assertEqual(len(self.builds), 1)
self.assertEqual(self.builds[0].name, 'project-test2')
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
'test-semaphore', [])), 1)
self.executor_server.release('project-test2')
self.waitUntilSettled()
# change A must be merged
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
# send change-merged event as the gerrit mock doesn't send it
self.fake_gerrit.addEvent(A.getChangeMergedEvent())
self.waitUntilSettled()
# now that change A was merged, the new semaphore max must be effective
tenant = self.sched.abide.tenants.get('tenant-one')
self.assertEqual(tenant.layout.semaphores.get('test-semaphore').max, 2)
# two builds must be in queue, two semaphores acquired
self.assertEqual(len(self.builds), 2)
self.assertEqual(self.builds[0].name, 'project-test2')
self.assertEqual(self.builds[1].name, 'project-test2')
self.assertTrue('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.assertEqual(len(tenant.semaphore_handler.semaphores.get(
'test-semaphore', [])), 2)
self.executor_server.release('project-test2')
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertFalse('test-semaphore' in
tenant.semaphore_handler.semaphores)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertEqual(len(self.builds), 0)
self.assertEqual(A.reported, 2)
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)