Add job inheritance and start refactoring
This begins a lot of related changes refactoring config loading,
the data model, etc., which will continue in subsequent changes.
Change-Id: I2ca52a079a837555c1f668e29d5a2fe0a80c1af5
diff --git a/tests/base.py b/tests/base.py
index 497d706..8efdfd1 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -50,6 +50,7 @@
import zuul.rpclistener
import zuul.launcher.gearman
import zuul.lib.swift
+import zuul.lib.connections
import zuul.merger.client
import zuul.merger.merger
import zuul.merger.server
@@ -864,6 +865,7 @@
class ZuulTestCase(BaseTestCase):
+ config_file = 'zuul.conf'
def setUp(self):
super(ZuulTestCase, self).setUp()
@@ -907,6 +909,8 @@
self.init_repo("org/experimental-project")
self.init_repo("org/no-jobs-project")
+ self.setup_repos()
+
self.statsd = FakeStatsd()
# note, use 127.0.0.1 rather than localhost to avoid getting ipv6
# see: https://github.com/jsocol/pystatsd/issues/61
@@ -940,7 +944,7 @@
self.sched.trigger_event_queue
]
- self.configure_connections()
+ self.configure_connections(self.sched)
self.sched.registerConnections(self.connections)
def URLOpenerFactory(*args, **kw):
@@ -979,7 +983,7 @@
self.addCleanup(self.assertFinalState)
self.addCleanup(self.shutdown)
- def configure_connections(self):
+ def configure_connections(self, sched):
# Register connections from the config
self.smtp_messages = []
@@ -993,7 +997,7 @@
# a virtual canonical database given by the configured hostname
self.gerrit_changes_dbs = {}
self.gerrit_queues_dbs = {}
- self.connections = {}
+ self.connections = zuul.lib.connections.ConnectionRegistry(sched)
for section_name in self.config.sections():
con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
@@ -1018,15 +1022,16 @@
Queue.Queue()
self.event_queues.append(
self.gerrit_queues_dbs[con_config['server']])
- self.connections[con_name] = FakeGerritConnection(
+ self.connections.connections[con_name] = FakeGerritConnection(
con_name, con_config,
changes_db=self.gerrit_changes_dbs[con_config['server']],
queues_db=self.gerrit_queues_dbs[con_config['server']],
upstream_root=self.upstream_root
)
- setattr(self, 'fake_' + con_name, self.connections[con_name])
+ setattr(self, 'fake_' + con_name,
+ self.connections.connections[con_name])
elif con_driver == 'smtp':
- self.connections[con_name] = \
+ self.connections.connections[con_name] = \
zuul.connection.smtp.SMTPConnection(con_name, con_config)
else:
raise Exception("Unknown driver, %s, for connection %s"
@@ -1039,20 +1044,24 @@
self.gerrit_changes_dbs['gerrit'] = {}
self.gerrit_queues_dbs['gerrit'] = Queue.Queue()
self.event_queues.append(self.gerrit_queues_dbs['gerrit'])
- self.connections['gerrit'] = FakeGerritConnection(
+ self.connections.connections['gerrit'] = FakeGerritConnection(
'_legacy_gerrit', dict(self.config.items('gerrit')),
changes_db=self.gerrit_changes_dbs['gerrit'],
queues_db=self.gerrit_queues_dbs['gerrit'])
if 'smtp' in self.config.sections():
- self.connections['smtp'] = \
+ self.connections.connections['smtp'] = \
zuul.connection.smtp.SMTPConnection(
'_legacy_smtp', dict(self.config.items('smtp')))
- def setup_config(self, config_file='zuul.conf'):
+ def setup_config(self):
"""Per test config object. Override to set different config."""
self.config = ConfigParser.ConfigParser()
- self.config.read(os.path.join(FIXTURE_DIR, config_file))
+ self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
+
+ def setup_repos(self):
+ """Subclasses can override to manipulate repos before tests"""
+ pass
def assertFinalState(self):
# Make sure that git.Repo objects have been garbage collected.
@@ -1063,10 +1072,10 @@
repos.append(obj)
self.assertEqual(len(repos), 0)
self.assertEmptyQueues()
+ ipm = zuul.manager.independent.IndependentPipelineManager
for tenant in self.sched.abide.tenants.values():
for pipeline in tenant.layout.pipelines.values():
- if isinstance(pipeline.manager,
- zuul.scheduler.IndependentPipelineManager):
+ if isinstance(pipeline.manager, ipm):
self.assertEqual(len(pipeline.queues), 0)
def shutdown(self):
diff --git a/tests/fixtures/config/in-repo/common.yaml b/tests/fixtures/config/in-repo/common.yaml
index 96aebd6..f38406b 100644
--- a/tests/fixtures/config/in-repo/common.yaml
+++ b/tests/fixtures/config/in-repo/common.yaml
@@ -1,6 +1,6 @@
pipelines:
- name: check
- manager: IndependentPipelineManager
+ manager: independent
source:
gerrit
trigger:
@@ -14,7 +14,7 @@
verified: -1
- name: tenant-one-gate
- manager: DependentPipelineManager
+ manager: dependent
success-message: Build succeeded (tenant-one-gate).
source:
gerrit
diff --git a/tests/fixtures/config/in-repo/zuul.conf b/tests/fixtures/config/in-repo/zuul.conf
index 14708aa..1910084 100644
--- a/tests/fixtures/config/in-repo/zuul.conf
+++ b/tests/fixtures/config/in-repo/zuul.conf
@@ -2,7 +2,7 @@
server=127.0.0.1
[zuul]
-tenant_config=tests/fixtures/config/in-repo/main.yaml
+tenant_config=config/in-repo/main.yaml
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
job_name_in_report=true
diff --git a/tests/fixtures/config/multi-tenant/common.yaml b/tests/fixtures/config/multi-tenant/common.yaml
index d36448e..8fc3bba 100644
--- a/tests/fixtures/config/multi-tenant/common.yaml
+++ b/tests/fixtures/config/multi-tenant/common.yaml
@@ -1,6 +1,6 @@
pipelines:
- name: check
- manager: IndependentPipelineManager
+ manager: independent
source:
gerrit
trigger:
diff --git a/tests/fixtures/config/multi-tenant/tenant-one.yaml b/tests/fixtures/config/multi-tenant/tenant-one.yaml
index 7b2298c..c9096ef 100644
--- a/tests/fixtures/config/multi-tenant/tenant-one.yaml
+++ b/tests/fixtures/config/multi-tenant/tenant-one.yaml
@@ -1,6 +1,6 @@
pipelines:
- name: tenant-one-gate
- manager: DependentPipelineManager
+ manager: dependent
success-message: Build succeeded (tenant-one-gate).
source:
gerrit
@@ -21,6 +21,10 @@
verified: 0
precedence: high
+jobs:
+ - name:
+ project1-test1
+
projects:
- name: org/project1
check:
diff --git a/tests/fixtures/config/multi-tenant/tenant-two.yaml b/tests/fixtures/config/multi-tenant/tenant-two.yaml
index 57ad64d..6cb2d9a 100644
--- a/tests/fixtures/config/multi-tenant/tenant-two.yaml
+++ b/tests/fixtures/config/multi-tenant/tenant-two.yaml
@@ -1,6 +1,6 @@
pipelines:
- name: tenant-two-gate
- manager: DependentPipelineManager
+ manager: dependent
success-message: Build succeeded (tenant-two-gate).
source:
gerrit
@@ -21,6 +21,10 @@
verified: 0
precedence: high
+jobs:
+ - name:
+ project2-test1
+
projects:
- name: org/project2
check:
diff --git a/tests/fixtures/config/multi-tenant/zuul.conf b/tests/fixtures/config/multi-tenant/zuul.conf
index ceb3903..346450e 100644
--- a/tests/fixtures/config/multi-tenant/zuul.conf
+++ b/tests/fixtures/config/multi-tenant/zuul.conf
@@ -2,7 +2,7 @@
server=127.0.0.1
[zuul]
-tenant_config=tests/fixtures/config/multi-tenant/main.yaml
+tenant_config=config/multi-tenant/main.yaml
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
job_name_in_report=true
diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml
index 99b135c..e30147f 100644
--- a/tests/fixtures/layout.yaml
+++ b/tests/fixtures/layout.yaml
@@ -3,7 +3,7 @@
pipelines:
- name: check
- manager: IndependentPipelineManager
+ manager: independent
source:
gerrit
trigger:
@@ -17,7 +17,7 @@
verified: -1
- name: post
- manager: IndependentPipelineManager
+ manager: independent
source:
gerrit
trigger:
@@ -26,7 +26,7 @@
ref: ^(?!refs/).*$
- name: gate
- manager: DependentPipelineManager
+ manager: dependent
failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
source:
gerrit
@@ -48,7 +48,7 @@
precedence: high
- name: unused
- manager: IndependentPipelineManager
+ manager: independent
dequeue-on-new-patchset: false
source:
gerrit
@@ -59,7 +59,7 @@
- approved: 1
- name: dup1
- manager: IndependentPipelineManager
+ manager: independent
source:
gerrit
trigger:
@@ -73,7 +73,7 @@
verified: -1
- name: dup2
- manager: IndependentPipelineManager
+ manager: independent
source:
gerrit
trigger:
@@ -87,7 +87,7 @@
verified: -1
- name: conflict
- manager: DependentPipelineManager
+ manager: dependent
failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
source:
gerrit
@@ -108,7 +108,7 @@
verified: 0
- name: experimental
- manager: IndependentPipelineManager
+ manager: independent
source:
gerrit
trigger:
diff --git a/tests/test_layoutvalidator.py b/tests/test_layoutvalidator.py
index 3dc3234..bd507d1 100644
--- a/tests/test_layoutvalidator.py
+++ b/tests/test_layoutvalidator.py
@@ -31,6 +31,9 @@
class TestLayoutValidator(testtools.TestCase):
+ def setUp(self):
+ self.skip("Disabled for early v3 development")
+
def test_layouts(self):
"""Test layout file validation"""
print
diff --git a/tests/test_merger_repo.py b/tests/test_merger_repo.py
index 454f3cc..7bf08ee 100644
--- a/tests/test_merger_repo.py
+++ b/tests/test_merger_repo.py
@@ -34,8 +34,11 @@
workspace_root = None
def setUp(self):
- super(TestMergerRepo, self).setUp()
- self.workspace_root = os.path.join(self.test_root, 'workspace')
+ self.skip("Disabled for early v3 development")
+
+ # def setUp(self):
+ # super(TestMergerRepo, self).setUp()
+ # self.workspace_root = os.path.join(self.test_root, 'workspace')
def test_ensure_cloned(self):
parent_path = os.path.join(self.upstream_root, 'org/project1')
diff --git a/tests/test_model.py b/tests/test_model.py
index d4c7880..f8f74dc 100644
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -12,8 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-from zuul import change_matcher as cm
from zuul import model
+from zuul import configloader
from tests.base import BaseTestCase
@@ -22,11 +22,12 @@
@property
def job(self):
- job = model.Job('job')
- job.skip_if_matcher = cm.MatchAll([
- cm.ProjectMatcher('^project$'),
- cm.MatchAllFiles([cm.FileMatcher('^docs/.*$')]),
- ])
+ layout = model.Layout()
+ job = configloader.JobParser.fromYaml(layout, {
+ 'name': 'job',
+ 'irrelevant-files': [
+ '^docs/.*$'
+ ]})
return job
def test_change_matches_returns_false_for_matched_skip_if(self):
@@ -39,10 +40,61 @@
change.files = ['foo']
self.assertTrue(self.job.changeMatches(change))
- def _assert_job_booleans_are_not_none(self, job):
- self.assertIsNotNone(job.voting)
- self.assertIsNotNone(job.hold_following_changes)
-
def test_job_sets_defaults_for_boolean_attributes(self):
- job = model.Job('job')
- self._assert_job_booleans_are_not_none(job)
+ self.assertIsNotNone(self.job.voting)
+
+ def test_job_inheritance(self):
+ layout = model.Layout()
+ base = configloader.JobParser.fromYaml(layout, {
+ 'name': 'base',
+ 'timeout': 30,
+ })
+ layout.addJob(base)
+ python27 = configloader.JobParser.fromYaml(layout, {
+ 'name': 'python27',
+ 'parent': 'base',
+ 'timeout': 40,
+ })
+ layout.addJob(python27)
+ python27diablo = configloader.JobParser.fromYaml(layout, {
+ 'name': 'python27',
+ 'branches': [
+ 'stable/diablo'
+ ],
+ 'timeout': 50,
+ })
+ layout.addJob(python27diablo)
+
+ pipeline = model.Pipeline('gate', layout)
+ layout.addPipeline(pipeline)
+ queue = model.ChangeQueue(pipeline)
+
+ project = model.Project('project')
+ tree = pipeline.addProject(project)
+ tree.addJob(layout.getJob('python27'))
+
+ change = model.Change(project)
+ change.branch = 'master'
+ item = queue.enqueueChange(change)
+
+ self.assertTrue(base.changeMatches(change))
+ self.assertTrue(python27.changeMatches(change))
+ self.assertFalse(python27diablo.changeMatches(change))
+
+ item.freezeJobTree()
+ self.assertEqual(len(item.getJobs()), 1)
+ job = item.getJobs()[0]
+ self.assertEqual(job.name, 'python27')
+ self.assertEqual(job.timeout, 40)
+
+ change.branch = 'stable/diablo'
+
+ self.assertTrue(base.changeMatches(change))
+ self.assertTrue(python27.changeMatches(change))
+ self.assertTrue(python27diablo.changeMatches(change))
+
+ item.freezeJobTree()
+ self.assertEqual(len(item.getJobs()), 1)
+ job = item.getJobs()[0]
+ self.assertEqual(job.name, 'python27')
+ self.assertEqual(job.timeout, 50)
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 7ef166c..85ac600 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -46,6 +46,9 @@
class TestSchedulerConfigParsing(BaseTestCase):
+ def setUp(self):
+ self.skip("Disabled for early v3 development")
+
def test_parse_skip_if(self):
job_yaml = """
jobs:
diff --git a/tests/test_v3.py b/tests/test_v3.py
index 69e66a0..73efcc9 100644
--- a/tests/test_v3.py
+++ b/tests/test_v3.py
@@ -26,13 +26,12 @@
'%(levelname)-8s %(message)s')
-class TestV3(ZuulTestCase):
+class TestMultipleTenants(ZuulTestCase):
# A temporary class to hold new tests while others are disabled
- def test_multiple_tenants(self):
- self.setup_config('config/multi-tenant/zuul.conf')
- self.sched.reconfigure(self.config)
+ config_file = 'config/multi-tenant/zuul.conf'
+ def test_multiple_tenants(self):
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
A.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
@@ -64,9 +63,18 @@
self.assertEqual(A.reported, 2, "Activity in tenant two should"
"not affect tenant one")
- def test_in_repo_config(self):
+
+class TestInRepoConfig(ZuulTestCase):
+ # A temporary class to hold new tests while others are disabled
+
+ config_file = 'config/in-repo/zuul.conf'
+
+ def setup_repos(self):
in_repo_conf = textwrap.dedent(
"""
+ jobs:
+ - name: project-test1
+
projects:
- name: org/project
tenant-one-gate:
@@ -76,9 +84,7 @@
self.addCommitToRepo('org/project', 'add zuul conf',
{'.zuul.yaml': in_repo_conf})
- self.setup_config('config/in-repo/zuul.conf')
- self.sched.reconfigure(self.config)
-
+ def test_in_repo_config(self):
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))