Make all configuration in-repo configuration
We need a way to get the ansible playbooks onto the ansible launch
servers. That's easy for jobs defined in-repo because their repo
will already be on the launch server by definition. But for the
tenant-global config, those playbooks (and roles, etc) need to be
made available as well. Rather than shipping them around on the
gearman bus which is inefficient, let's just say that those will
also be in a repo. Probably the same repo that defines the jobs,
ie in our case, the project-config repo.
In other words -- by making the global config one or more repos
that zuul knows about, the merger component of the launch server
can prepare *that* repo as well as the others involved in any
job, so that the playbooks are available to ansible.
This also has the potential upside of making changes that Depends-On
zuul configuration or ansible playbook changes much more natural
to implement.
We also keep track of the source repo for job definitions so that
we can potentially use that to tell the launch server what repos
should be included for playbooks, though this is not plumbed through
yet.
This adds some features to the test framework to copy a directory
in the fixtures dir into a git repo since we're going to have a
lot more 'git repo content' that should just be in the fixtures dir.
It also removes the merge worker from the tests because it was
racing with the ansible launcher cat handler. The merge worker
can probably be completely removed in a later change.
Change-Id: I8fc5a8f627e3d915d54d15e70b7960655a6332a1
diff --git a/tests/base.py b/tests/base.py
index b265bd0..736d2f7 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -974,10 +974,6 @@
old_urlopen = urllib2.urlopen
urllib2.urlopen = URLOpenerFactory
- self.merge_server = zuul.merger.server.MergeServer(self.config,
- self.connections)
- self.merge_server.start()
-
self.launcher = zuul.launcher.launchclient.LaunchClient(
self.config, self.sched, self.swift)
self.merge_client = zuul.merger.client.MergeClient(
@@ -1076,6 +1072,32 @@
self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
if hasattr(self, 'tenant_config_file'):
self.config.set('zuul', 'tenant_config', self.tenant_config_file)
+ git_path = os.path.join(
+ os.path.dirname(
+ os.path.join(FIXTURE_DIR, self.tenant_config_file)),
+ 'git')
+ if os.path.exists(git_path):
+ for reponame in os.listdir(git_path):
+ self.copyDirToRepo(reponame,
+ os.path.join(git_path, reponame))
+
+ def copyDirToRepo(self, project, source_path):
+ repo_path = os.path.join(self.upstream_root, project)
+ if not os.path.exists(repo_path):
+ self.init_repo(project)
+
+ files = {}
+ for (dirpath, dirnames, filenames) in os.walk(source_path):
+ for filename in filenames:
+ test_tree_filepath = os.path.join(dirpath, filename)
+ common_path = os.path.commonprefix([test_tree_filepath,
+ source_path])
+ relative_filepath = test_tree_filepath[len(common_path) + 1:]
+ with open(test_tree_filepath, 'r') as f:
+ content = f.read()
+ files[relative_filepath] = content
+ self.addCommitToRepo(project, 'add content from fixture',
+ files, branch='master')
def setup_repos(self):
"""Subclasses can override to manipulate repos before tests"""
@@ -1099,8 +1121,6 @@
def shutdown(self):
self.log.debug("Shutting down after tests")
self.launcher.stop()
- self.merge_server.stop()
- self.merge_server.join()
self.merge_client.stop()
self.sched.stop()
self.sched.join()
@@ -1233,7 +1253,6 @@
time.sleep(0)
self.gearman_server.functions = set()
self.rpc.register()
- self.merge_server.register()
def haveAllBuildsReported(self):
# See if Zuul is waiting on a meta job to complete
@@ -1331,13 +1350,13 @@
jobs = filter(lambda x: x.result == result, jobs)
return len(jobs)
- def getJobFromHistory(self, name):
+ def getJobFromHistory(self, name, project=None):
history = self.ansible_server.job_history
for job in history:
params = json.loads(job.arguments)
- if params['job'] == name:
+ if (params['job'] == name and
+ (project is None or params['ZUUL_PROJECT'] == project)):
result = json.loads(job.data[-1])
- print result
ret = BuildHistory(job=job,
name=params['job'],
result=result['result'])
diff --git a/tests/fixtures/config/in-repo/common.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
similarity index 100%
rename from tests/fixtures/config/in-repo/common.yaml
rename to tests/fixtures/config/in-repo/git/common-config/zuul.yaml
diff --git a/tests/fixtures/config/in-repo/main.yaml b/tests/fixtures/config/in-repo/main.yaml
index e8b7665..d9868fa 100644
--- a/tests/fixtures/config/in-repo/main.yaml
+++ b/tests/fixtures/config/in-repo/main.yaml
@@ -1,8 +1,8 @@
- tenant:
name: tenant-one
- include:
- - common.yaml
source:
gerrit:
- repos:
+ config-repos:
+ - common-config
+ project-repos:
- org/project
diff --git a/tests/fixtures/config/multi-tenant/common.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
similarity index 87%
rename from tests/fixtures/config/multi-tenant/common.yaml
rename to tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
index 6014227..7719573 100644
--- a/tests/fixtures/config/multi-tenant/common.yaml
+++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
@@ -12,3 +12,7 @@
failure:
gerrit:
verified: -1
+
+- job:
+ name:
+ python27
diff --git a/tests/fixtures/config/multi-tenant/tenant-one.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
similarity index 93%
rename from tests/fixtures/config/multi-tenant/tenant-one.yaml
rename to tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
index 86a98da..785f8a5 100644
--- a/tests/fixtures/config/multi-tenant/tenant-one.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml
@@ -29,7 +29,9 @@
name: org/project1
check:
jobs:
+ - python27
- project1-test1
tenant-one-gate:
jobs:
+ - python27
- project1-test1
diff --git a/tests/fixtures/config/multi-tenant/tenant-two.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
similarity index 93%
rename from tests/fixtures/config/multi-tenant/tenant-two.yaml
rename to tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
index 3f80a95..c6127ca 100644
--- a/tests/fixtures/config/multi-tenant/tenant-two.yaml
+++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml
@@ -29,7 +29,9 @@
name: org/project2
check:
jobs:
+ - python27
- project2-test1
tenant-two-gate:
jobs:
+ - python27
- project2-test1
diff --git a/tests/fixtures/config/multi-tenant/main.yaml b/tests/fixtures/config/multi-tenant/main.yaml
index b9d780c..b1c47b1 100644
--- a/tests/fixtures/config/multi-tenant/main.yaml
+++ b/tests/fixtures/config/multi-tenant/main.yaml
@@ -1,11 +1,15 @@
- tenant:
name: tenant-one
- include:
- - common.yaml
- - tenant-one.yaml
+ source:
+ gerrit:
+ config-repos:
+ - common-config
+ - tenant-one-config
- tenant:
name: tenant-two
- include:
- - common.yaml
- - tenant-two.yaml
+ source:
+ gerrit:
+ config-repos:
+ - common-config
+ - tenant-two-config
diff --git a/tests/fixtures/config/project-template/common.yaml b/tests/fixtures/config/project-template/git/common-config/zuul.yaml
similarity index 100%
rename from tests/fixtures/config/project-template/common.yaml
rename to tests/fixtures/config/project-template/git/common-config/zuul.yaml
diff --git a/tests/fixtures/config/project-template/main.yaml b/tests/fixtures/config/project-template/main.yaml
index 25dea57..a22ed5c 100644
--- a/tests/fixtures/config/project-template/main.yaml
+++ b/tests/fixtures/config/project-template/main.yaml
@@ -1,4 +1,6 @@
- tenant:
name: tenant-one
- include:
- - common.yaml
+ source:
+ gerrit:
+ config-repos:
+ - common-config
diff --git a/tests/test_v3.py b/tests/test_v3.py
index 8425383..8874015 100644
--- a/tests/test_v3.py
+++ b/tests/test_v3.py
@@ -38,6 +38,8 @@
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project1-test1').result,
'SUCCESS')
+ self.assertEqual(self.getJobFromHistory('python27').result,
+ 'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2,
"A should report start and success")
@@ -50,6 +52,9 @@
B.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('python27',
+ 'org/project2').result,
+ 'SUCCESS')
self.assertEqual(self.getJobFromHistory('project2-test1').result,
'SUCCESS')
self.assertEqual(B.data['status'], 'MERGED')