Load in-repo configuration
Change-Id: I225934407ce31f92a9b6df4bc282fbd5ec2968b3
diff --git a/tests/base.py b/tests/base.py
index 5e1befe..497d706 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1307,7 +1307,7 @@
# processed
self.eventQueuesJoin()
self.sched.run_handler_lock.acquire()
- if (not self.merge_client.build_sets and
+ if (not self.merge_client.jobs and
all(self.eventQueuesEmpty()) and
self.haveAllBuildsReported() and
self.areAllBuildsWaiting()):
@@ -1376,3 +1376,19 @@
""" % os.path.abspath(path))
f.close()
self.config.set('zuul', 'tenant_config', f.name)
+
+ def addCommitToRepo(self, project, message, files, branch='master'):
+ path = os.path.join(self.upstream_root, project)
+ repo = git.Repo(path)
+ repo.head.reference = branch
+ zuul.merger.merger.reset_repo_to_head(repo)
+ for fn, content in files.items():
+ fn = os.path.join(path, fn)
+ with open(fn, 'w') as f:
+ f.write(content)
+ repo.index.add([fn])
+ commit = repo.index.commit(message)
+ repo.heads[branch].commit = commit
+ repo.head.reference = branch
+ repo.git.clean('-x', '-f', '-d')
+ repo.heads[branch].checkout()
diff --git a/tests/fixtures/config/in-repo/common.yaml b/tests/fixtures/config/in-repo/common.yaml
new file mode 100644
index 0000000..96aebd6
--- /dev/null
+++ b/tests/fixtures/config/in-repo/common.yaml
@@ -0,0 +1,36 @@
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+ - name: tenant-one-gate
+ manager: DependentPipelineManager
+ success-message: Build succeeded (tenant-one-gate).
+ source:
+ gerrit
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
diff --git a/tests/fixtures/config/in-repo/main.yaml b/tests/fixtures/config/in-repo/main.yaml
new file mode 100644
index 0000000..df7dadd
--- /dev/null
+++ b/tests/fixtures/config/in-repo/main.yaml
@@ -0,0 +1,8 @@
+tenants:
+ - name: tenant-one
+ include:
+ - common.yaml
+ source:
+ gerrit:
+ repos:
+ - org/project
diff --git a/tests/fixtures/config/in-repo/zuul.conf b/tests/fixtures/config/in-repo/zuul.conf
new file mode 100644
index 0000000..14708aa
--- /dev/null
+++ b/tests/fixtures/config/in-repo/zuul.conf
@@ -0,0 +1,36 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+tenant_config=tests/fixtures/config/in-repo/main.yaml
+url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
+job_name_in_report=true
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[swift]
+authurl=https://identity.api.example.org/v2.0/
+user=username
+key=password
+tenant_name=" "
+
+default_container=logs
+region_name=EXP
+logserver_prefix=http://logs.example.org/server.app/
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=none
+
+[connection smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com
diff --git a/tests/test_v3.py b/tests/test_v3.py
index 2e16742..69e66a0 100644
--- a/tests/test_v3.py
+++ b/tests/test_v3.py
@@ -15,6 +15,7 @@
# under the License.
import logging
+import textwrap
from tests.base import (
ZuulTestCase,
@@ -62,3 +63,30 @@
self.assertEqual(A.reported, 2, "Activity in tenant two should"
"not affect tenant one")
+
+ def test_in_repo_config(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ projects:
+ - name: org/project
+ tenant-one-gate:
+ - project-test1
+ """)
+
+ self.addCommitToRepo('org/project', 'add zuul conf',
+ {'.zuul.yaml': in_repo_conf})
+
+ self.setup_config('config/in-repo/zuul.conf')
+ self.sched.reconfigure(self.config)
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-test1').result,
+ 'SUCCESS')
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2,
+ "A should report start and success")
+ self.assertIn('tenant-one-gate', A.messages[1],
+ "A should transit tenant-one gate")
diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py
index e7bddba..0adb78c 100644
--- a/zuul/layoutvalidator.py
+++ b/zuul/layoutvalidator.py
@@ -37,7 +37,7 @@
def validateTenantSource(self, value, path=[]):
# TODOv3(jeblair): validate against connections
- self.tenant_source.schema(value)
+ self.tenant_source(value)
def getSchema(self, data, connections=None):
tenant = {v.Required('name'): str,
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 950c385..ce04795 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -14,6 +14,7 @@
import json
import logging
+import threading
from uuid import uuid4
import gear
@@ -55,6 +56,18 @@
self.__merge_client.onBuildCompleted(job)
+class MergeJob(gear.Job):
+ def __init__(self, *args, **kw):
+ super(MergeJob, self).__init__(*args, **kw)
+ self.__event = threading.Event()
+
+ def setComplete(self):
+ self.__event.set()
+
+ def wait(self, timeout=300):
+ return self.__event.wait(timeout)
+
+
class MergeClient(object):
log = logging.getLogger("zuul.MergeClient")
@@ -71,26 +84,28 @@
self.gearman.addServer(server, port)
self.log.debug("Waiting for gearman")
self.gearman.waitForServer()
- self.build_sets = {}
+ self.jobs = set()
def stop(self):
self.gearman.shutdown()
def areMergesOutstanding(self):
- if self.build_sets:
+ if self.jobs:
return True
return False
def submitJob(self, name, data, build_set,
precedence=zuul.model.PRECEDENCE_NORMAL):
uuid = str(uuid4().hex)
- job = gear.Job(name,
+ job = MergeJob(name,
json.dumps(data),
unique=uuid)
+ job.build_set = build_set
self.log.debug("Submitting job %s with data %s" % (job, data))
- self.build_sets[uuid] = build_set
+ self.jobs.add(job)
self.gearman.submitJob(job, precedence=precedence,
timeout=300)
+ return job
def mergeChanges(self, items, build_set,
precedence=zuul.model.PRECEDENCE_NORMAL):
@@ -103,21 +118,29 @@
url=url)
self.submitJob('merger:update', data, build_set, precedence)
+ def getFiles(self, project, url, branch, files,
+ precedence=zuul.model.PRECEDENCE_HIGH):
+ data = dict(project=project,
+ url=url,
+ branch=branch,
+ files=files)
+ job = self.submitJob('merger:cat', data, None, precedence)
+ return job
+
def onBuildCompleted(self, job):
- build_set = self.build_sets.get(job.unique)
- if build_set:
- data = getJobData(job)
- zuul_url = data.get('zuul_url')
- merged = data.get('merged', False)
- updated = data.get('updated', False)
- commit = data.get('commit')
- self.log.info("Merge %s complete, merged: %s, updated: %s, "
- "commit: %s" %
- (job, merged, updated, build_set.commit))
- self.sched.onMergeCompleted(build_set, zuul_url,
+ data = getJobData(job)
+ zuul_url = data.get('zuul_url')
+ merged = data.get('merged', False)
+ updated = data.get('updated', False)
+ commit = data.get('commit')
+ job.files = data.get('files', {})
+ self.log.info("Merge %s complete, merged: %s, updated: %s, "
+ "commit: %s" %
+ (job, merged, updated, commit))
+ job.setComplete()
+ if job.build_set:
+ self.sched.onMergeCompleted(job.build_set, zuul_url,
merged, updated, commit)
- # The test suite expects the build_set to be removed from
- # the internal dict after the wake flag is set.
- del self.build_sets[job.unique]
- else:
- self.log.error("Unable to find build set for uuid %s" % job.unique)
+ # The test suite expects the job to be removed from the
+ # internal account after the wake flag is set.
+ self.jobs.remove(job)
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index c6ae35d..b7e1842 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -184,6 +184,17 @@
origin = repo.remotes.origin
origin.update()
+ def getFiles(self, branch, files):
+ ret = {}
+ repo = self.createRepoObject()
+ for fn in files:
+ tree = repo.heads[branch].commit.tree
+ if fn in tree:
+ ret[fn] = tree[fn].data_stream.read()
+ else:
+ ret[fn] = None
+ return ret
+
class Merger(object):
log = logging.getLogger("zuul.Merger")
@@ -342,3 +353,7 @@
if not commit:
return None
return commit.hexsha
+
+ def getFiles(self, project, url, branch, files):
+ repo = self.getRepo(project, url)
+ return repo.getFiles(branch, files)
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index 30cd732..813c602 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -68,6 +68,7 @@
def register(self):
self.worker.registerFunction("merger:merge")
self.worker.registerFunction("merger:update")
+ self.worker.registerFunction("merger:cat")
def stop(self):
self.log.debug("Stopping")
@@ -90,6 +91,9 @@
elif job.name == 'merger:update':
self.log.debug("Got update job: %s" % job.unique)
self.update(job)
+ elif job.name == 'merger:cat':
+ self.log.debug("Got cat job: %s" % job.unique)
+ self.cat(job)
else:
self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail()
@@ -113,3 +117,13 @@
result = dict(updated=True,
zuul_url=self.zuul_url)
job.sendWorkComplete(json.dumps(result))
+
+ def cat(self, job):
+ args = json.loads(job.arguments)
+ self.merger.updateRepo(args['project'], args['url'])
+ files = self.merger.getFiles(args['project'], args['url'],
+ args['branch'], args['files'])
+ result = dict(updated=True,
+ files=files,
+ zuul_url=self.zuul_url)
+ job.sendWorkComplete(json.dumps(result))
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 06c54cf..5ae9e9b 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -353,6 +353,7 @@
raise Exception("Unable to read tenant config file at %s" %
config_path)
with open(config_path) as config_file:
+ self.log.info("Loading configuration from %s" % (config_path,))
data = yaml.load(config_file)
base = os.path.dirname(os.path.realpath(config_path))
@@ -368,8 +369,11 @@
fn = os.path.join(base, fn)
fn = os.path.expanduser(fn)
with open(fn) as config_file:
+ self.log.info("Loading configuration from %s" % (fn,))
incdata = yaml.load(config_file)
extend_dict(tenant_config, incdata)
+ incdata = self._parseTenantInRepoLayouts(conf_tenant)
+ extend_dict(tenant_config, incdata)
tenant.layout = self._parseLayout(base, tenant_config, connections)
return abide
@@ -583,6 +587,39 @@
return layout
+ def _parseTenantInRepoLayouts(self, conf_tenant):
+ config = {}
+ jobs = []
+ for source_name, conf_source in conf_tenant.get('source', {}).items():
+ # TODOv3(jeblair,jhesketh): sources should just be
+ # set up at the start of the zuul.conf parsing
+ if source_name not in self.sources:
+ self.sources[source_name] = self._getSourceDriver(
+ source_name)
+ for conf_repo in conf_source.get('repos'):
+ source = self.sources[source_name]
+ project = source.getProject(conf_repo)
+ url = source.getGitUrl(project)
+ # TODOv3(jeblair): config should be branch specific
+ job = self.merger.getFiles(project.name, url, 'master',
+ files=['.zuul.yaml'])
+ job.project = project
+ jobs.append(job)
+ for job in jobs:
+ self.log.debug("Waiting for cat job %s" % (job,))
+ job.wait()
+ if job.files.get('.zuul.yaml'):
+ self.log.info("Loading configuration from %s/.zuul.yaml" %
+ (job.project,))
+ incdata = self._parseInRepoLayout(job.files['.zuul.yaml'])
+ extend_dict(config, incdata)
+ return config
+
+ def _parseInRepoLayout(self, data):
+ # TODOv3(jeblair): this should implement some rules to protect
+ # aspects of the config that should not be changed in-repo
+ return yaml.load(data)
+
def setLauncher(self, launcher):
self.launcher = launcher