Merge "Fix misleading message about statsd not installed" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index a87c196..7473ad3 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -65,7 +65,5 @@
- zuul-stream-functional
post:
jobs:
- - publish-openstack-sphinx-docs-infra:
- vars:
- sphinx_python: python3
+ - publish-openstack-sphinx-docs-infra-python3
- publish-openstack-python-branch-tarball
diff --git a/bindep.txt b/bindep.txt
index 85254b4..3dcc3e7 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -8,7 +8,7 @@
zookeeperd [platform:dpkg]
build-essential [platform:dpkg]
gcc [platform:rpm]
-graphviz [test]
+graphviz [doc]
libssl-dev [platform:dpkg]
openssl-devel [platform:rpm]
libffi-dev [platform:dpkg]
diff --git a/doc/source/admin/connections.rst b/doc/source/admin/connections.rst
index 29ca3be..55ac629 100644
--- a/doc/source/admin/connections.rst
+++ b/doc/source/admin/connections.rst
@@ -55,6 +55,7 @@
drivers/gerrit
drivers/github
+ drivers/git
drivers/smtp
drivers/sql
drivers/timer
diff --git a/doc/source/admin/drivers/git.rst b/doc/source/admin/drivers/git.rst
new file mode 100644
index 0000000..e0acec1
--- /dev/null
+++ b/doc/source/admin/drivers/git.rst
@@ -0,0 +1,59 @@
+:title: Git Driver
+
+Git
+===
+
+This driver can be used to load Zuul configuration from public Git repositories,
+for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by
+any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events
+in a pipeline.
+
+Connection Configuration
+------------------------
+
+The supported options in ``zuul.conf`` connections are:
+
+.. attr:: <git connection>
+
+ .. attr:: driver
+ :required:
+
+ .. value:: git
+
+ The connection must set ``driver=git`` for Git connections.
+
+ .. attr:: baseurl
+
+ Path to the base Git URL. Git repos name will be appended to it.
+
+ .. attr:: poll_delay
+ :default: 7200
+
+ The delay in seconds of the Git repositories polling loop.
+
+Trigger Configuration
+---------------------
+
+.. attr:: pipeline.trigger.<git source>
+
+ The dictionary passed to the Git pipeline ``trigger`` attribute
+ supports the following attributes:
+
+ .. attr:: event
+ :required:
+
+ Only ``ref-updated`` is supported.
+
+ .. attr:: ref
+
+ On ref-updated events, a ref such as ``refs/heads/master`` or
+ ``^refs/tags/.*$``. This field is treated as a regular expression,
+ and multiple refs may be listed.
+
+ .. attr:: ignore-deletes
+ :default: true
+
+ When a ref is deleted, a ref-updated event is emitted with a
+ newrev of all zeros specified. The ``ignore-deletes`` field is a
+ boolean value that describes whether or not these newrevs
+ trigger ref-updated events.
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 916e66a..fff673b 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -1032,11 +1032,12 @@
The following attributes may appear in a project:
.. attr:: name
- :required:
The name of the project. If Zuul is configured with two or more
unique projects with the same name, the canonical hostname for
the project should be included (e.g., `git.example.com/foo`).
+ If not given it is implicitly derived from the project where this
+ is defined.
.. attr:: templates
diff --git a/tests/base.py b/tests/base.py
index 69d9f55..7e63129 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1432,7 +1432,8 @@
self.log.debug("hostlist")
hosts = super(RecordingAnsibleJob, self).getHostList(args)
for host in hosts:
- host['host_vars']['ansible_connection'] = 'local'
+ if not host['host_vars'].get('ansible_connection'):
+ host['host_vars']['ansible_connection'] = 'local'
hosts.append(dict(
name=['localhost'],
@@ -1738,6 +1739,9 @@
executor='fake-nodepool')
if 'fakeuser' in node_type:
data['username'] = 'fakeuser'
+ if 'windows' in node_type:
+ data['connection_type'] = 'winrm'
+
data = json.dumps(data).encode('utf8')
path = self.client.create(path, data,
makepath=True,
@@ -2833,6 +2837,16 @@
os.path.join(FIXTURE_DIR, f.name))
self.setupAllProjectKeys()
+ def addTagToRepo(self, project, name, sha):
+ path = os.path.join(self.upstream_root, project)
+ repo = git.Repo(path)
+ repo.git.tag(name, sha)
+
+ def delTagFromRepo(self, project, name):
+ path = os.path.join(self.upstream_root, project)
+ repo = git.Repo(path)
+ repo.git.tag('-d', name)
+
def addCommitToRepo(self, project, message, files,
branch='master', tag=None):
path = os.path.join(self.upstream_root, project)
diff --git a/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
index 784b5f2..53fc210 100644
--- a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml
@@ -19,6 +19,10 @@
name: project-test1
run: playbooks/project-test1.yaml
+- job:
+ name: project-test2
+ run: playbooks/project-test2.yaml
+
- project:
name: org/project
check:
diff --git a/tests/fixtures/config/implicit-project/git/common-config/playbooks/test-common.yaml b/tests/fixtures/config/implicit-project/git/common-config/playbooks/test-common.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-project/git/common-config/playbooks/test-common.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-project/git/common-config/zuul.yaml b/tests/fixtures/config/implicit-project/git/common-config/zuul.yaml
new file mode 100644
index 0000000..038c412
--- /dev/null
+++ b/tests/fixtures/config/implicit-project/git/common-config/zuul.yaml
@@ -0,0 +1,57 @@
+- pipeline:
+ name: check
+ manager: independent
+ post-review: true
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+ start:
+ gerrit:
+ Verified: 0
+ precedence: high
+
+
+- job:
+ name: base
+ parent: null
+
+- job:
+ name: test-common
+ run: playbooks/test-common.yaml
+
+- project:
+ check:
+ jobs:
+ - test-common
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - test-common
+ gate:
+ jobs:
+ - test-common
diff --git a/tests/fixtures/config/implicit-project/git/org_project/.zuul.yaml b/tests/fixtures/config/implicit-project/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..bce195c
--- /dev/null
+++ b/tests/fixtures/config/implicit-project/git/org_project/.zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+ name: test-project
+ run: playbooks/test-project.yaml
+
+- project:
+ check:
+ jobs:
+ - test-project
+ gate:
+ jobs:
+ - test-project
diff --git a/tests/fixtures/config/implicit-project/git/org_project/playbooks/test-project.yaml b/tests/fixtures/config/implicit-project/git/org_project/playbooks/test-project.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/implicit-project/git/org_project/playbooks/test-project.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/implicit-project/main.yaml b/tests/fixtures/config/implicit-project/main.yaml
new file mode 100644
index 0000000..208e274
--- /dev/null
+++ b/tests/fixtures/config/implicit-project/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index ad530a7..36789a3 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -38,6 +38,8 @@
label: default-label
- name: fakeuser
label: fakeuser-label
+ - name: windows
+ label: windows-label
- job:
name: base
diff --git a/tests/fixtures/layouts/basic-git.yaml b/tests/fixtures/layouts/basic-git.yaml
new file mode 100644
index 0000000..068d0a0
--- /dev/null
+++ b/tests/fixtures/layouts/basic-git.yaml
@@ -0,0 +1,37 @@
+- pipeline:
+ name: post
+ manager: independent
+ trigger:
+ git:
+ - event: ref-updated
+ ref: ^refs/heads/.*$
+
+- pipeline:
+ name: tag
+ manager: independent
+ trigger:
+ git:
+ - event: ref-updated
+ ref: ^refs/tags/.*$
+
+- job:
+ name: base
+ parent: null
+ run: playbooks/base.yaml
+
+- job:
+ name: post-job
+ run: playbooks/post-job.yaml
+
+- job:
+ name: tag-job
+ run: playbooks/post-job.yaml
+
+- project:
+ name: org/project
+ post:
+ jobs:
+ - post-job
+ tag:
+ jobs:
+ - tag-job
diff --git a/tests/fixtures/zuul-git-driver.conf b/tests/fixtures/zuul-git-driver.conf
index b24b0a1..23a2a62 100644
--- a/tests/fixtures/zuul-git-driver.conf
+++ b/tests/fixtures/zuul-git-driver.conf
@@ -21,6 +21,7 @@
[connection git]
driver=git
baseurl=""
+poll_delay=0.1
[connection outgoing_smtp]
driver=smtp
diff --git a/tests/unit/test_git_driver.py b/tests/unit/test_git_driver.py
index 1cfadf4..b9e6c6e 100644
--- a/tests/unit/test_git_driver.py
+++ b/tests/unit/test_git_driver.py
@@ -12,7 +12,12 @@
# License for the specific language governing permissions and limitations
# under the License.
-from tests.base import ZuulTestCase
+
+import os
+import time
+import yaml
+
+from tests.base import ZuulTestCase, simple_layout
class TestGitDriver(ZuulTestCase):
@@ -23,7 +28,7 @@
super(TestGitDriver, self).setup_config()
self.config.set('connection git', 'baseurl', self.upstream_root)
- def test_git_driver(self):
+ def test_basic(self):
tenant = self.sched.abide.tenants.get('tenant-one')
# Check that we have the git source for common-config and the
# gerrit source for the project.
@@ -40,3 +45,127 @@
self.waitUntilSettled()
self.assertEqual(len(self.history), 1)
self.assertEqual(A.reported, 1)
+
+ def test_config_refreshed(self):
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 1)
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(self.history[0].name, 'project-test1')
+
+ # Update zuul.yaml to force a tenant reconfiguration
+ path = os.path.join(self.upstream_root, 'common-config', 'zuul.yaml')
+ config = yaml.load(open(path, 'r').read())
+ change = {
+ 'name': 'org/project',
+ 'check': {
+ 'jobs': [
+ 'project-test2'
+ ]
+ }
+ }
+ config[4]['project'] = change
+ files = {'zuul.yaml': yaml.dump(config)}
+ self.addCommitToRepo(
+ 'common-config', 'Change zuul.yaml configuration', files)
+
+ # Let some time for the tenant reconfiguration to happen
+ time.sleep(2)
+ self.waitUntilSettled()
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 2)
+ self.assertEqual(A.reported, 1)
+ # We make sure the new job has run
+ self.assertEqual(self.history[1].name, 'project-test2')
+
+ # Let's stop the git Watcher to let us merge some changes commits
+ # We want to verify that config changes are detected for commits
+ # on the range oldrev..newrev
+ self.sched.connections.getSource('git').connection.w_pause = True
+ # Add a config change
+ change = {
+ 'name': 'org/project',
+ 'check': {
+ 'jobs': [
+ 'project-test1'
+ ]
+ }
+ }
+ config[4]['project'] = change
+ files = {'zuul.yaml': yaml.dump(config)}
+ self.addCommitToRepo(
+ 'common-config', 'Change zuul.yaml configuration', files)
+ # Add two other changes
+ self.addCommitToRepo(
+ 'common-config', 'Adding f1',
+ {'f1': "Content"})
+ self.addCommitToRepo(
+ 'common-config', 'Adding f2',
+ {'f2': "Content"})
+ # Restart the git watcher
+ self.sched.connections.getSource('git').connection.w_pause = False
+
+ # Let some time for the tenant reconfiguration to happen
+ time.sleep(2)
+ self.waitUntilSettled()
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 3)
+ self.assertEqual(A.reported, 1)
+ # We make sure the new job has run
+ self.assertEqual(self.history[2].name, 'project-test1')
+
+ def ensure_watcher_has_context(self):
+ # Make sure watcher have read initial refs shas
+ cnx = self.sched.connections.getSource('git').connection
+ delay = 0.1
+ max_delay = 1
+ while not cnx.projects_refs:
+ time.sleep(delay)
+ max_delay -= delay
+ if max_delay <= 0:
+ raise Exception("Timeout waiting for initial read")
+
+ @simple_layout('layouts/basic-git.yaml', driver='git')
+ def test_ref_updated_event(self):
+ self.ensure_watcher_has_context()
+ # Add a commit to trigger a ref-updated event
+ self.addCommitToRepo(
+ 'org/project', 'A change for ref-updated', {'f1': 'Content'})
+ # Let some time for the git watcher to detect the ref-update event
+ time.sleep(0.2)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 1)
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('post-job').result)
+
+ @simple_layout('layouts/basic-git.yaml', driver='git')
+ def test_ref_created(self):
+ self.ensure_watcher_has_context()
+ # Tag HEAD to trigger a ref-updated event
+ self.addTagToRepo(
+ 'org/project', 'atag', 'HEAD')
+ # Let some time for the git watcher to detect the ref-update event
+ time.sleep(0.2)
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 1)
+ self.assertEqual('SUCCESS',
+ self.getJobFromHistory('tag-job').result)
+
+ @simple_layout('layouts/basic-git.yaml', driver='git')
+ def test_ref_deleted(self):
+ self.ensure_watcher_has_context()
+ # Delete default tag init to trigger a ref-updated event
+ self.delTagFromRepo(
+ 'org/project', 'init')
+ # Let some time for the git watcher to detect the ref-update event
+ time.sleep(0.2)
+ self.waitUntilSettled()
+ # Make sure no job as run as ignore-delete is True by default
+ self.assertEqual(len(self.history), 0)
diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py
index 1c41f5f..be50447 100644
--- a/tests/unit/test_inventory.py
+++ b/tests/unit/test_inventory.py
@@ -119,5 +119,15 @@
self.assertEqual(
inventory['all']['hosts'][node_name]['ansible_user'], username)
+ # check if the nodes use the correct or no ansible_connection
+ if node_name == 'windows':
+ self.assertEqual(
+ inventory['all']['hosts'][node_name]['ansible_connection'],
+ 'winrm')
+ else:
+ self.assertEqual(
+ 'local',
+ inventory['all']['hosts'][node_name]['ansible_connection'])
+
self.executor_server.release()
self.waitUntilSettled()
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index aacc81e..6bbf098 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -6070,6 +6070,77 @@
self.assertEqual(B.reported, 1)
+class TestImplicitProject(ZuulTestCase):
+ tenant_config_file = 'config/implicit-project/main.yaml'
+
+ def test_implicit_project(self):
+ # config project should work with implicit project name
+ A = self.fake_gerrit.addFakeChange('common-config', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+
+ # untrusted project should work with implicit project name
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertEqual(B.reported, 1)
+ self.assertHistory([
+ dict(name='test-common', result='SUCCESS', changes='1,1'),
+ dict(name='test-common', result='SUCCESS', changes='2,1'),
+ dict(name='test-project', result='SUCCESS', changes='2,1'),
+ ], ordered=False)
+
+ # now test adding a further project in repo
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: test-project
+ run: playbooks/test-project.yaml
+ - job:
+ name: test2-project
+ run: playbooks/test-project.yaml
+
+ - project:
+ check:
+ jobs:
+ - test-project
+ gate:
+ jobs:
+ - test-project
+
+ - project:
+ check:
+ jobs:
+ - test2-project
+ gate:
+ jobs:
+ - test2-project
+
+ """)
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ C = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ C.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(C.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ # change C must be merged
+ self.assertEqual(C.data['status'], 'MERGED')
+ self.assertEqual(C.reported, 2)
+ self.assertHistory([
+ dict(name='test-common', result='SUCCESS', changes='1,1'),
+ dict(name='test-common', result='SUCCESS', changes='2,1'),
+ dict(name='test-project', result='SUCCESS', changes='2,1'),
+ dict(name='test-common', result='SUCCESS', changes='3,1'),
+ dict(name='test-project', result='SUCCESS', changes='3,1'),
+ dict(name='test2-project', result='SUCCESS', changes='3,1'),
+ ], ordered=False)
+
+
class TestSemaphoreInRepo(ZuulTestCase):
config_file = 'zuul-connections-gerrit-and-github.conf'
tenant_config_file = 'config/in-repo/main.yaml'
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 44aa966..2779e6e 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -543,11 +543,23 @@
name: project-test2
run: playbooks/project-test2.yaml
+ - job:
+ name: project-test3
+ run: playbooks/project-test2.yaml
+
+ # add a job by the short project name
- project:
name: org/project
tenant-one-gate:
jobs:
- project-test2
+
+ # add a job by the canonical project name
+ - project:
+ name: review.example.com/org/project
+ tenant-one-gate:
+ jobs:
+ - project-test3
""")
in_repo_playbook = textwrap.dedent(
@@ -569,7 +581,9 @@
self.assertIn('tenant-one-gate', A.messages[1],
"A should transit tenant-one gate")
self.assertHistory([
- dict(name='project-test2', result='SUCCESS', changes='1,1')])
+ dict(name='project-test2', result='SUCCESS', changes='1,1'),
+ dict(name='project-test3', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
self.fake_gerrit.addEvent(A.getChangeMergedEvent())
self.waitUntilSettled()
@@ -584,7 +598,10 @@
'SUCCESS')
self.assertHistory([
dict(name='project-test2', result='SUCCESS', changes='1,1'),
- dict(name='project-test2', result='SUCCESS', changes='2,1')])
+ dict(name='project-test3', result='SUCCESS', changes='1,1'),
+ dict(name='project-test2', result='SUCCESS', changes='2,1'),
+ dict(name='project-test3', result='SUCCESS', changes='2,1'),
+ ], ordered=False)
def test_dynamic_template(self):
# Tests that a project can't update a template in another
diff --git a/tools/github-debugging.py b/tools/github-debugging.py
new file mode 100644
index 0000000..171627a
--- /dev/null
+++ b/tools/github-debugging.py
@@ -0,0 +1,55 @@
+import github3
+import logging
+import time
+
+# This is a template with boilerplate code for debugging github issues
+
+# TODO: for real use override the following variables
+url = 'https://example.com'
+api_token = 'xxxx'
+org = 'org'
+project = 'project'
+pull_nr = 3
+
+
+# Send the logs to stderr as well
+stream_handler = logging.StreamHandler()
+
+
+logger_urllib3 = logging.getLogger('requests.packages.logger_urllib3')
+# logger_urllib3.addHandler(stream_handler)
+logger_urllib3.setLevel(logging.DEBUG)
+
+logger = logging.getLogger('github3')
+# logger.addHandler(stream_handler)
+logger.setLevel(logging.DEBUG)
+
+
+github = github3.GitHubEnterprise(url)
+
+
+# This is the currently broken cache adapter, enable or replace it to debug
+# caching
+
+# import cachecontrol
+# from cachecontrol.cache import DictCache
+# cache_adapter = cachecontrol.CacheControlAdapter(
+# DictCache(),
+# cache_etags=True)
+#
+# github.session.mount('http://', cache_adapter)
+# github.session.mount('https://', cache_adapter)
+
+
+github.login(token=api_token)
+
+i = 0
+while True:
+ pr = github.pull_request(org, project, pull_nr)
+ prdict = pr.as_dict()
+ issue = pr.issue()
+ labels = list(issue.labels())
+ print(labels)
+ i += 1
+ print(i)
+ time.sleep(1)
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 71c4ccc..3a7e9b9 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -852,7 +852,7 @@
def getSchema(self):
project = {
- vs.Required('name'): str,
+ 'name': str,
'description': str,
'templates': [str],
'merge-mode': vs.Any('merge', 'merge-resolve',
@@ -1228,8 +1228,8 @@
tenant.config_projects,
tenant.untrusted_projects,
cached, tenant)
- unparsed_config.extend(tenant.config_projects_config, tenant=tenant)
- unparsed_config.extend(tenant.untrusted_projects_config, tenant=tenant)
+ unparsed_config.extend(tenant.config_projects_config, tenant)
+ unparsed_config.extend(tenant.untrusted_projects_config, tenant)
tenant.layout = TenantParser._parseLayout(base, tenant,
unparsed_config,
scheduler,
@@ -1484,10 +1484,10 @@
(job.project,))
if job.config_project:
config_projects_config.extend(
- job.project.unparsed_config)
+ job.project.unparsed_config, tenant)
else:
untrusted_projects_config.extend(
- job.project.unparsed_config)
+ job.project.unparsed_config, tenant)
continue
TenantParser.log.debug("Waiting for cat job %s" % (job,))
job.wait()
@@ -1518,17 +1518,18 @@
branch = source_context.branch
if source_context.trusted:
incdata = TenantParser._parseConfigProjectLayout(
- job.files[fn], source_context)
- config_projects_config.extend(incdata)
+ job.files[fn], source_context, tenant)
+ config_projects_config.extend(incdata, tenant)
else:
incdata = TenantParser._parseUntrustedProjectLayout(
- job.files[fn], source_context)
- untrusted_projects_config.extend(incdata)
- new_project_unparsed_config[project].extend(incdata)
+ job.files[fn], source_context, tenant)
+ untrusted_projects_config.extend(incdata, tenant)
+ new_project_unparsed_config[project].extend(
+ incdata, tenant)
if branch in new_project_unparsed_branch_config.get(
project, {}):
new_project_unparsed_branch_config[project][branch].\
- extend(incdata)
+ extend(incdata, tenant)
# Now that we've sucessfully loaded all of the configuration,
# cache the unparsed data on the project objects.
for project, data in new_project_unparsed_config.items():
@@ -1540,18 +1541,18 @@
return config_projects_config, untrusted_projects_config
@staticmethod
- def _parseConfigProjectLayout(data, source_context):
+ def _parseConfigProjectLayout(data, source_context, tenant):
# This is the top-level configuration for a tenant.
config = model.UnparsedTenantConfig()
with early_configuration_exceptions(source_context):
- config.extend(safe_load_yaml(data, source_context))
+ config.extend(safe_load_yaml(data, source_context), tenant)
return config
@staticmethod
- def _parseUntrustedProjectLayout(data, source_context):
+ def _parseUntrustedProjectLayout(data, source_context, tenant):
config = model.UnparsedTenantConfig()
with early_configuration_exceptions(source_context):
- config.extend(safe_load_yaml(data, source_context))
+ config.extend(safe_load_yaml(data, source_context), tenant)
if config.pipelines:
with configuration_exceptions('pipeline', config.pipelines[0]):
raise PipelineNotPermittedError()
@@ -1753,7 +1754,7 @@
else:
incdata = project.unparsed_branch_config.get(branch)
if incdata:
- config.extend(incdata)
+ config.extend(incdata, tenant)
continue
# Otherwise, do not use the cached config (even if the
# files are empty as that likely means they were deleted).
@@ -1782,12 +1783,12 @@
if trusted:
incdata = TenantParser._parseConfigProjectLayout(
- data, source_context)
+ data, source_context, tenant)
else:
incdata = TenantParser._parseUntrustedProjectLayout(
- data, source_context)
+ data, source_context, tenant)
- config.extend(incdata)
+ config.extend(incdata, tenant)
def createDynamicLayout(self, tenant, files,
include_config_projects=False,
diff --git a/zuul/driver/git/__init__.py b/zuul/driver/git/__init__.py
index 0faa036..1fe43f6 100644
--- a/zuul/driver/git/__init__.py
+++ b/zuul/driver/git/__init__.py
@@ -15,6 +15,7 @@
from zuul.driver import Driver, ConnectionInterface, SourceInterface
from zuul.driver.git import gitconnection
from zuul.driver.git import gitsource
+from zuul.driver.git import gittrigger
class GitDriver(Driver, ConnectionInterface, SourceInterface):
@@ -23,9 +24,15 @@
def getConnection(self, name, config):
return gitconnection.GitConnection(self, name, config)
+ def getTrigger(self, connection, config=None):
+ return gittrigger.GitTrigger(self, connection, config)
+
def getSource(self, connection):
return gitsource.GitSource(self, connection)
+ def getTriggerSchema(self):
+ return gittrigger.getSchema()
+
def getRequireSchema(self):
return {}
diff --git a/zuul/driver/git/gitconnection.py b/zuul/driver/git/gitconnection.py
index f93824d..03b24ca 100644
--- a/zuul/driver/git/gitconnection.py
+++ b/zuul/driver/git/gitconnection.py
@@ -13,12 +13,119 @@
# License for the specific language governing permissions and limitations
# under the License.
+import os
+import git
+import time
import logging
import urllib
+import threading
import voluptuous as v
from zuul.connection import BaseConnection
+from zuul.driver.git.gitmodel import GitTriggerEvent, EMPTY_GIT_REF
+from zuul.model import Ref, Branch
+
+
+class GitWatcher(threading.Thread):
+ log = logging.getLogger("connection.git.GitWatcher")
+
+ def __init__(self, git_connection, baseurl, poll_delay):
+ threading.Thread.__init__(self)
+ self.daemon = True
+ self.git_connection = git_connection
+ self.baseurl = baseurl
+ self.poll_delay = poll_delay
+ self._stopped = False
+ self.projects_refs = self.git_connection.projects_refs
+
+ def compareRefs(self, project, refs):
+ partial_events = []
+ # Fetch previous refs state
+ base_refs = self.projects_refs.get(project)
+ # Create list of created refs
+ rcreateds = set(refs.keys()) - set(base_refs.keys())
+ # Create list of deleted refs
+ rdeleteds = set(base_refs.keys()) - set(refs.keys())
+ # Create the list of updated refs
+ updateds = {}
+ for ref, sha in refs.items():
+ if ref in base_refs and base_refs[ref] != sha:
+ updateds[ref] = sha
+ for ref in rcreateds:
+ event = {
+ 'ref': ref,
+ 'branch_created': True,
+ 'oldrev': EMPTY_GIT_REF,
+ 'newrev': refs[ref]
+ }
+ partial_events.append(event)
+ for ref in rdeleteds:
+ event = {
+ 'ref': ref,
+ 'branch_deleted': True,
+ 'oldrev': base_refs[ref],
+ 'newrev': EMPTY_GIT_REF
+ }
+ partial_events.append(event)
+ for ref, sha in updateds.items():
+ event = {
+ 'ref': ref,
+ 'branch_updated': True,
+ 'oldrev': base_refs[ref],
+ 'newrev': sha
+ }
+ partial_events.append(event)
+ events = []
+ for pevent in partial_events:
+ event = GitTriggerEvent()
+ event.type = 'ref-updated'
+ event.project_hostname = self.git_connection.canonical_hostname
+ event.project_name = project
+ for attr in ('ref', 'oldrev', 'newrev', 'branch_created',
+ 'branch_deleted', 'branch_updated'):
+ if attr in pevent:
+ setattr(event, attr, pevent[attr])
+ events.append(event)
+ return events
+
+ def _run(self):
+ self.log.debug("Walk through projects refs for connection: %s" %
+ self.git_connection.connection_name)
+ try:
+ for project in self.git_connection.projects:
+ refs = self.git_connection.lsRemote(project)
+ self.log.debug("Read refs %s for project %s" % (refs, project))
+ if not self.projects_refs.get(project):
+ # State for this project does not exist yet so add it.
+ # No event will be triggered in this loop as
+ # projects_refs['project'] and refs are equal
+ self.projects_refs[project] = refs
+ events = self.compareRefs(project, refs)
+ self.projects_refs[project] = refs
+ # Send events to the scheduler
+ for event in events:
+ self.log.debug("Handling event: %s" % event)
+ # Force changes cache update before passing
+ # the event to the scheduler
+ self.git_connection.getChange(event)
+ self.git_connection.logEvent(event)
+ # Pass the event to the scheduler
+ self.git_connection.sched.addEvent(event)
+ except Exception as e:
+ self.log.debug("Unexpected issue in _run loop: %s" % str(e))
+
+ def run(self):
+ while not self._stopped:
+ if not self.git_connection.w_pause:
+ self._run()
+ # Polling wait delay
+ else:
+ self.log.debug("Watcher is on pause")
+ time.sleep(self.poll_delay)
+
+ def stop(self):
+ self._stopped = True
class GitConnection(BaseConnection):
@@ -32,6 +139,8 @@
raise Exception('baseurl is required for git connections in '
'%s' % self.connection_name)
self.baseurl = self.connection_config.get('baseurl')
+ self.poll_timeout = float(
+ self.connection_config.get('poll_delay', 3600 * 2))
self.canonical_hostname = self.connection_config.get(
'canonical_hostname')
if not self.canonical_hostname:
@@ -40,7 +149,10 @@
self.canonical_hostname = r.hostname
else:
self.canonical_hostname = 'localhost'
+ self.w_pause = False
self.projects = {}
+ self.projects_refs = {}
+ self._change_cache = {}
def getProject(self, name):
return self.projects.get(name)
@@ -48,15 +160,97 @@
def addProject(self, project):
self.projects[project.name] = project
+ def getChangeFilesUpdated(self, project_name, branch, tosha):
+ job = self.sched.merger.getFilesChanges(
+ self.connection_name, project_name, branch, tosha)
+ self.log.debug("Waiting for fileschanges job %s" % job)
+ job.wait()
+ if not job.updated:
+ raise Exception("Fileschanges job %s failed" % job)
+ self.log.debug("Fileschanges job %s got changes on files %s" %
+ (job, job.files))
+ return job.files
+
+ def lsRemote(self, project):
+ refs = {}
+ client = git.cmd.Git()
+ output = client.ls_remote(
+ os.path.join(self.baseurl, project))
+ for line in output.splitlines():
+ sha, ref = line.split('\t')
+ if ref.startswith('refs/'):
+ refs[ref] = sha
+ return refs
+
+ def maintainCache(self, relevant):
+ remove = {}
+ for branch, refschange in self._change_cache.items():
+ for ref, change in refschange.items():
+ if change not in relevant:
+ remove.setdefault(branch, []).append(ref)
+ for branch, refs in remove.items():
+ for ref in refs:
+ del self._change_cache[branch][ref]
+ if not self._change_cache[branch]:
+ del self._change_cache[branch]
+
+ def getChange(self, event, refresh=False):
+ if event.ref and event.ref.startswith('refs/heads/'):
+ branch = event.ref[len('refs/heads/'):]
+ change = self._change_cache.get(branch, {}).get(event.newrev)
+ if change:
+ return change
+ project = self.getProject(event.project_name)
+ change = Branch(project)
+ change.branch = branch
+ for attr in ('ref', 'oldrev', 'newrev'):
+ setattr(change, attr, getattr(event, attr))
+ change.url = ""
+ change.files = self.getChangeFilesUpdated(
+ event.project_name, change.branch, event.oldrev)
+ self._change_cache.setdefault(branch, {})[event.newrev] = change
+ elif event.ref:
+ # catch-all ref (ie, not a branch or head)
+ project = self.getProject(event.project_name)
+ change = Ref(project)
+ for attr in ('ref', 'oldrev', 'newrev'):
+ setattr(change, attr, getattr(event, attr))
+ change.url = ""
+ else:
+ self.log.warning("Unable to get change for %s" % (event,))
+ change = None
+ return change
+
def getProjectBranches(self, project, tenant):
- # TODO(jeblair): implement; this will need to handle local or
- # remote git urls.
- return ['master']
+ refs = self.lsRemote(project.name)
+ branches = [ref[len('refs/heads/'):] for ref in
+ refs if ref.startswith('refs/heads/')]
+ return branches
def getGitUrl(self, project):
url = '%s/%s' % (self.baseurl, project.name)
return url
+ def onLoad(self):
+ self.log.debug("Starting Git Watcher")
+ self._start_watcher_thread()
+
+ def onStop(self):
+ self.log.debug("Stopping Git Watcher")
+ self._stop_watcher_thread()
+
+ def _stop_watcher_thread(self):
+ if self.watcher_thread:
+ self.watcher_thread.stop()
+ self.watcher_thread.join()
+
+ def _start_watcher_thread(self):
+ self.watcher_thread = GitWatcher(
+ self,
+ self.baseurl,
+ self.poll_timeout)
+ self.watcher_thread.start()
+
def getSchema():
git_connection = v.Any(str, v.Schema(dict))
diff --git a/zuul/driver/git/gitmodel.py b/zuul/driver/git/gitmodel.py
new file mode 100644
index 0000000..5d12b36
--- /dev/null
+++ b/zuul/driver/git/gitmodel.py
@@ -0,0 +1,86 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+
+from zuul.model import TriggerEvent
+from zuul.model import EventFilter
+
+
+EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
+
+
+class GitTriggerEvent(TriggerEvent):
+ """Incoming event from an external system."""
+
+ def __repr__(self):
+ ret = '<GitTriggerEvent %s %s' % (self.type,
+ self.project_name)
+
+ if self.branch:
+ ret += " %s" % self.branch
+ ret += " oldrev:%s" % self.oldrev
+ ret += " newrev:%s" % self.newrev
+ ret += '>'
+
+ return ret
+
+
+class GitEventFilter(EventFilter):
+ def __init__(self, trigger, types=[], refs=[],
+ ignore_deletes=True):
+
+ super().__init__(trigger)
+
+ self._refs = refs
+ self.types = types
+ self.refs = [re.compile(x) for x in refs]
+ self.ignore_deletes = ignore_deletes
+
+ def __repr__(self):
+ ret = '<GitEventFilter'
+
+ if self.types:
+ ret += ' types: %s' % ', '.join(self.types)
+ if self._refs:
+ ret += ' refs: %s' % ', '.join(self._refs)
+ if self.ignore_deletes:
+ ret += ' ignore_deletes: %s' % self.ignore_deletes
+ ret += '>'
+
+ return ret
+
+ def matches(self, event, change):
+ # event types are ORed
+ matches_type = False
+ for etype in self.types:
+ if etype == event.type:
+ matches_type = True
+ if self.types and not matches_type:
+ return False
+
+ # refs are ORed
+ matches_ref = False
+ if event.ref is not None:
+ for ref in self.refs:
+ if ref.match(event.ref):
+ matches_ref = True
+ if self.refs and not matches_ref:
+ return False
+ if self.ignore_deletes and event.newrev == EMPTY_GIT_REF:
+ # If the updated ref has an empty git sha (all 0s),
+ # then the ref is being deleted
+ return False
+
+ return True
diff --git a/zuul/driver/git/gitsource.py b/zuul/driver/git/gitsource.py
index 8d85c08..78ae04e 100644
--- a/zuul/driver/git/gitsource.py
+++ b/zuul/driver/git/gitsource.py
@@ -36,7 +36,7 @@
raise NotImplemented()
def getChange(self, event, refresh=False):
- raise NotImplemented()
+ return self.connection.getChange(event, refresh)
def getProject(self, name):
p = self.connection.getProject(name)
diff --git a/zuul/driver/git/gittrigger.py b/zuul/driver/git/gittrigger.py
new file mode 100644
index 0000000..2885230
--- /dev/null
+++ b/zuul/driver/git/gittrigger.py
@@ -0,0 +1,49 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+from zuul.trigger import BaseTrigger
+from zuul.driver.git.gitmodel import GitEventFilter
+from zuul.driver.util import scalar_or_list, to_list
+
+
+class GitTrigger(BaseTrigger):
+ name = 'git'
+ log = logging.getLogger("zuul.GitTrigger")
+
+ def getEventFilters(self, trigger_conf):
+ efilters = []
+ for trigger in to_list(trigger_conf):
+ f = GitEventFilter(
+ trigger=self,
+ types=to_list(trigger['event']),
+ refs=to_list(trigger.get('ref')),
+ ignore_deletes=trigger.get(
+ 'ignore-deletes', True)
+ )
+ efilters.append(f)
+
+ return efilters
+
+
+def getSchema():
+ git_trigger = {
+ v.Required('event'):
+ scalar_or_list(v.Any('ref-updated')),
+ 'ref': scalar_or_list(str),
+ 'ignore-deletes': bool,
+ }
+
+ return git_trigger
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index f987f47..62dd45c 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -24,6 +24,7 @@
import cachecontrol
from cachecontrol.cache import DictCache
+from cachecontrol.heuristics import BaseHeuristic
import iso8601
import jwt
import requests
@@ -137,7 +138,6 @@
"""Move events from GitHub into the scheduler"""
log = logging.getLogger("zuul.GithubEventConnector")
- delay = 10.0
def __init__(self, connection):
super(GithubEventConnector, self).__init__()
@@ -153,14 +153,6 @@
ts, json_body, event_type = self.connection.getEvent()
if self._stopped:
return
- # Github can produce inconsistent data immediately after an
- # event, So ensure that we do not deliver the event to Zuul
- # until at least a certain amount of time has passed. Note
- # that if we receive several events in succession, we will
- # only need to delay for the first event. In essence, Zuul
- # should always be a constant number of seconds behind Github.
- now = time.time()
- time.sleep(max((ts + self.delay) - now, 0.0))
# If there's any installation mapping information in the body then
# update the project mapping before any requests are made.
@@ -431,9 +423,26 @@
# NOTE(jamielennox): Better here would be to cache to memcache or file
# or something external - but zuul already sucks at restarting so in
# memory probably doesn't make this much worse.
+
+ # NOTE(tobiash): Unlike documented cachecontrol doesn't priorize
+ # the etag caching but doesn't even re-request until max-age was
+ # elapsed.
+ #
+ # Thus we need to add a custom caching heuristic which simply drops
+ # the cache-control header containing max-age. This way we force
+ # cachecontrol to only rely on the etag headers.
+ #
+ # http://cachecontrol.readthedocs.io/en/latest/etags.html
+ # http://cachecontrol.readthedocs.io/en/latest/custom_heuristics.html
+ class NoAgeHeuristic(BaseHeuristic):
+ def update_headers(self, response):
+ if 'cache-control' in response.headers:
+ del response.headers['cache-control']
+
self.cache_adapter = cachecontrol.CacheControlAdapter(
DictCache(),
- cache_etags=True)
+ cache_etags=True,
+ heuristic=NoAgeHeuristic())
# The regex is based on the connection host. We do not yet support
# cross-connection dependency gathering
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 7a93f89..5a710a6 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -931,6 +931,10 @@
if username:
host_vars['ansible_user'] = username
+ connection_type = node.get('connection_type')
+ if connection_type:
+ host_vars['ansible_connection'] = connection_type
+
host_keys = []
for key in node.get('host_keys'):
if port != 22:
@@ -1706,6 +1710,7 @@
self.merger_worker.registerFunction("merger:merge")
self.merger_worker.registerFunction("merger:cat")
self.merger_worker.registerFunction("merger:refstate")
+ self.merger_worker.registerFunction("merger:fileschanges")
def register_work(self):
if self._running:
@@ -1859,6 +1864,9 @@
elif job.name == 'merger:refstate':
self.log.debug("Got refstate job: %s" % job.unique)
self.refstate(job)
+ elif job.name == 'merger:fileschanges':
+ self.log.debug("Got fileschanges job: %s" % job.unique)
+ self.fileschanges(job)
else:
self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail()
@@ -1970,6 +1978,19 @@
files=files)
job.sendWorkComplete(json.dumps(result))
+ def fileschanges(self, job):
+ args = json.loads(job.arguments)
+ task = self.update(args['connection'], args['project'])
+ task.wait()
+ with self.merger_lock:
+ files = self.merger.getFilesChanges(
+ args['connection'], args['project'],
+ args['branch'],
+ args['tosha'])
+ result = dict(updated=True,
+ files=files)
+ job.sendWorkComplete(json.dumps(result))
+
def refstate(self, job):
args = json.loads(job.arguments)
with self.merger_lock:
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 2614e58..c89a6fb 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -131,6 +131,15 @@
job = self.submitJob('merger:cat', data, None, precedence)
return job
+ def getFilesChanges(self, connection_name, project_name, branch,
+ tosha=None, precedence=zuul.model.PRECEDENCE_HIGH):
+ data = dict(connection=connection_name,
+ project=project_name,
+ branch=branch,
+ tosha=tosha)
+ job = self.submitJob('merger:fileschanges', data, None, precedence)
+ return job
+
def onBuildCompleted(self, job):
data = getJobData(job)
merged = data.get('merged', False)
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 06ec4b2..bd4ca58 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -314,6 +314,18 @@
'utf-8')
return ret
+ def getFilesChanges(self, branch, tosha=None):
+ repo = self.createRepoObject()
+ files = set()
+ head = repo.heads[branch].commit
+ files.update(set(head.stats.files.keys()))
+ if tosha:
+ for cmt in head.iter_parents():
+ if cmt.hexsha == tosha:
+ break
+ files.update(set(cmt.stats.files.keys()))
+ return list(files)
+
def deleteRemote(self, remote):
repo = self.createRepoObject()
repo.delete_remote(repo.remotes[remote])
@@ -581,3 +593,8 @@
def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
repo = self.getRepo(connection_name, project_name)
return repo.getFiles(files, dirs, branch=branch)
+
+ def getFilesChanges(self, connection_name, project_name, branch,
+ tosha=None):
+ repo = self.getRepo(connection_name, project_name)
+ return repo.getFilesChanges(branch, tosha)
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index 576d41e..aa04fc2 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -81,6 +81,7 @@
self.worker.registerFunction("merger:merge")
self.worker.registerFunction("merger:cat")
self.worker.registerFunction("merger:refstate")
+ self.worker.registerFunction("merger:fileschanges")
def stop(self):
self.log.debug("Stopping")
@@ -117,6 +118,9 @@
elif job.name == 'merger:refstate':
self.log.debug("Got refstate job: %s" % job.unique)
self.refstate(job)
+ elif job.name == 'merger:fileschanges':
+ self.log.debug("Got fileschanges job: %s" % job.unique)
+ self.fileschanges(job)
else:
self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail()
@@ -158,3 +162,12 @@
result = dict(updated=True,
files=files)
job.sendWorkComplete(json.dumps(result))
+
+ def fileschanges(self, job):
+ args = json.loads(job.arguments)
+ self.merger.updateRepo(args['connection'], args['project'])
+ files = self.merger.getFilesChanges(
+ args['connection'], args['project'], args['branch'], args['tosha'])
+ result = dict(updated=True,
+ files=files)
+ job.sendWorkComplete(json.dumps(result))
diff --git a/zuul/model.py b/zuul/model.py
index 77770b7..cc2fea7 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -384,6 +384,7 @@
self.private_ipv4 = None
self.public_ipv6 = None
self.connection_port = 22
+ self.connection_type = None
self._keys = []
self.az = None
self.provider = None
@@ -2255,7 +2256,7 @@
class ProjectConfig(object):
- # Represents a project cofiguration
+ # Represents a project configuration
def __init__(self, name, source_context=None):
self.name = name
# If this is a template, it will have a source_context, but
@@ -2400,7 +2401,7 @@
r.semaphores = copy.deepcopy(self.semaphores)
return r
- def extend(self, conf, tenant=None):
+ def extend(self, conf, tenant):
if isinstance(conf, UnparsedTenantConfig):
self.pragmas.extend(conf.pragmas)
self.pipelines.extend(conf.pipelines)
@@ -2408,16 +2409,14 @@
self.project_templates.extend(conf.project_templates)
for k, v in conf.projects.items():
name = k
- # If we have the tenant add the projects to
- # the according canonical name instead of the given project
- # name. If it is not found, it's ok to add this to the given
- # name. We also don't need to throw the
+ # Add the projects to the according canonical name instead of
+ # the given project name. If it is not found, it's ok to add
+ # this to the given name. We also don't need to throw the
# ProjectNotFoundException here as semantic validation occurs
# later where it will fail then.
- if tenant is not None:
- trusted, project = tenant.getProject(k)
- if project is not None:
- name = project.canonical_name
+ trusted, project = tenant.getProject(k)
+ if project is not None:
+ name = project.canonical_name
self.projects.setdefault(name, []).extend(v)
self.nodesets.extend(conf.nodesets)
self.secrets.extend(conf.secrets)
@@ -2434,7 +2433,12 @@
raise ConfigItemMultipleKeysError()
key, value = list(item.items())[0]
if key == 'project':
- name = value['name']
+ name = value.get('name')
+ if not name:
+ # There is no name defined so implicitly add the name
+ # of the project where it is defined.
+ name = value['_source_context'].project.canonical_name
+ value['name'] = name
self.projects.setdefault(name, []).append(value)
elif key == 'job':
self.jobs.append(value)
@@ -2643,11 +2647,11 @@
repr(variant), change)
item.debug("Pipeline variant {variant} matched".format(
variant=repr(variant)), indent=2)
- else:
- self.log.debug("Pipeline variant %s did not match %s",
- repr(variant), change)
- item.debug("Pipeline variant {variant} did not match".format(
- variant=repr(variant)), indent=2)
+ else:
+ self.log.debug("Pipeline variant %s did not match %s",
+ repr(variant), change)
+ item.debug("Pipeline variant {variant} did not match".
+ format(variant=repr(variant)), indent=2)
if not matched:
# A change must match at least one project pipeline
# job variant.