Merge "Expose buildset to the executor and url formatter" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index d8f88b7..76d604f 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -605,21 +605,6 @@
def getPullRequestClosedEvent(self):
return self._getPullRequestEvent('closed')
- def getPushEvent(self, old_sha, ref='refs/heads/master'):
- name = 'push'
- data = {
- 'ref': ref,
- 'before': old_sha,
- 'after': self.head_sha,
- 'repository': {
- 'full_name': self.project
- },
- 'sender': {
- 'login': 'ghuser'
- }
- }
- return (name, data)
-
def addComment(self, message):
self.comments.append(message)
self._updateTimeStamp()
@@ -909,7 +894,8 @@
self.pull_requests.append(pull_request)
return pull_request
- def getPushEvent(self, project, ref, old_rev=None, new_rev=None):
+ def getPushEvent(self, project, ref, old_rev=None, new_rev=None,
+ added_files=[], removed_files=[], modified_files=[]):
if not old_rev:
old_rev = '00000000000000000000000000000000'
if not new_rev:
@@ -921,7 +907,14 @@
'after': new_rev,
'repository': {
'full_name': project
- }
+ },
+ 'commits': [
+ {
+ 'added': added_files,
+ 'removed': removed_files,
+ 'modified': modified_files
+ }
+ ]
}
return (name, data)
diff --git a/tests/fixtures/config/roles/git/bare-role/tasks/main.yaml b/tests/fixtures/config/roles/git/bare-role/tasks/main.yaml
new file mode 100644
index 0000000..75943b1
--- /dev/null
+++ b/tests/fixtures/config/roles/git/bare-role/tasks/main.yaml
@@ -0,0 +1,3 @@
+- file:
+ path: "{{zuul._test.test_root}}/{{zuul.uuid}}.bare-role.flag"
+ state: touch
diff --git a/tests/fixtures/config/roles/git/common-config/playbooks/common-config-test.yaml b/tests/fixtures/config/roles/git/common-config/playbooks/common-config-test.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/roles/git/common-config/playbooks/common-config-test.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/roles/git/common-config/zuul.yaml b/tests/fixtures/config/roles/git/common-config/zuul.yaml
new file mode 100644
index 0000000..1fdaf2e
--- /dev/null
+++ b/tests/fixtures/config/roles/git/common-config/zuul.yaml
@@ -0,0 +1,42 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: tenant-one-gate
+ manager: dependent
+ success-message: Build succeeded (tenant-one-gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: common-config-test
+
+- project:
+ name: common-config
+ tenant-one-gate:
+ jobs:
+ - common-config-test
diff --git a/tests/fixtures/config/roles/git/org_project/.zuul.yaml b/tests/fixtures/config/roles/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..35c2153
--- /dev/null
+++ b/tests/fixtures/config/roles/git/org_project/.zuul.yaml
@@ -0,0 +1,10 @@
+- job:
+ name: project-test
+ roles:
+ - zuul: bare-role
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-test
diff --git a/tests/fixtures/config/roles/git/org_project/README b/tests/fixtures/config/roles/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/roles/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/roles/git/org_project/playbooks/project-test.yaml b/tests/fixtures/config/roles/git/org_project/playbooks/project-test.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/roles/git/org_project/playbooks/project-test.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/roles/main.yaml b/tests/fixtures/config/roles/main.yaml
new file mode 100644
index 0000000..9ccece9
--- /dev/null
+++ b/tests/fixtures/config/roles/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project
+ - bare-role
diff --git a/tests/fixtures/layout-merge-failure.yaml b/tests/fixtures/layout-merge-failure.yaml
deleted file mode 100644
index 72bc9c9..0000000
--- a/tests/fixtures/layout-merge-failure.yaml
+++ /dev/null
@@ -1,56 +0,0 @@
-pipelines:
- - name: check
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: patchset-created
- success:
- gerrit:
- verified: 1
- failure:
- gerrit:
- verified: -1
-
- - name: post
- manager: IndependentPipelineManager
- trigger:
- gerrit:
- - event: ref-updated
- ref: ^(?!refs/).*$
-
- - name: gate
- manager: DependentPipelineManager
- failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
- merge-failure-message: "The merge failed! For more information..."
- trigger:
- gerrit:
- - event: comment-added
- approval:
- - approved: 1
- success:
- gerrit:
- verified: 2
- submit: true
- failure:
- gerrit:
- verified: -2
- merge-failure:
- gerrit:
- verified: -1
- smtp:
- to: you@example.com
- start:
- gerrit:
- verified: 0
- precedence: high
-
-projects:
- - name: org/project
- check:
- - project-merge:
- - project-test1
- - project-test2
- gate:
- - project-merge:
- - project-test1
- - project-test2
diff --git a/tests/fixtures/layouts/merge-failure.yaml b/tests/fixtures/layouts/merge-failure.yaml
new file mode 100644
index 0000000..228963f
--- /dev/null
+++ b/tests/fixtures/layouts/merge-failure.yaml
@@ -0,0 +1,74 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: post
+ manager: independent
+ trigger:
+ gerrit:
+ - event: ref-updated
+ ref: ^(?!refs/).*$
+
+- pipeline:
+ name: gate
+ manager: dependent
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ merge-failure-message: "The merge failed! For more information..."
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ merge-failure:
+ gerrit:
+ verified: -1
+ smtp:
+ to: you@example.com
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: project-merge
+ hold-following-changes: true
+
+- job:
+ name: project-test1
+
+- job:
+ name: project-test2
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
+ gate:
+ jobs:
+ - project-merge
+ - project-test1:
+ dependencies: project-merge
+ - project-test2:
+ dependencies: project-merge
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 636abb4..4979087 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -524,3 +524,18 @@
self.assertNotIn('merge', A.labels)
self.assertNotIn('merge', B.labels)
self.assertNotIn('merge', C.labels)
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_push_event_reconfigure(self):
+ pevent = self.fake_github.getPushEvent(project='common-config',
+ ref='refs/heads/master',
+ modified_files=['zuul.yaml'])
+
+ # record previous tenant reconfiguration time, which may not be set
+ old = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
+ time.sleep(1)
+ self.fake_github.emitEvent(pevent)
+ self.waitUntilSettled()
+ new = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
+ # New timestamp should be greater than the old timestamp
+ self.assertLess(old, new)
diff --git a/tests/unit/test_push_reqs.py b/tests/unit/test_push_reqs.py
index 657d9b8..d3a1feb 100644
--- a/tests/unit/test_push_reqs.py
+++ b/tests/unit/test_push_reqs.py
@@ -28,7 +28,10 @@
# Create a github change, add a change and emit a push event
A = self.fake_github.openFakePullRequest('org/project1', 'master', 'A')
old_sha = A.head_sha
- self.fake_github.emitEvent(A.getPushEvent(old_sha))
+ pevent = self.fake_github.getPushEvent(project='org/project1',
+ ref='refs/heads/master',
+ old_rev=old_sha)
+ self.fake_github.emitEvent(pevent)
self.waitUntilSettled()
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 0ac42c1..de8246c 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -29,6 +29,7 @@
import testtools
import zuul.change_matcher
+from zuul.driver.gerrit import gerritreporter
import zuul.scheduler
import zuul.rpcclient
import zuul.model
@@ -3333,7 +3334,6 @@
self.assertEqual(queue.window_floor, 1)
self.assertEqual(C.data['status'], 'MERGED')
- @skip("Disabled for early v3 development")
def test_worker_update_metadata(self):
"Test if a worker can send back metadata about itself"
self.executor_server.hold_jobs_in_build = True
@@ -3343,17 +3343,17 @@
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
- self.assertEqual(len(self.executor.builds), 1)
+ self.assertEqual(len(self.executor_client.builds), 1)
self.log.debug('Current builds:')
- self.log.debug(self.executor.builds)
+ self.log.debug(self.executor_client.builds)
start = time.time()
while True:
if time.time() - start > 10:
raise Exception("Timeout waiting for gearman server to report "
+ "back to the client")
- build = list(self.executor.builds.values())[0]
+ build = list(self.executor_client.builds.values())[0]
if build.worker.name == "My Worker":
break
else:
@@ -3361,12 +3361,6 @@
self.log.debug(build)
self.assertEqual("My Worker", build.worker.name)
- self.assertEqual("localhost", build.worker.hostname)
- self.assertEqual(['127.0.0.1', '192.168.1.1'], build.worker.ips)
- self.assertEqual("zuul.example.org", build.worker.fqdn)
- self.assertEqual("FakeBuilder", build.worker.program)
- self.assertEqual("v1.1", build.worker.version)
- self.assertEqual({'something': 'else'}, build.worker.extra)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
@@ -3410,49 +3404,45 @@
self.assertEqual(0, len(A.messages))
- @skip("Disabled for early v3 development")
+ @simple_layout('layouts/merge-failure.yaml')
def test_merge_failure_reporters(self):
"""Check that the config is set up correctly"""
- self.updateConfigLayout(
- 'tests/fixtures/layout-merge-failure.yaml')
- self.sched.reconfigure(self.config)
- self.registerJobs()
-
+ tenant = self.sched.abide.tenants.get('tenant-one')
self.assertEqual(
"Merge Failed.\n\nThis change or one of its cross-repo "
"dependencies was unable to be automatically merged with the "
"current state of its repository. Please rebase the change and "
"upload a new patchset.",
- self.sched.layout.pipelines['check'].merge_failure_message)
+ tenant.layout.pipelines['check'].merge_failure_message)
self.assertEqual(
"The merge failed! For more information...",
- self.sched.layout.pipelines['gate'].merge_failure_message)
+ tenant.layout.pipelines['gate'].merge_failure_message)
self.assertEqual(
- len(self.sched.layout.pipelines['check'].merge_failure_actions), 1)
+ len(tenant.layout.pipelines['check'].merge_failure_actions), 1)
self.assertEqual(
- len(self.sched.layout.pipelines['gate'].merge_failure_actions), 2)
+ len(tenant.layout.pipelines['gate'].merge_failure_actions), 2)
self.assertTrue(isinstance(
- self.sched.layout.pipelines['check'].merge_failure_actions[0],
- zuul.reporter.gerrit.GerritReporter))
+ tenant.layout.pipelines['check'].merge_failure_actions[0],
+ gerritreporter.GerritReporter))
self.assertTrue(
(
- isinstance(self.sched.layout.pipelines['gate'].
+ isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[0],
- zuul.reporter.smtp.SMTPReporter) and
- isinstance(self.sched.layout.pipelines['gate'].
+ zuul.driver.smtp.smtpreporter.SMTPReporter) and
+ isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[1],
- zuul.reporter.gerrit.GerritReporter)
+ gerritreporter.GerritReporter)
) or (
- isinstance(self.sched.layout.pipelines['gate'].
+ isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[0],
- zuul.reporter.gerrit.GerritReporter) and
- isinstance(self.sched.layout.pipelines['gate'].
+ gerritreporter.GerritReporter) and
+ isinstance(tenant.layout.pipelines['gate'].
merge_failure_actions[1],
- zuul.reporter.smtp.SMTPReporter)
+ zuul.driver.smtp.smtpreporter.SMTPReporter)
)
)
@@ -3563,7 +3553,7 @@
self.assertEqual(False, job['retry'])
self.assertEqual('https://server/job/project-merge/0/',
job['url'])
- self.assertEqual(7, len(job['worker']))
+ self.assertEqual(2, len(job['worker']))
self.assertEqual(False, job['canceled'])
self.assertEqual(True, job['voting'])
self.assertIsNone(job['result'])
@@ -3868,9 +3858,12 @@
self.assertEqual(B.reported, 0)
self.assertEqual(len(self.history), 0)
- # Simulate change B being gated outside this layout
- self.fake_gerrit.addEvent(B.addApproval('approved', 1))
+ # Simulate change B being gated outside this layout Set the
+ # change merged before submitting the event so that when the
+ # event triggers a gerrit query to update the change, we get
+ # the information that it was merged.
B.setMerged()
+ self.fake_gerrit.addEvent(B.addApproval('approved', 1))
self.waitUntilSettled()
self.assertEqual(len(self.history), 0)
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 707515a..3854804 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -456,3 +456,21 @@
# Make sure it's the right length
self.assertEqual(4096, private_key.key_size)
+
+
+class TestRoles(ZuulTestCase):
+ tenant_config_file = 'config/roles/main.yaml'
+
+ def test_role(self):
+ # This exercises a proposed change to a role being checked out
+ # and used.
+ A = self.fake_gerrit.addFakeChange('bare-role', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['id'])
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
+ ])
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index 487096f..3f67a38 100644
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -284,22 +284,6 @@
'worker.hostname': {
'title': 'Worker Hostname'
},
- 'worker.ips': {
- 'title': 'Worker IPs',
- 'transform': self._format_list
- },
- 'worker.fqdn': {
- 'title': 'Worker Domain'
- },
- 'worker.program': {
- 'title': 'Worker Program'
- },
- 'worker.version': {
- 'title': 'Worker Version'
- },
- 'worker.extra': {
- 'title': 'Worker Extra'
- },
}
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index a5e1f22..fa43e66 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -75,6 +75,14 @@
time.sleep(max((ts + self.delay) - now, 0.0))
event = GerritTriggerEvent()
event.type = data.get('type')
+ # This catches when a change is merged, as it could potentially
+ # have merged layout info which will need to be read in.
+ # Ideally this would be done with a refupdate event so as to catch
+ # directly pushed things as well as full changes being merged.
+ # But we do not yet get files changed data for pure refupdate events.
+ # TODO(jlk): handle refupdated events instead of just changes
+ if event.type == 'change-merged':
+ event.branch_updated = True
event.trigger_name = 'gerrit'
change = data.get('change')
event.project_hostname = self.connection.canonical_hostname
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 6a3c09e..659d88b 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -129,10 +129,12 @@
event.trigger_name = 'github'
event.project_name = base_repo.get('full_name')
event.type = 'push'
+ event.branch_updated = True
event.ref = body.get('ref')
event.oldrev = body.get('before')
event.newrev = body.get('after')
+ event.commits = body.get('commits')
ref_parts = event.ref.split('/') # ie, ['refs', 'heads', 'master']
@@ -490,6 +492,7 @@
change.newrev = event.newrev
change.url = self.getGitwebUrl(project, sha=event.newrev)
change.source_event = event
+ change.files = self.getPushedFileNames(event)
else:
change = Ref(project)
return change
@@ -728,6 +731,13 @@
pr = self.getPull(project, number)
return pr.get('head').get('sha') == sha
+ def getPushedFileNames(self, event):
+ files = set()
+ for c in event.commits:
+ for f in c.get('added') + c.get('modified') + c.get('removed'):
+ files.add(f)
+ return list(files)
+
def _ghTimestampToDate(self, timestamp):
return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index f71bb92..9514454 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -804,17 +804,6 @@
'worker_name': 'My Worker',
}
- # TODOv3:
- # 'name': self.name,
- # 'manager': self.executor_server.hostname,
- # 'worker_name': 'My Worker',
- # 'worker_hostname': 'localhost',
- # 'worker_ips': ['127.0.0.1', '192.168.1.1'],
- # 'worker_fqdn': 'zuul.example.org',
- # 'worker_program': 'FakeBuilder',
- # 'worker_version': 'v1.1',
- # 'worker_extra': {'something': 'else'}
-
self.job.sendWorkData(json.dumps(data))
self.job.sendWorkStatus(0, 100)
diff --git a/zuul/model.py b/zuul/model.py
index 612aa6f..e504dca 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1108,21 +1108,11 @@
def __init__(self):
self.name = "Unknown"
self.hostname = None
- self.ips = []
- self.fqdn = None
- self.program = None
- self.version = None
- self.extra = {}
def updateFromData(self, data):
"""Update worker information if contained in the WORK_DATA response."""
self.name = data.get('worker_name', self.name)
self.hostname = data.get('worker_hostname', self.hostname)
- self.ips = data.get('worker_ips', self.ips)
- self.fqdn = data.get('worker_fqdn', self.fqdn)
- self.program = data.get('worker_program', self.program)
- self.version = data.get('worker_version', self.version)
- self.extra = data.get('worker_extra', self.extra)
def __repr__(self):
return '<Worker %s>' % self.name
@@ -1701,11 +1691,6 @@
worker = {
'name': build.worker.name,
'hostname': build.worker.hostname,
- 'ips': build.worker.ips,
- 'fqdn': build.worker.fqdn,
- 'program': build.worker.program,
- 'version': build.worker.version,
- 'extra': build.worker.extra
}
if remaining and remaining > max_remaining:
max_remaining = remaining
@@ -1821,6 +1806,8 @@
self.oldrev = None
self.newrev = None
+ self.files = []
+
def getBasePath(self):
base_path = ''
if hasattr(self, 'ref'):
@@ -1863,6 +1850,8 @@
return set()
def updatesConfig(self):
+ if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
+ return True
return False
def getSafeAttributes(self):
@@ -1882,7 +1871,6 @@
self.patchset = None
self.refspec = None
- self.files = []
self.needs_changes = []
self.needed_by_changes = []
self.is_current_patchset = True
@@ -1930,11 +1918,6 @@
related.update(c.getRelatedChanges())
return related
- def updatesConfig(self):
- if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
- return True
- return False
-
def getSafeAttributes(self):
return Attributes(project=self.project,
number=self.number,
@@ -1948,6 +1931,7 @@
self.data = None
# common
self.type = None
+ self.branch_updated = False
# For management events (eg: enqueue / promote)
self.tenant_name = None
self.project_hostname = None
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 61f1e5f..a63d270 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -228,6 +228,7 @@
self.zuul_version = zuul_version.version_info.release_string()
self.last_reconfigured = None
+ self.tenant_last_reconfigured = {}
def stop(self):
self._stopped = True
@@ -590,6 +591,7 @@
trigger.postConfig(pipeline)
for reporter in pipeline.actions:
reporter.postConfig()
+ self.tenant_last_reconfigured[tenant.name] = int(time.time())
if self.statsd:
try:
for pipeline in tenant.layout.pipelines.values():
@@ -747,7 +749,7 @@
"source %s",
e.change, project.source)
continue
- if (event.type == 'change-merged' and
+ if (event.branch_updated and
hasattr(change, 'files') and
change.updatesConfig()):
# The change that just landed updates the config.