Merge "Re-enable test_merge_failure_reporters" into feature/zuulv3
diff --git a/doc/source/developer/datamodel.rst b/doc/source/developer/datamodel.rst
index acb8612..c4ff4a0 100644
--- a/doc/source/developer/datamodel.rst
+++ b/doc/source/developer/datamodel.rst
@@ -8,7 +8,7 @@
Pipelines have a configured
:py:class:`~zuul.manager.PipelineManager` which controlls how
-the :py:class:`Change <zuul.model.Changeish>` objects are enqueued and
+the :py:class:`Ref <zuul.model.Ref>` objects are enqueued and
processed.
There are currently two,
@@ -35,7 +35,7 @@
.. autoclass:: zuul.model.Build
The :py:class:`~zuul.manager.base.PipelineManager` enqueues each
-:py:class:`Change <zuul.model.Changeish>` into the
+:py:class:`Ref <zuul.model.Ref>` into the
:py:class:`~zuul.model.ChangeQueue` in a :py:class:`~zuul.model.QueueItem`.
.. autoclass:: zuul.model.QueueItem
diff --git a/tests/base.py b/tests/base.py
index d8f88b7..76d604f 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -605,21 +605,6 @@
def getPullRequestClosedEvent(self):
return self._getPullRequestEvent('closed')
- def getPushEvent(self, old_sha, ref='refs/heads/master'):
- name = 'push'
- data = {
- 'ref': ref,
- 'before': old_sha,
- 'after': self.head_sha,
- 'repository': {
- 'full_name': self.project
- },
- 'sender': {
- 'login': 'ghuser'
- }
- }
- return (name, data)
-
def addComment(self, message):
self.comments.append(message)
self._updateTimeStamp()
@@ -909,7 +894,8 @@
self.pull_requests.append(pull_request)
return pull_request
- def getPushEvent(self, project, ref, old_rev=None, new_rev=None):
+ def getPushEvent(self, project, ref, old_rev=None, new_rev=None,
+ added_files=[], removed_files=[], modified_files=[]):
if not old_rev:
old_rev = '00000000000000000000000000000000'
if not new_rev:
@@ -921,7 +907,14 @@
'after': new_rev,
'repository': {
'full_name': project
- }
+ },
+ 'commits': [
+ {
+ 'added': added_files,
+ 'removed': removed_files,
+ 'modified': modified_files
+ }
+ ]
}
return (name, data)
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 6cc010e..2013ee0 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -513,3 +513,18 @@
self.assertNotIn('merge', A.labels)
self.assertNotIn('merge', B.labels)
self.assertNotIn('merge', C.labels)
+
+ @simple_layout('layouts/basic-github.yaml', driver='github')
+ def test_push_event_reconfigure(self):
+ pevent = self.fake_github.getPushEvent(project='common-config',
+ ref='refs/heads/master',
+ modified_files=['zuul.yaml'])
+
+ # record previous tenant reconfiguration time, which may not be set
+ old = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
+ time.sleep(1)
+ self.fake_github.emitEvent(pevent)
+ self.waitUntilSettled()
+ new = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
+ # New timestamp should be greater than the old timestamp
+ self.assertLess(old, new)
diff --git a/tests/unit/test_push_reqs.py b/tests/unit/test_push_reqs.py
index 657d9b8..d3a1feb 100644
--- a/tests/unit/test_push_reqs.py
+++ b/tests/unit/test_push_reqs.py
@@ -28,7 +28,10 @@
# Create a github change, add a change and emit a push event
A = self.fake_github.openFakePullRequest('org/project1', 'master', 'A')
old_sha = A.head_sha
- self.fake_github.emitEvent(A.getPushEvent(old_sha))
+ pevent = self.fake_github.getPushEvent(project='org/project1',
+ ref='refs/heads/master',
+ old_rev=old_sha)
+ self.fake_github.emitEvent(pevent)
self.waitUntilSettled()
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 88bfc38..f78e8a4 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -820,12 +820,12 @@
for source_name, require_config in conf.get('require', {}).items():
source = connections.getSource(source_name)
- manager.changeish_filters.extend(
+ manager.ref_filters.extend(
source.getRequireFilters(require_config))
for source_name, reject_config in conf.get('reject', {}).items():
source = connections.getSource(source_name)
- manager.changeish_filters.extend(
+ manager.ref_filters.extend(
source.getRejectFilters(reject_config))
for trigger_name, trigger_config in conf.get('trigger').items():
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index a5e1f22..fa43e66 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -75,6 +75,14 @@
time.sleep(max((ts + self.delay) - now, 0.0))
event = GerritTriggerEvent()
event.type = data.get('type')
+ # This catches when a change is merged, as it could potentially
+ # have merged layout info which will need to be read in.
+ # Ideally this would be done with a refupdate event so as to catch
+ # directly pushed things as well as full changes being merged.
+ # But we do not yet get files changed data for pure refupdate events.
+ # TODO(jlk): handle refupdated events instead of just changes
+ if event.type == 'change-merged':
+ event.branch_updated = True
event.trigger_name = 'gerrit'
change = data.get('change')
event.project_hostname = self.connection.canonical_hostname
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 6a3c09e..659d88b 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -129,10 +129,12 @@
event.trigger_name = 'github'
event.project_name = base_repo.get('full_name')
event.type = 'push'
+ event.branch_updated = True
event.ref = body.get('ref')
event.oldrev = body.get('before')
event.newrev = body.get('after')
+ event.commits = body.get('commits')
ref_parts = event.ref.split('/') # ie, ['refs', 'heads', 'master']
@@ -490,6 +492,7 @@
change.newrev = event.newrev
change.url = self.getGitwebUrl(project, sha=event.newrev)
change.source_event = event
+ change.files = self.getPushedFileNames(event)
else:
change = Ref(project)
return change
@@ -728,6 +731,13 @@
pr = self.getPull(project, number)
return pr.get('head').get('sha') == sha
+ def getPushedFileNames(self, event):
+ files = set()
+ for c in event.commits:
+ for f in c.get('added') + c.get('modified') + c.get('removed'):
+ files.add(f)
+ return list(files)
+
def _ghTimestampToDate(self, timestamp):
return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 0f1d9a4..9514454 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -417,12 +417,13 @@
self.command_socket = commandsocket.CommandSocket(path)
ansible_dir = os.path.join(state_dir, 'ansible')
self.ansible_dir = ansible_dir
+ if os.path.exists(ansible_dir):
+ shutil.rmtree(ansible_dir)
zuul_dir = os.path.join(ansible_dir, 'zuul')
plugin_dir = os.path.join(zuul_dir, 'ansible')
- if not os.path.exists(plugin_dir):
- os.makedirs(plugin_dir)
+ os.makedirs(plugin_dir, mode=0o0755)
self.library_dir = os.path.join(plugin_dir, 'library')
self.action_dir = os.path.join(plugin_dir, 'action')
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 3728c73..c3958d7 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -47,7 +47,7 @@
self.sched = sched
self.pipeline = pipeline
self.event_filters = []
- self.changeish_filters = []
+ self.ref_filters = []
def __str__(self):
return "<%s %s>" % (self.__class__.__name__, self.pipeline.name)
@@ -55,7 +55,7 @@
def _postConfig(self, layout):
self.log.info("Configured Pipeline Manager %s" % self.pipeline.name)
self.log.info(" Requirements:")
- for f in self.changeish_filters:
+ for f in self.ref_filters:
self.log.info(" %s" % f)
self.log.info(" Events:")
for e in self.event_filters:
@@ -281,7 +281,7 @@
return False
if not ignore_requirements:
- for f in self.changeish_filters:
+ for f in self.ref_filters:
if f.connection_name != change.project.connection_name:
self.log.debug("Filter %s skipped for change %s due "
"to mismatched connections" % (f, change))
diff --git a/zuul/model.py b/zuul/model.py
index f336198..1bd077b 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1312,7 +1312,7 @@
def __init__(self, queue, change):
self.pipeline = queue.pipeline
self.queue = queue
- self.change = change # a changeish
+ self.change = change # a ref
self.build_sets = []
self.dequeued_needing_change = False
self.current_build_set = BuildSet(self)
@@ -1621,15 +1621,14 @@
return (result, url)
def formatJSON(self):
- changeish = self.change
ret = {}
ret['active'] = self.active
ret['live'] = self.live
- if hasattr(changeish, 'url') and changeish.url is not None:
- ret['url'] = changeish.url
+ if hasattr(self.change, 'url') and self.change.url is not None:
+ ret['url'] = self.change.url
else:
ret['url'] = None
- ret['id'] = changeish._id()
+ ret['id'] = self.change._id()
if self.item_ahead:
ret['item_ahead'] = self.item_ahead.change._id()
else:
@@ -1637,8 +1636,8 @@
ret['items_behind'] = [i.change._id() for i in self.items_behind]
ret['failing_reasons'] = self.current_build_set.failing_reasons
ret['zuul_ref'] = self.current_build_set.ref
- if changeish.project:
- ret['project'] = changeish.project.name
+ if self.change.project:
+ ret['project'] = self.change.project.name
else:
# For cross-project dependencies with the depends-on
# project not known to zuul, the project is None
@@ -1646,8 +1645,8 @@
ret['project'] = "Unknown Project"
ret['enqueue_time'] = int(self.enqueue_time * 1000)
ret['jobs'] = []
- if hasattr(changeish, 'owner'):
- ret['owner'] = changeish.owner
+ if hasattr(self.change, 'owner'):
+ ret['owner'] = self.change.owner
else:
ret['owner'] = None
max_remaining = 0
@@ -1710,20 +1709,19 @@
return ret
def formatStatus(self, indent=0, html=False):
- changeish = self.change
indent_str = ' ' * indent
ret = ''
- if html and hasattr(changeish, 'url') and changeish.url is not None:
+ if html and getattr(self.change, 'url', None) is not None:
ret += '%sProject %s change <a href="%s">%s</a>\n' % (
indent_str,
- changeish.project.name,
- changeish.url,
- changeish._id())
+ self.change.project.name,
+ self.change.url,
+ self.change._id())
else:
ret += '%sProject %s change %s based on %s\n' % (
indent_str,
- changeish.project.name,
- changeish._id(),
+ self.change.project.name,
+ self.change._id(),
self.item_ahead)
for job in self.getJobs():
build = self.current_build_set.getBuild(job.name)
@@ -1794,6 +1792,8 @@
self.oldrev = None
self.newrev = None
+ self.files = []
+
def getBasePath(self):
base_path = ''
if hasattr(self, 'ref'):
@@ -1836,6 +1836,8 @@
return set()
def updatesConfig(self):
+ if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
+ return True
return False
def getSafeAttributes(self):
@@ -1855,7 +1857,6 @@
self.patchset = None
self.refspec = None
- self.files = []
self.needs_changes = []
self.needed_by_changes = []
self.is_current_patchset = True
@@ -1903,11 +1904,6 @@
related.update(c.getRelatedChanges())
return related
- def updatesConfig(self):
- if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
- return True
- return False
-
def getSafeAttributes(self):
return Attributes(project=self.project,
number=self.number,
@@ -1921,6 +1917,7 @@
self.data = None
# common
self.type = None
+ self.branch_updated = False
# For management events (eg: enqueue / promote)
self.tenant_name = None
self.project_hostname = None
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 61f1e5f..a63d270 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -228,6 +228,7 @@
self.zuul_version = zuul_version.version_info.release_string()
self.last_reconfigured = None
+ self.tenant_last_reconfigured = {}
def stop(self):
self._stopped = True
@@ -590,6 +591,7 @@
trigger.postConfig(pipeline)
for reporter in pipeline.actions:
reporter.postConfig()
+ self.tenant_last_reconfigured[tenant.name] = int(time.time())
if self.statsd:
try:
for pipeline in tenant.layout.pipelines.values():
@@ -747,7 +749,7 @@
"source %s",
e.change, project.source)
continue
- if (event.type == 'change-merged' and
+ if (event.branch_updated and
hasattr(change, 'files') and
change.updatesConfig()):
# The change that just landed updates the config.