Merge "Update merge status after merge:merge is submitted"
diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst
index b5b8d7b..d8d72e6 100644
--- a/doc/source/zuul.rst
+++ b/doc/source/zuul.rst
@@ -765,6 +765,12 @@
Boolean value (``true`` or ``false``) that indicates whatever
a job is voting or not. Default: ``true``.
+**tags (optional)**
+ A list of arbitrary strings which will be associated with the job.
+ Can be used by the parameter-function to alter behavior based on
+ their presence on a job. If the job name is a regular expression,
+ tags will accumulate on jobs that match.
+
**parameter-function (optional)**
Specifies a function that should be applied to the parameters before
the job is launched. The function should be defined in a python file
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index c63700a..9df44ce 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -490,10 +490,12 @@
$header_div.append($heading);
if (typeof pipeline.description === 'string') {
+ var descr = $('<small />')
+ $.each( pipeline.description.split(/\r?\n\r?\n/), function(index, descr_part){
+ descr.append($('<p />').text(descr_part));
+ });
$header_div.append(
- $('<p />').append(
- $('<small />').text(pipeline.description)
- )
+ $('<p />').append(descr)
);
}
return $header_div;
diff --git a/requirements.txt b/requirements.txt
index 01fd245..8388f0b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
pbr>=1.1.0
-argparse
PyYAML>=3.1.0
Paste
WebOb>=1.2.3
diff --git a/tests/base.py b/tests/base.py
index f3bfa4e..405caa0 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -620,6 +620,7 @@
BuildHistory(name=self.name, number=self.number,
result=result, changes=changes, node=self.node,
uuid=self.unique, description=self.description,
+ parameters=self.parameters,
pipeline=self.parameters['ZUUL_PIPELINE'])
)
diff --git a/tests/fixtures/layout-tags.yaml b/tests/fixtures/layout-tags.yaml
new file mode 100644
index 0000000..d5b8bf9
--- /dev/null
+++ b/tests/fixtures/layout-tags.yaml
@@ -0,0 +1,42 @@
+includes:
+ - python-file: tags_custom_functions.py
+
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+jobs:
+ - name: ^.*$
+ parameter-function: apply_tags
+ - name: ^.*-merge$
+ failure-message: Unable to merge change
+ hold-following-changes: true
+ tags: merge
+ - name: project1-merge
+ tags:
+ - project1
+ - extratag
+
+projects:
+ - name: org/project1
+ check:
+ - project1-merge:
+ - project1-test1
+ - project1-test2
+ - project1-project2-integration
+
+ - name: org/project2
+ check:
+ - project2-merge:
+ - project2-test1
+ - project2-test2
+ - project1-project2-integration
diff --git a/tests/fixtures/layout.yaml b/tests/fixtures/layout.yaml
index e8f035e..2e48ff1 100644
--- a/tests/fixtures/layout.yaml
+++ b/tests/fixtures/layout.yaml
@@ -107,6 +107,7 @@
- name: ^.*-merge$
failure-message: Unable to merge change
hold-following-changes: true
+ tags: merge
- name: nonvoting-project-test2
voting: false
- name: project-testfile
@@ -120,6 +121,10 @@
mutex: test-mutex
- name: mutex-two
mutex: test-mutex
+ - name: project1-merge
+ tags:
+ - project1
+ - extratag
project-templates:
- name: test-one-and-two
diff --git a/tests/fixtures/tags_custom_functions.py b/tests/fixtures/tags_custom_functions.py
new file mode 100644
index 0000000..67e7ef1
--- /dev/null
+++ b/tests/fixtures/tags_custom_functions.py
@@ -0,0 +1,2 @@
+def apply_tags(item, job, params):
+ params['BUILD_TAGS'] = ' '.join(sorted(job.tags))
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 81c2948..b585fea 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -693,8 +693,8 @@
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
- source = self.sched.layout.pipelines['gate'].source
- source.maintainCache([])
+ for connection in self.connections.values():
+ connection.maintainCache([])
self.worker.hold_jobs_in_build = True
A.addApproval('APRV', 1)
@@ -791,7 +791,6 @@
A.addApproval('APRV', 1)
a = source._getChange(1, 2, refresh=True)
self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
- source.maintainCache([])
def test_build_configuration(self):
"Test that zuul merges the right commits for testing"
@@ -2243,8 +2242,8 @@
headers = f.info()
self.assertIn('Content-Length', headers)
self.assertIn('Content-Type', headers)
- self.assertEqual(headers['Content-Type'],
- 'application/json; charset=UTF-8')
+ self.assertIsNotNone(re.match('^application/json(; charset=UTF-8)?$',
+ headers['Content-Type']))
self.assertIn('Access-Control-Allow-Origin', headers)
self.assertIn('Cache-Control', headers)
self.assertIn('Last-Modified', headers)
@@ -2609,6 +2608,53 @@
# Ensure the removed job was not included in the report.
self.assertNotIn('project1-project2-integration', A.messages[0])
+ def test_double_live_reconfiguration_shared_queue(self):
+ # This was a real-world regression. A change is added to
+ # gate; a reconfigure happens, a second change which depends
+ # on the first is added, and a second reconfiguration happens.
+ # Ensure that both changes merge.
+
+ # A failure may indicate incorrect caching or cleaning up of
+ # references during a reconfiguration.
+ self.worker.hold_jobs_in_build = True
+
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ B.setDependsOn(A, 1)
+ A.addApproval('CRVW', 2)
+ B.addApproval('CRVW', 2)
+
+ # Add the parent change.
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ self.worker.release('.*-merge')
+ self.waitUntilSettled()
+
+ # Reconfigure (with only one change in the pipeline).
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ # Add the child change.
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+ self.waitUntilSettled()
+ self.worker.release('.*-merge')
+ self.waitUntilSettled()
+
+ # Reconfigure (with both in the pipeline).
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.history), 8)
+
+ self.assertEqual(A.data['status'], 'MERGED')
+ self.assertEqual(A.reported, 2)
+ self.assertEqual(B.data['status'], 'MERGED')
+ self.assertEqual(B.reported, 2)
+
def test_live_reconfiguration_del_project(self):
# Test project deletion from layout
# while changes are enqueued
@@ -2747,6 +2793,25 @@
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
+ def test_tags(self):
+ "Test job tags"
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-tags.yaml')
+ self.sched.reconfigure(self.config)
+
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ results = {'project1-merge': 'extratag merge project1',
+ 'project2-merge': 'merge'}
+
+ for build in self.history:
+ self.assertEqual(results.get(build.name, ''),
+ build.parameters.get('BUILD_TAGS'))
+
def test_timer(self):
"Test that a periodic job is triggered"
self.worker.hold_jobs_in_build = True
@@ -3656,8 +3721,8 @@
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
- source = self.sched.layout.pipelines['gate'].source
- source.maintainCache([])
+ for connection in self.connections.values():
+ connection.maintainCache([])
self.worker.hold_jobs_in_build = True
B.addApproval('APRV', 1)
diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py
index 402528f..066b4db 100644
--- a/zuul/connection/__init__.py
+++ b/zuul/connection/__init__.py
@@ -62,3 +62,10 @@
def registerUse(self, what, instance):
self.attached_to[what].append(instance)
+
+ def maintainCache(self, relevant):
+ """Make cache contain relevant changes.
+
+ This lets the user supply a list of change objects that are
+ still in use. Anything in our cache that isn't in the supplied
+ list should be safe to remove from the cache."""
diff --git a/zuul/connection/gerrit.py b/zuul/connection/gerrit.py
index f8e5add..4671ff9 100644
--- a/zuul/connection/gerrit.py
+++ b/zuul/connection/gerrit.py
@@ -94,7 +94,7 @@
try:
event.account = data.get(accountfield_from_type[event.type])
except KeyError:
- self.log.error("Received unrecognized event type '%s' from Gerrit.\
+ self.log.warning("Received unrecognized event type '%s' from Gerrit.\
Can not get account information." % event.type)
event.account = None
diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py
index a01eed3..e1e8ac6 100644
--- a/zuul/layoutvalidator.py
+++ b/zuul/layoutvalidator.py
@@ -104,6 +104,7 @@
'hold-following-changes': bool,
'voting': bool,
'mutex': str,
+ 'tags': toList(str),
'parameter-function': str,
'branch': toList(str),
'files': toList(str),
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
index 0ac7f0f..f0235a6 100644
--- a/zuul/lib/cloner.py
+++ b/zuul/lib/cloner.py
@@ -70,9 +70,10 @@
# Check for a cached git repo first
git_cache = '%s/%s' % (self.cache_dir, project)
git_upstream = '%s/%s' % (self.git_url, project)
+ repo_is_cloned = os.path.exists(os.path.join(dest, '.git'))
if (self.cache_dir and
os.path.exists(git_cache) and
- not os.path.exists(dest)):
+ not repo_is_cloned):
# file:// tells git not to hard-link across repos
git_cache = 'file://%s' % git_cache
self.log.info("Creating repo %s from cache %s",
@@ -102,7 +103,14 @@
repo.fetchFrom(zuul_remote, ref)
self.log.debug("Fetched ref %s from %s", ref, project)
return True
- except (ValueError, GitCommandError):
+ except ValueError:
+ self.log.debug("Project %s in Zuul does not have ref %s",
+ project, ref)
+ return False
+ except GitCommandError as error:
+ # Bail out if fetch fails due to infrastructure reasons
+ if error.stderr.startswith('fatal: unable to access'):
+ raise
self.log.debug("Project %s in Zuul does not have ref %s",
project, ref)
return False
diff --git a/zuul/model.py b/zuul/model.py
index 75f727d..d2cf13b 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -444,6 +444,7 @@
self.failure_pattern = None
self.success_pattern = None
self.parameter_function = None
+ self.tags = set()
self.mutex = None
# A metajob should only supply values for attributes that have
# been explicitly provided, so avoid setting boolean defaults.
@@ -493,6 +494,11 @@
self.swift.update(other.swift)
if other.mutex:
self.mutex = other.mutex
+ # Tags are merged via a union rather than a destructive copy
+ # because they are intended to accumulate as metajobs are
+ # applied.
+ if other.tags:
+ self.tags = self.tags.union(other.tags)
# Only non-None values should be copied for boolean attributes.
if other.hold_following_changes is not None:
self.hold_following_changes = other.hold_following_changes
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 28b42d3..48bb5e3 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -527,6 +527,13 @@
m = config_job.get('mutex', None)
if m is not None:
job.mutex = m
+ tags = toList(config_job.get('tags'))
+ if tags:
+ # Tags are merged via a union rather than a
+ # destructive copy because they are intended to
+ # accumulate onto any previously applied tags from
+ # metajobs.
+ job.tags = job.tags.union(set(tags))
fname = config_job.get('parameter-function', None)
if fname:
func = config_env.get(fname, None)
@@ -841,7 +848,7 @@
"Exception while canceling build %s "
"for change %s" % (build, item.change))
self.layout = layout
- self.maintainTriggerCache()
+ self.maintainConnectionCache()
for trigger in self.triggers.values():
trigger.postConfig()
for pipeline in self.layout.pipelines.values():
@@ -971,16 +978,18 @@
finally:
self.run_handler_lock.release()
- def maintainTriggerCache(self):
+ def maintainConnectionCache(self):
relevant = set()
for pipeline in self.layout.pipelines.values():
- self.log.debug("Start maintain trigger cache for: %s" % pipeline)
+ self.log.debug("Gather relevant cache items for: %s" % pipeline)
for item in pipeline.getAllItems():
relevant.add(item.change)
relevant.update(item.change.getRelatedChanges())
- pipeline.source.maintainCache(relevant)
- self.log.debug("End maintain trigger cache for: %s" % pipeline)
- self.log.debug("Trigger cache size: %s" % len(relevant))
+ for connection in self.connections.values():
+ connection.maintainCache(relevant)
+ self.log.debug(
+ "End maintain connection cache for: %s" % connection)
+ self.log.debug("Connection cache size: %s" % len(relevant))
def process_event_queue(self):
self.log.debug("Fetching trigger event")
diff --git a/zuul/source/__init__.py b/zuul/source/__init__.py
index 25fe974..cb4501a 100644
--- a/zuul/source/__init__.py
+++ b/zuul/source/__init__.py
@@ -49,13 +49,6 @@
def canMerge(self, change, allow_needs):
"""Determine if change can merge."""
- def maintainCache(self, relevant):
- """Make cache contain relevant changes.
-
- This lets the user supply a list of change objects that are
- still in use. Anything in our cache that isn't in the supplied
- list should be safe to remove from the cache."""
-
def postConfig(self):
"""Called after configuration has been processed."""
diff --git a/zuul/source/gerrit.py b/zuul/source/gerrit.py
index f35ab73..eb8705d 100644
--- a/zuul/source/gerrit.py
+++ b/zuul/source/gerrit.py
@@ -319,6 +319,3 @@
def _getGitwebUrl(self, project, sha=None):
return self.connection.getGitwebUrl(project, sha)
-
- def maintainCache(self, relevant):
- self.connection.maintainCache(relevant)