Merge "Add driver-specific pipeline requirements" into feature/zuulv3
diff --git a/bindep.txt b/bindep.txt
index 6895444..5db144b 100644
--- a/bindep.txt
+++ b/bindep.txt
@@ -14,3 +14,4 @@
libffi-devel [platform:rpm]
python-dev [platform:dpkg]
python-devel [platform:rpm]
+bubblewrap [platform:rpm]
diff --git a/etc/status/public_html/index.html b/etc/status/public_html/index.html
index 97025a6..ca5bb56 100644
--- a/etc/status/public_html/index.html
+++ b/etc/status/public_html/index.html
@@ -30,8 +30,11 @@
<script src="jquery.zuul.js"></script>
<script src="zuul.app.js"></script>
<script>
+ // @license magnet:?xt=urn:btih:8e4f440f4c65981c5bf93c76d35135ba5064d8b7&dn=apache-2.0.txt
+Apache 2.0
zuul_build_dom(jQuery, '#zuul_container');
zuul_start(jQuery);
+ // @license-end
</script>
</body>
</html>
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index 5c69bd1..c7e23b2 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -1,5 +1,8 @@
// jquery plugin for Zuul status page
//
+// @licstart The following is the entire license notice for the
+// JavaScript code in this page.
+//
// Copyright 2012 OpenStack Foundation
// Copyright 2013 Timo Tijhof
// Copyright 2013 Wikimedia Foundation
@@ -16,6 +19,9 @@
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
+//
+// @licend The above is the entire license notice
+// for the JavaScript code in this page.
(function ($) {
'use strict';
diff --git a/etc/status/public_html/zuul.app.js b/etc/status/public_html/zuul.app.js
index 6321af8..ae950e8 100644
--- a/etc/status/public_html/zuul.app.js
+++ b/etc/status/public_html/zuul.app.js
@@ -1,5 +1,8 @@
// Client script for Zuul status page
//
+// @licstart The following is the entire license notice for the
+// JavaScript code in this page.
+//
// Copyright 2013 OpenStack Foundation
// Copyright 2013 Timo Tijhof
// Copyright 2013 Wikimedia Foundation
@@ -16,6 +19,9 @@
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
+//
+// @licend The above is the entire license notice
+// for the JavaScript code in this page.
/*exported zuul_build_dom, zuul_start */
diff --git a/tests/base.py b/tests/base.py
index 9bacf21..0105ffa 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -496,11 +496,6 @@
if cat != 'submit':
change.addApproval(cat, action[cat], username=self.user)
- # TODOv3(jeblair): can this be removed?
- if 'label' in action:
- parts = action['label'].split('=')
- change.addApproval(parts[0], parts[2], username=self.user)
-
change.messages.append(message)
if 'submit' in action:
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 34bd9cd..2bb61ee 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -69,7 +69,6 @@
- job:
name: project1-project2-integration
- queue-name: integration
- job:
name: project-testfile
diff --git a/tests/fixtures/layouts/ignore-dependencies.yaml b/tests/fixtures/layouts/ignore-dependencies.yaml
index 02aea36..86fe674 100644
--- a/tests/fixtures/layouts/ignore-dependencies.yaml
+++ b/tests/fixtures/layouts/ignore-dependencies.yaml
@@ -32,7 +32,6 @@
- job:
name: project1-project2-integration
- queue-name: integration
- project:
name: org/project1
diff --git a/tools/test-setup.sh b/tools/test-setup.sh
index 3bdedf5..d3697c9 100755
--- a/tools/test-setup.sh
+++ b/tools/test-setup.sh
@@ -35,3 +35,9 @@
SET default_storage_engine=MYISAM;
DROP DATABASE IF EXISTS openstack_citest;
CREATE DATABASE openstack_citest CHARACTER SET utf8;"
+
+# TODO(pabelanger): Move this into bindep after we figure out how to enable our
+# PPA.
+sudo add-apt-repository ppa:openstack-ci-core/bubblewrap
+sudo apt-get update
+sudo apt-get --assume-yes install bubblewrap
diff --git a/tox.ini b/tox.ini
index 174a496..6a50c6d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,6 +51,6 @@
[flake8]
# These are ignored intentionally in openstack-infra projects;
# please don't submit patches that solely correct them or enable them.
-ignore = E305,E125,E129,E402,H,F405,W503
+ignore = E305,E125,E129,E402,H,W503
show-source = True
exclude = .venv,.tox,dist,doc,build,*.egg
diff --git a/zuul/ansible/library/zuul_afs.py b/zuul/ansible/library/zuul_afs.py
index 3ba426b..710c15d 100644
--- a/zuul/ansible/library/zuul_afs.py
+++ b/zuul/ansible/library/zuul_afs.py
@@ -116,6 +116,7 @@
module.exit_json(changed=True, build_roots=output)
from ansible.module_utils.basic import * # noqa
+from ansible.module_utils.basic import AnsibleModule
if __name__ == '__main__':
main()
diff --git a/zuul/ansible/library/zuul_console.py b/zuul/ansible/library/zuul_console.py
index 1932cf9..b1dc2d9 100644
--- a/zuul/ansible/library/zuul_console.py
+++ b/zuul/ansible/library/zuul_console.py
@@ -17,8 +17,10 @@
import os
import sys
+import select
import socket
import threading
+import time
LOG_STREAM_FILE = '/tmp/console.log'
LOG_STREAM_PORT = 19885
@@ -181,6 +183,7 @@
s.run()
from ansible.module_utils.basic import * # noqa
+from ansible.module_utils.basic import AnsibleModule
if __name__ == '__main__':
main()
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
old mode 100644
new mode 100755
index f2a2612..d31c5b8
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -98,6 +98,6 @@
else:
logging.basicConfig(level=logging.DEBUG)
- def configure_connections(self):
+ def configure_connections(self, source_only=False):
self.connections = zuul.lib.connections.ConnectionRegistry()
- self.connections.configure(self.config)
+ self.connections.configure(self.config, source_only)
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
old mode 100644
new mode 100755
index 96ba4b3..1893f5a
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -106,7 +106,7 @@
server.send_command(server.args.command)
sys.exit(0)
- server.configure_connections()
+ server.configure_connections(source_only=True)
if server.config.has_option('executor', 'pidfile'):
pid_fn = os.path.expanduser(server.config.get('executor', 'pidfile'))
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
old mode 100644
new mode 100755
index 797a990..686f34a
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -77,7 +77,7 @@
server.parse_arguments()
server.read_config()
- server.configure_connections()
+ server.configure_connections(source_only=True)
if server.config.has_option('zuul', 'state_dir'):
state_dir = os.path.expanduser(server.config.get('zuul', 'state_dir'))
diff --git a/zuul/configloader.py b/zuul/configloader.py
index d95e861..1374e9b 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -231,7 +231,6 @@
job = {vs.Required('name'): str,
'parent': str,
- 'queue-name': str,
'failure-message': str,
'success-message': str,
'failure-url': str,
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 275c185..dcbc172 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -20,6 +20,7 @@
import time
from six.moves import queue as Queue
from six.moves import urllib
+from six.moves import shlex_quote
import paramiko
import logging
import pprint
@@ -172,11 +173,15 @@
self._stopped = False
def _read(self, fd):
- l = fd.readline()
- data = json.loads(l)
- self.log.debug("Received data from Gerrit event stream: \n%s" %
- pprint.pformat(data))
- self.gerrit_connection.addEvent(data)
+ while True:
+ l = fd.readline()
+ data = json.loads(l)
+ self.log.debug("Received data from Gerrit event stream: \n%s" %
+ pprint.pformat(data))
+ self.gerrit_connection.addEvent(data)
+ # Continue until all the lines received are consumed
+ if fd._pos == fd._realpos:
+ break
def _listen(self, stdout, stderr):
poll = select.poll()
@@ -615,7 +620,7 @@
def review(self, project, change, message, action={}):
cmd = 'gerrit review --project %s' % project
if message:
- cmd += ' --message "%s"' % message
+ cmd += ' --message %s' % shlex_quote(message)
for key, val in action.items():
if val is True:
cmd += ' --%s' % key
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 958d4d9..99d2a9c 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -219,6 +219,16 @@
self.condition.release()
+def _copy_ansible_files(python_module, target_dir):
+ library_path = os.path.dirname(os.path.abspath(python_module.__file__))
+ for fn in os.listdir(library_path):
+ full_path = os.path.join(library_path, fn)
+ if os.path.isdir(full_path):
+ shutil.copytree(full_path, os.path.join(target_dir, fn))
+ else:
+ shutil.copy(os.path.join(library_path, fn), target_dir)
+
+
class ExecutorServer(object):
log = logging.getLogger("zuul.ExecutorServer")
@@ -286,40 +296,10 @@
if not os.path.exists(self.lookup_dir):
os.makedirs(self.lookup_dir)
- library_path = os.path.dirname(os.path.abspath(
- zuul.ansible.library.__file__))
- for fn in os.listdir(library_path):
- full_path = os.path.join(library_path, fn)
- if os.path.isdir(full_path):
- shutil.copytree(full_path, os.path.join(self.library_dir, fn))
- else:
- shutil.copy(os.path.join(library_path, fn), self.library_dir)
- action_path = os.path.dirname(os.path.abspath(
- zuul.ansible.action.__file__))
- for fn in os.listdir(action_path):
- full_path = os.path.join(action_path, fn)
- if os.path.isdir(full_path):
- shutil.copytree(full_path, os.path.join(self.action_dir, fn))
- else:
- shutil.copy(full_path, self.action_dir)
-
- callback_path = os.path.dirname(os.path.abspath(
- zuul.ansible.callback.__file__))
- for fn in os.listdir(callback_path):
- full_path = os.path.join(callback_path, fn)
- if os.path.isdir(full_path):
- shutil.copytree(full_path, os.path.join(self.callback_dir, fn))
- else:
- shutil.copy(os.path.join(callback_path, fn), self.callback_dir)
-
- lookup_path = os.path.dirname(os.path.abspath(
- zuul.ansible.lookup.__file__))
- for fn in os.listdir(lookup_path):
- full_path = os.path.join(lookup_path, fn)
- if os.path.isdir(full_path):
- shutil.copytree(full_path, os.path.join(self.lookup_dir, fn))
- else:
- shutil.copy(os.path.join(lookup_path, fn), self.lookup_dir)
+ _copy_ansible_files(zuul.ansible.library, self.library_dir)
+ _copy_ansible_files(zuul.ansible.action, self.action_dir)
+ _copy_ansible_files(zuul.ansible.callback, self.callback_dir)
+ _copy_ansible_files(zuul.ansible.lookup, self.lookup_dir)
self.job_workers = {}
@@ -958,7 +938,7 @@
for item in self.getHostList(args):
inventory.write(item['name'])
for k, v in item['host_vars'].items():
- inventory.write(' %s=%s' % (k, v))
+ inventory.write(' %s="%s"' % (k, v))
inventory.write('\n')
for key in item['host_keys']:
keys.append(key)
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index f5cce7b..720299a 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -23,6 +23,7 @@
import zuul.driver.timer
import zuul.driver.sql
from zuul.connection import BaseConnection
+from zuul.driver import SourceInterface
class DefaultConnection(BaseConnection):
@@ -78,7 +79,7 @@
for driver in self.drivers.values():
driver.stop()
- def configure(self, config):
+ def configure(self, config, source_only=False):
# Register connections from the config
connections = {}
@@ -100,6 +101,13 @@
% (con_config['driver'], con_name))
driver = self.drivers[con_driver]
+
+ # The merger and the reporter only needs source driver.
+ # This makes sure Reporter like the SQLDriver are only created by
+ # the scheduler process
+ if source_only and not issubclass(driver, SourceInterface):
+ continue
+
connection = driver.getConnection(con_name, con_config)
connections[con_name] = connection
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 069cbf5..e164195 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -113,12 +113,6 @@
files=files)
self.submitJob('merger:merge', data, build_set, precedence)
- def updateRepo(self, connection_name, project_name, build_set,
- precedence=zuul.model.PRECEDENCE_NORMAL):
- data = dict(connection=connection_name,
- project=project_name)
- self.submitJob('merger:update', data, build_set, precedence)
-
def getFiles(self, connection_name, project_name, branch, files,
precedence=zuul.model.PRECEDENCE_HIGH):
data = dict(connection=connection_name,
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index 04fd03b..c09d7ba 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -67,7 +67,6 @@
def register(self):
self.worker.registerFunction("merger:merge")
- self.worker.registerFunction("merger:update")
self.worker.registerFunction("merger:cat")
def stop(self):
@@ -88,9 +87,6 @@
if job.name == 'merger:merge':
self.log.debug("Got merge job: %s" % job.unique)
self.merge(job)
- elif job.name == 'merger:update':
- self.log.debug("Got update job: %s" % job.unique)
- self.update(job)
elif job.name == 'merger:cat':
self.log.debug("Got cat job: %s" % job.unique)
self.cat(job)
@@ -119,13 +115,6 @@
result['commit'] = ret
job.sendWorkComplete(json.dumps(result))
- def update(self, job):
- args = json.loads(job.arguments)
- self.merger.updateRepo(args['connection'], args['project'])
- result = dict(updated=True,
- zuul_url=self.zuul_url)
- job.sendWorkComplete(json.dumps(result))
-
def cat(self, job):
args = json.loads(job.arguments)
self.merger.updateRepo(args['connection'], args['project'])