Merge "Add inventory variables for checkouts" into feature/zuulv3
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index b20aba7..b3c2e44 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -601,6 +601,12 @@
Base URL on which the websocket service is exposed, if different
than the base URL of the web app.
+ .. attr:: static_cache_expiry
+ :default: 3600
+
+ The Cache-Control max-age response header value for static files served
+ by the zuul-web. Set to 0 during development to disable Cache-Control.
+
Operation
~~~~~~~~~
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index fa874a9..3ea20ab 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -658,8 +658,17 @@
* In the case of a job variant defined within a
:ref:`project-template`, if no branch specifier appears, the
- implied branch specifier for the :ref:`project` definition which
- uses the project-template will be used.
+ implied branch containing the project-template definition is
+ used as an implied branch specifier. This means that
+ definitions of the same project-template on different branches
+ may run different jobs.
+
+ When that project-template is used by a :ref:`project`
+ definition within a :term:`untrusted-project`, the branch
+ containing that project definition is combined with the branch
+ specifier of the project-template. This means it is possible
+ for a project to use a template on one branch, but not on
+ another.
This allows for the very simple and expected workflow where if a
project defines a job on the ``master`` branch with no branch
diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample
index 76494ad..f0e1765 100644
--- a/etc/zuul.conf-sample
+++ b/etc/zuul.conf-sample
@@ -37,6 +37,7 @@
[web]
listen_address=127.0.0.1
port=9000
+static_cache_expiry=0
[webapp]
listen_address=0.0.0.0
diff --git a/tests/base.py b/tests/base.py
index 176c535..036515d 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1713,6 +1713,8 @@
image_id=None,
host_keys=["fake-key1", "fake-key2"],
executor='fake-nodepool')
+ if 'fakeuser' in node_type:
+ data['username'] = 'fakeuser'
data = json.dumps(data).encode('utf8')
path = self.client.create(path, data,
makepath=True,
diff --git a/tests/fixtures/config/branch-templates/git/project-config/zuul.yaml b/tests/fixtures/config/branch-templates/git/project-config/zuul.yaml
new file mode 100644
index 0000000..ce08877
--- /dev/null
+++ b/tests/fixtures/config/branch-templates/git/project-config/zuul.yaml
@@ -0,0 +1,26 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- job:
+ name: base
+ parent: null
+
+- project:
+ name: project-config
+ check:
+ jobs: []
+
+- project:
+ name: puppet-integration
+ check:
+ jobs: []
diff --git a/tests/fixtures/config/branch-templates/git/puppet-integration/.zuul.yaml b/tests/fixtures/config/branch-templates/git/puppet-integration/.zuul.yaml
new file mode 100644
index 0000000..dfea632
--- /dev/null
+++ b/tests/fixtures/config/branch-templates/git/puppet-integration/.zuul.yaml
@@ -0,0 +1,25 @@
+- job:
+ name: puppet-unit-base
+ run: playbooks/run-unit-tests.yaml
+
+- job:
+ name: puppet-unit-3.8
+ parent: puppet-unit-base
+ branches: ^(stable/(newton|ocata)).*$
+ vars:
+ puppet_gem_version: 3.8
+
+- job:
+ name: puppet-something
+ run: playbooks/run-unit-tests.yaml
+
+- project-template:
+ name: puppet-unit
+ check:
+ jobs:
+ - puppet-unit-3.8
+
+- project:
+ name: puppet-integration
+ templates:
+ - puppet-unit
diff --git a/tests/fixtures/config/branch-templates/git/puppet-integration/README b/tests/fixtures/config/branch-templates/git/puppet-integration/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/branch-templates/git/puppet-integration/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/branch-templates/git/puppet-integration/playbooks/run-unit-tests.yaml b/tests/fixtures/config/branch-templates/git/puppet-integration/playbooks/run-unit-tests.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/branch-templates/git/puppet-integration/playbooks/run-unit-tests.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/branch-templates/git/puppet-tripleo/.zuul.yaml b/tests/fixtures/config/branch-templates/git/puppet-tripleo/.zuul.yaml
new file mode 100644
index 0000000..4be8146
--- /dev/null
+++ b/tests/fixtures/config/branch-templates/git/puppet-tripleo/.zuul.yaml
@@ -0,0 +1,4 @@
+- project:
+ name: puppet-tripleo
+ templates:
+ - puppet-unit
diff --git a/tests/fixtures/config/branch-templates/git/puppet-tripleo/README b/tests/fixtures/config/branch-templates/git/puppet-tripleo/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/branch-templates/git/puppet-tripleo/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/branch-templates/main.yaml b/tests/fixtures/config/branch-templates/main.yaml
new file mode 100644
index 0000000..f7677a3
--- /dev/null
+++ b/tests/fixtures/config/branch-templates/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - project-config
+ untrusted-projects:
+ - puppet-integration
+ - puppet-tripleo
diff --git a/tests/fixtures/config/inventory/git/common-config/playbooks/hostvars-inventory.yaml b/tests/fixtures/config/inventory/git/common-config/playbooks/hostvars-inventory.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/inventory/git/common-config/playbooks/hostvars-inventory.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index 900abd6..74ddf2d 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -31,6 +31,14 @@
- compute1
- compute2
+- nodeset:
+ name: nodeset2
+ nodes:
+ - name: default
+ label: default-label
+ - name: fakeuser
+ label: fakeuser-label
+
- job:
name: base
parent: null
@@ -47,3 +55,8 @@
name: group-inventory
nodeset: nodeset1
run: playbooks/group-inventory.yaml
+
+- job:
+ name: hostvars-inventory
+ run: playbooks/hostvars-inventory.yaml
+ nodeset: nodeset2
diff --git a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml
index 26310a0..1a8bf5d 100644
--- a/tests/fixtures/config/inventory/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/inventory/git/org_project/.zuul.yaml
@@ -4,3 +4,4 @@
jobs:
- single-inventory
- group-inventory
+ - hostvars-inventory
diff --git a/tests/fixtures/config/zuultrigger/project-change-merged/main.yaml b/tests/fixtures/config/zuultrigger/project-change-merged/main.yaml
index 9d01f54..208e274 100644
--- a/tests/fixtures/config/zuultrigger/project-change-merged/main.yaml
+++ b/tests/fixtures/config/zuultrigger/project-change-merged/main.yaml
@@ -4,3 +4,5 @@
gerrit:
config-projects:
- common-config
+ untrusted-projects:
+ - org/project
diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py
index 2835d30..04dcb05 100644
--- a/tests/unit/test_inventory.py
+++ b/tests/unit/test_inventory.py
@@ -80,3 +80,24 @@
self.executor_server.release()
self.waitUntilSettled()
+
+ def test_hostvars_inventory(self):
+
+ inventory = self._get_build_inventory('hostvars-inventory')
+
+ all_nodes = ('default', 'fakeuser')
+ self.assertIn('all', inventory)
+ self.assertIn('hosts', inventory['all'])
+ self.assertIn('vars', inventory['all'])
+ for node_name in all_nodes:
+ self.assertIn(node_name, inventory['all']['hosts'])
+ # check if the nodes use the correct username
+ if node_name == 'fakeuser':
+ username = 'fakeuser'
+ else:
+ username = 'zuul'
+ self.assertEqual(
+ inventory['all']['hosts'][node_name]['ansible_user'], username)
+
+ self.executor_server.release()
+ self.waitUntilSettled()
diff --git a/tests/unit/test_scheduler_cmd.py b/tests/unit/test_scheduler_cmd.py
deleted file mode 100644
index ee6200f..0000000
--- a/tests/unit/test_scheduler_cmd.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import os
-
-import testtools
-import zuul.cmd.scheduler
-
-from tests import base
-
-
-class TestSchedulerCmdArguments(testtools.TestCase):
-
- def setUp(self):
- super(TestSchedulerCmdArguments, self).setUp()
- self.app = zuul.cmd.scheduler.Scheduler()
-
- def test_test_config(self):
- conf_path = os.path.join(base.FIXTURE_DIR, 'zuul.conf')
- self.app.parse_arguments(['-t', '-c', conf_path])
- self.assertTrue(self.app.args.validate)
- self.app.read_config()
- self.assertEqual(0, self.app.test_config())
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index c04604d..e2da808 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -157,6 +157,115 @@
self.assertIn('Unable to modify final job', A.messages[0])
+class TestBranchTemplates(ZuulTestCase):
+ tenant_config_file = 'config/branch-templates/main.yaml'
+
+ def test_template_removal_from_branch(self):
+ # Test that a template can be removed from one branch but not
+ # another.
+ # This creates a new branch with a copy of the config in master
+ self.create_branch('puppet-integration', 'stable/newton')
+ self.create_branch('puppet-integration', 'stable/ocata')
+ self.create_branch('puppet-tripleo', 'stable/newton')
+ self.create_branch('puppet-tripleo', 'stable/ocata')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-integration', 'stable/newton'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-integration', 'stable/ocata'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-tripleo', 'stable/newton'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-tripleo', 'stable/ocata'))
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - project:
+ name: puppet-tripleo
+ check:
+ jobs:
+ - puppet-something
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('puppet-tripleo', 'stable/newton',
+ 'A', files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='puppet-something', result='SUCCESS', changes='1,1')])
+
+ def test_template_change_on_branch(self):
+ # Test that the contents of a template can be changed on one
+ # branch without affecting another.
+
+ # This creates a new branch with a copy of the config in master
+ self.create_branch('puppet-integration', 'stable/newton')
+ self.create_branch('puppet-integration', 'stable/ocata')
+ self.create_branch('puppet-tripleo', 'stable/newton')
+ self.create_branch('puppet-tripleo', 'stable/ocata')
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-integration', 'stable/newton'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-integration', 'stable/ocata'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-tripleo', 'stable/newton'))
+ self.fake_gerrit.addEvent(
+ self.fake_gerrit.getFakeBranchCreatedEvent(
+ 'puppet-tripleo', 'stable/ocata'))
+ self.waitUntilSettled()
+
+ in_repo_conf = textwrap.dedent("""
+ - job:
+ name: puppet-unit-base
+ run: playbooks/run-unit-tests.yaml
+
+ - job:
+ name: puppet-unit-3.8
+ parent: puppet-unit-base
+ branches: ^(stable/(newton|ocata)).*$
+ vars:
+ puppet_gem_version: 3.8
+
+ - job:
+ name: puppet-something
+ run: playbooks/run-unit-tests.yaml
+
+ - project-template:
+ name: puppet-unit
+ check:
+ jobs:
+ - puppet-something
+
+ - project:
+ name: puppet-integration
+ templates:
+ - puppet-unit
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('puppet-integration',
+ 'stable/newton',
+ 'A', files=file_dict)
+ B = self.fake_gerrit.addFakeChange('puppet-tripleo',
+ 'stable/newton',
+ 'B')
+ B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
+ B.subject, A.data['id'])
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertHistory([
+ dict(name='puppet-something', result='SUCCESS',
+ changes='1,1 2,1')])
+
+
class TestBranchVariants(ZuulTestCase):
tenant_config_file = 'config/branch-variants/main.yaml'
diff --git a/tests/unit/test_zuultrigger.py b/tests/unit/test_zuultrigger.py
index 3c4dead..3954a21 100644
--- a/tests/unit/test_zuultrigger.py
+++ b/tests/unit/test_zuultrigger.py
@@ -64,9 +64,6 @@
class TestZuulTriggerProjectChangeMerged(ZuulTestCase):
- def setUp(self):
- self.skip("Disabled because v3 noop job does not perform merge")
-
tenant_config_file = 'config/zuultrigger/project-change-merged/main.yaml'
def test_zuul_trigger_project_change_merged(self):
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index 86f7f12..e150f9c 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -14,7 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
+import argparse
import configparser
+import daemon
import extras
import io
import logging
@@ -28,8 +30,13 @@
yappi = extras.try_import('yappi')
objgraph = extras.try_import('objgraph')
+# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
+# instead it depends on lockfile-0.9.1 which uses pidfile.
+pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
+
from zuul.ansible import logconfig
import zuul.lib.connections
+from zuul.lib.config import get_default
# Do not import modules that will pull in paramiko which must not be
# imported until after the daemonization.
@@ -87,6 +94,8 @@
class ZuulApp(object):
+ app_name = None # type: str
+ app_description = None # type: str
def __init__(self):
self.args = None
@@ -97,7 +106,21 @@
from zuul.version import version_info as zuul_version_info
return "Zuul version: %s" % zuul_version_info.release_string()
- def read_config(self):
+ def createParser(self):
+ parser = argparse.ArgumentParser(description=self.app_description)
+ parser.add_argument('-c', dest='config',
+ help='specify the config file')
+ parser.add_argument('--version', dest='version', action='version',
+ version=self._get_version(),
+ help='show zuul version')
+ return parser
+
+ def parseArguments(self, args=None):
+ parser = self.createParser()
+ self.args = parser.parse_args(args)
+ return parser
+
+ def readConfig(self):
self.config = configparser.ConfigParser()
if self.args.config:
locations = [self.args.config]
@@ -130,3 +153,34 @@
def configure_connections(self, source_only=False):
self.connections = zuul.lib.connections.ConnectionRegistry()
self.connections.configure(self.config, source_only)
+
+
+class ZuulDaemonApp(ZuulApp):
+ def createParser(self):
+ parser = super(ZuulDaemonApp, self).createParser()
+ parser.add_argument('-d', dest='nodaemon', action='store_true',
+ help='do not run as a daemon')
+ return parser
+
+ def getPidFile(self):
+ pid_fn = get_default(self.config, self.app_name, 'pidfile',
+ '/var/run/zuul/%s.pid' % self.app_name,
+ expand_user=True)
+ return pid_fn
+
+ def main(self):
+ self.parseArguments()
+ self.readConfig()
+
+ pid_fn = self.getPidFile()
+ pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
+
+ if self.args.nodaemon:
+ self.run()
+ else:
+ # Exercise the pidfile before we do anything else (including
+ # logging or daemonizing)
+ with daemon.DaemonContext(pidfile=pid):
+ pass
+ with daemon.DaemonContext(pidfile=pid):
+ self.run()
diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py
index 7a26a62..ebf59b9 100755
--- a/zuul/cmd/client.py
+++ b/zuul/cmd/client.py
@@ -30,18 +30,14 @@
class Client(zuul.cmd.ZuulApp):
+ app_name = 'zuul'
+ app_description = 'Zuul RPC client.'
log = logging.getLogger("zuul.Client")
- def parse_arguments(self):
- parser = argparse.ArgumentParser(
- description='Zuul Project Gating System Client.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
+ def createParser(self):
+ parser = super(Client, self).createParser()
parser.add_argument('-v', dest='verbose', action='store_true',
help='verbose output')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
subparsers = parser.add_subparsers(title='commands',
description='valid commands',
@@ -133,7 +129,10 @@
# TODO: add filters such as queue, project, changeid etc
show_running_jobs.set_defaults(func=self.show_running_jobs)
- self.args = parser.parse_args()
+ return parser
+
+ def parseArguments(self, args=None):
+ parser = super(Client, self).parseArguments()
if not getattr(self.args, 'func', None):
parser.print_help()
sys.exit(1)
@@ -156,8 +155,8 @@
logging.basicConfig(level=logging.DEBUG)
def main(self):
- self.parse_arguments()
- self.read_config()
+ self.parseArguments()
+ self.readConfig()
self.setup_logging()
self.server = self.config.get('gearman', 'server')
@@ -363,10 +362,8 @@
def main():
- client = Client()
- client.main()
+ Client().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index 979989d..aef8c95 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -14,14 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
-import daemon
-import extras
-
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
import grp
import logging
import os
@@ -41,25 +33,24 @@
# Similar situation with gear and statsd.
-class Executor(zuul.cmd.ZuulApp):
+class Executor(zuul.cmd.ZuulDaemonApp):
+ app_name = 'executor'
+ app_description = 'A standalone Zuul executor.'
- def parse_arguments(self):
- parser = argparse.ArgumentParser(description='Zuul executor.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
+ def createParser(self):
+ parser = super(Executor, self).createParser()
parser.add_argument('--keep-jobdir', dest='keep_jobdir',
action='store_true',
help='keep local jobdirs after run completes')
parser.add_argument('command',
choices=zuul.executor.server.COMMANDS,
nargs='?')
+ return parser
- self.args = parser.parse_args()
+ def parseArguments(self, args=None):
+ super(Executor, self).parseArguments()
+ if self.args.command:
+ self.args.nodaemon = True
def send_command(self, cmd):
state_dir = get_default(self.config, 'executor', 'state_dir',
@@ -111,8 +102,12 @@
os.chdir(pw.pw_dir)
os.umask(0o022)
- def main(self, daemon=True):
- # See comment at top of file about zuul imports
+ def run(self):
+ if self.args.command in zuul.executor.server.COMMANDS:
+ self.send_command(self.args.command)
+ sys.exit(0)
+
+ self.configure_connections(source_only=True)
self.user = get_default(self.config, 'executor', 'user', 'zuul')
@@ -145,9 +140,8 @@
self.executor.start()
signal.signal(signal.SIGUSR2, zuul.cmd.stack_dump_handler)
- if daemon:
- self.executor.join()
- else:
+
+ if self.args.nodaemon:
while True:
try:
signal.pause()
@@ -155,31 +149,13 @@
print("Ctrl + C: asking executor to exit nicely...\n")
self.exit_handler()
sys.exit(0)
+ else:
+ self.executor.join()
def main():
- server = Executor()
- server.parse_arguments()
- server.read_config()
-
- if server.args.command in zuul.executor.server.COMMANDS:
- server.send_command(server.args.command)
- sys.exit(0)
-
- server.configure_connections(source_only=True)
-
- pid_fn = get_default(server.config, 'executor', 'pidfile',
- '/var/run/zuul-executor/zuul-executor.pid',
- expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if server.args.nodaemon:
- server.main(False)
- else:
- with daemon.DaemonContext(pidfile=pid):
- server.main(True)
+ Executor().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py
index 9771fff..56b6b44 100755
--- a/zuul/cmd/merger.py
+++ b/zuul/cmd/merger.py
@@ -14,19 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
-import daemon
-import extras
-
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
-import sys
import signal
import zuul.cmd
-from zuul.lib.config import get_default
# No zuul imports here because they pull in paramiko which must not be
# imported until after the daemonization.
@@ -34,28 +24,21 @@
# Similar situation with gear and statsd.
-class Merger(zuul.cmd.ZuulApp):
-
- def parse_arguments(self):
- parser = argparse.ArgumentParser(description='Zuul merge worker.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
- self.args = parser.parse_args()
+class Merger(zuul.cmd.ZuulDaemonApp):
+ app_name = 'merger'
+ app_description = 'A standalone Zuul merger.'
def exit_handler(self, signum, frame):
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
self.merger.stop()
self.merger.join()
- def main(self):
+ def run(self):
# See comment at top of file about zuul imports
import zuul.merger.server
+ self.configure_connections(source_only=True)
+
self.setup_logging('merger', 'log_config')
self.merger = zuul.merger.server.MergeServer(self.config,
@@ -73,24 +56,8 @@
def main():
- server = Merger()
- server.parse_arguments()
-
- server.read_config()
- server.configure_connections(source_only=True)
-
- pid_fn = get_default(server.config, 'merger', 'pidfile',
- '/var/run/zuul-merger/zuul-merger.pid',
- expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if server.args.nodaemon:
- server.main()
- else:
- with daemon.DaemonContext(pidfile=pid):
- server.main()
+ Merger().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index 2d71f4d..bfcbef8 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -14,14 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
-import daemon
-import extras
-
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
import logging
import os
import sys
@@ -37,25 +29,14 @@
# Similar situation with gear and statsd.
-class Scheduler(zuul.cmd.ZuulApp):
+class Scheduler(zuul.cmd.ZuulDaemonApp):
+ app_name = 'scheduler'
+ app_description = 'The main zuul process.'
+
def __init__(self):
super(Scheduler, self).__init__()
self.gear_server_pid = None
- def parse_arguments(self, args=None):
- parser = argparse.ArgumentParser(description='Project gating system.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('-t', dest='validate', action='store_true',
- help='validate config file syntax (Does not'
- 'validate config repo validity)')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
- self.args = parser.parse_args(args)
-
def reconfigure_handler(self, signum, frame):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
self.log.debug("Reconfiguration triggered")
@@ -77,20 +58,6 @@
self.stop_gear_server()
os._exit(0)
- def test_config(self):
- # See comment at top of file about zuul imports
- import zuul.scheduler
- import zuul.executor.client
-
- logging.basicConfig(level=logging.DEBUG)
- try:
- self.sched = zuul.scheduler.Scheduler(self.config,
- testonly=True)
- except Exception as e:
- self.log.error("%s" % e)
- return -1
- return 0
-
def start_gear_server(self):
pipe_read, pipe_write = os.pipe()
child_pid = os.fork()
@@ -134,7 +101,7 @@
if self.gear_server_pid:
os.kill(self.gear_server_pid, signal.SIGKILL)
- def main(self):
+ def run(self):
# See comment at top of file about zuul imports
import zuul.scheduler
import zuul.executor.client
@@ -206,26 +173,8 @@
def main():
- scheduler = Scheduler()
- scheduler.parse_arguments()
-
- scheduler.read_config()
-
- if scheduler.args.validate:
- sys.exit(scheduler.test_config())
-
- pid_fn = get_default(scheduler.config, 'scheduler', 'pidfile',
- '/var/run/zuul-scheduler/zuul-scheduler.pid',
- expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if scheduler.args.nodaemon:
- scheduler.main()
- else:
- with daemon.DaemonContext(pidfile=pid):
- scheduler.main()
+ Scheduler().main()
if __name__ == "__main__":
- sys.path.insert(0, '.')
main()
diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py
index 9869a2c..6e5489f 100755
--- a/zuul/cmd/web.py
+++ b/zuul/cmd/web.py
@@ -13,10 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-import argparse
import asyncio
-import daemon
-import extras
import logging
import signal
import sys
@@ -27,34 +24,24 @@
from zuul.lib.config import get_default
-# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
-# instead it depends on lockfile-0.9.1 which uses pidfile.
-pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
-
-class WebServer(zuul.cmd.ZuulApp):
-
- def parse_arguments(self):
- parser = argparse.ArgumentParser(description='Zuul Web Server.')
- parser.add_argument('-c', dest='config',
- help='specify the config file')
- parser.add_argument('-d', dest='nodaemon', action='store_true',
- help='do not run as a daemon')
- parser.add_argument('--version', dest='version', action='version',
- version=self._get_version(),
- help='show zuul version')
- self.args = parser.parse_args()
+class WebServer(zuul.cmd.ZuulDaemonApp):
+ app_name = 'web'
+ app_description = 'A standalone Zuul web server.'
def exit_handler(self, signum, frame):
self.web.stop()
- def _main(self):
+ def _run(self):
params = dict()
params['listen_address'] = get_default(self.config,
'web', 'listen_address',
'127.0.0.1')
params['listen_port'] = get_default(self.config, 'web', 'port', 9000)
+ params['static_cache_expiry'] = get_default(self.config, 'web',
+ 'static_cache_expiry',
+ 3600)
params['gear_server'] = get_default(self.config, 'gearman', 'server')
params['gear_port'] = get_default(self.config, 'gearman', 'port', 4730)
params['ssl_key'] = get_default(self.config, 'gearman', 'ssl_key')
@@ -88,28 +75,19 @@
loop.close()
self.log.info("Zuul Web Server stopped")
- def main(self):
+ def run(self):
self.setup_logging('web', 'log_config')
self.log = logging.getLogger("zuul.WebServer")
try:
- self._main()
+ self._run()
except Exception:
self.log.exception("Exception from WebServer:")
def main():
- server = WebServer()
- server.parse_arguments()
- server.read_config()
+ WebServer().main()
- pid_fn = get_default(server.config, 'web', 'pidfile',
- '/var/run/zuul-web/zuul-web.pid', expand_user=True)
- pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
-
- if server.args.nodaemon:
- server.main()
- else:
- with daemon.DaemonContext(pidfile=pid):
- server.main()
+if __name__ == "__main__":
+ main()
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 759f327..791652f 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -924,6 +924,10 @@
private_ipv4=node.get('private_ipv4'),
public_ipv6=node.get('public_ipv6')))
+ username = node.get('username')
+ if username:
+ host_vars['ansible_user'] = username
+
host_keys = []
for key in node.get('host_keys'):
if port != 22:
diff --git a/zuul/model.py b/zuul/model.py
index b027c53..081d165 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -388,6 +388,7 @@
self.az = None
self.provider = None
self.region = None
+ self.username = None
@property
def state(self):
@@ -948,6 +949,28 @@
matchers.append(change_matcher.BranchMatcher(branch))
self.branch_matcher = change_matcher.MatchAny(matchers)
+ def getSimpleBranchMatcher(self):
+ # If the job has a simple branch matcher, return it; otherwise None.
+ if not self.branch_matcher:
+ return None
+ m = self.branch_matcher
+ if not isinstance(m, change_matcher.AbstractMatcherCollection):
+ return None
+ if len(m.matchers) != 1:
+ return None
+ m = m.matchers[0]
+ if not isinstance(m, change_matcher.BranchMatcher):
+ return None
+ return m._regex
+
+ def addBranchMatcher(self, branch):
+ # Add a branch matcher that combines as a boolean *and* with
+ # existing branch matchers, if any.
+ matchers = [change_matcher.BranchMatcher(branch)]
+ if self.branch_matcher:
+ matchers.append(self.branch_matcher)
+ self.branch_matcher = change_matcher.MatchAll(matchers)
+
def updateVariables(self, other_vars):
v = copy.deepcopy(self.variables)
Job._deepUpdate(v, other_vars)
@@ -1097,9 +1120,26 @@
for jobname, jobs in other.jobs.items():
joblist = self.jobs.setdefault(jobname, [])
for job in jobs:
- if not job.branch_matcher and implied_branch:
- job = job.copy()
- job.setBranchMatcher([implied_branch])
+ if implied_branch:
+ # If setting an implied branch and the current
+ # branch matcher is a simple match for a different
+ # branch, then simply do not add this job. If it
+ # is absent, set it to the implied branch.
+ # Otherwise, combine it with the implied branch to
+ # ensure that it still only affects this branch
+ # (whatever else it may do).
+ simple_branch = job.getSimpleBranchMatcher()
+ if simple_branch and simple_branch != implied_branch:
+ # Job is for a different branch, don't add it.
+ continue
+ if not simple_branch:
+ # The branch matcher could be complex, or
+ # missing. Add our implied matcher.
+ job = job.copy()
+ job.addBranchMatcher(implied_branch)
+ # Otherwise we have a simple branch matcher which
+ # is the same as our implied branch, the job can
+ # be added as-is.
if job not in joblist:
joblist.append(job)
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a725fcd..ed7d64b 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -184,14 +184,6 @@
self.request_id = request.id
-def toList(item):
- if not item:
- return []
- if isinstance(item, list):
- return item
- return [item]
-
-
class Scheduler(threading.Thread):
"""The engine of Zuul.
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index 61a1cee..766a21d 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -200,11 +200,13 @@
def __init__(self, listen_address, listen_port,
gear_server, gear_port,
- ssl_key=None, ssl_cert=None, ssl_ca=None):
+ ssl_key=None, ssl_cert=None, ssl_ca=None,
+ static_cache_expiry=3600):
self.listen_address = listen_address
self.listen_port = listen_port
self.event_loop = None
self.term = None
+ self.static_cache_expiry = static_cache_expiry
# instanciate handlers
self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port,
ssl_key, ssl_cert, ssl_ca)
@@ -228,7 +230,11 @@
fp = os.path.join(STATIC_DIR, "index.html")
elif request.path.endswith("status.html"):
fp = os.path.join(STATIC_DIR, "status.html")
- return web.FileResponse(fp)
+ headers = {}
+ if self.static_cache_expiry:
+ headers['Cache-Control'] = "public, max-age=%d" % \
+ self.static_cache_expiry
+ return web.FileResponse(fp, headers=headers)
def run(self, loop=None):
"""
diff --git a/zuul/web/static/README b/zuul/web/static/README
index 487c3ee..f17ea5f 100644
--- a/zuul/web/static/README
+++ b/zuul/web/static/README
@@ -5,6 +5,35 @@
* /static/js/jquery.graphite.min.js
* /static/bootstrap/css/bootstrap.min.css
+
+Use python2-rjsmin or another js minifier:
+```
+DEST_DIR=/var/www/html/static/
+mkdir -p $DEST_DIR/js
+echo "Fetching angular..."
+curl -L --silent https://ajax.googleapis.com/ajax/libs/angularjs/1.5.6/angular.min.js > $DEST_DIR/js/angular.min.js
+
+echo "Fetching jquery..."
+curl -L --silent http://code.jquery.com/jquery.min.js > $DEST_DIR/js/jquery.min.js
+
+echo "Fetching jquery-visibility..."
+curl -L --silent https://raw.githubusercontent.com/mathiasbynens/jquery-visibility/master/jquery-visibility.js > $DEST_DIR/js/jquery-visibility.js
+python2 -mrjsmin < $DEST_DIR/js/jquery-visibility.js > $DEST_DIR/js/jquery-visibility.min.js
+
+echo "Fetching bootstrap..."
+curl -L --silent https://github.com/twbs/bootstrap/releases/download/v3.1.1/bootstrap-3.1.1-dist.zip > bootstrap.zip
+unzip -q -o bootstrap.zip -d $DEST_DIR/
+mv $DEST_DIR/bootstrap-3.1.1-dist $DEST_DIR/bootstrap
+rm -f bootstrap.zip
+
+echo "Fetching jquery-graphite..."
+curl -L --silent https://github.com/prestontimmons/graphitejs/archive/master.zip > jquery-graphite.zip
+unzip -q -o jquery-graphite.zip -d $DEST_DIR/
+python2 -mrjsmin < $DEST_DIR/graphitejs-master/jquery.graphite.js > $DEST_DIR/js/jquery.graphite.min.js
+rm -Rf jquery-graphite.zip $DEST_DIR/graphitejs-master
+```
+
+
Here is an example apache vhost configuration:
<VirtualHost zuul-web.example.com:80>
DocumentRoot /var/www/zuul-web