Merge "Remove updateChange history from github driver" into feature/zuulv3
diff --git a/README.rst b/README.rst
index 52b89df..8d00665 100644
--- a/README.rst
+++ b/README.rst
@@ -10,6 +10,14 @@
The latest documentation for Zuul v3 is published at:
https://docs.openstack.org/infra/zuul/feature/zuulv3/
+If you are looking for the Edge routing service named Zuul that is
+related to Netflix, it can be found here:
+https://github.com/Netflix/zuul
+
+If you are looking for the Javascript testing tool named Zuul, it
+can be found here:
+https://github.com/defunctzombie/zuul
+
Contributing
------------
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 3bec28a..18bbfa3 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -408,7 +408,7 @@
Path to command socket file for the executor process.
.. attr:: finger_port
- :default: 79
+ :default: 7900
Port to use for finger log streamer.
@@ -451,13 +451,6 @@
SSH private key file to be used when logging into worker nodes.
- .. attr:: user
- :default: zuul
-
- User ID for the zuul-executor process. In normal operation as a
- daemon, the executor should be started as the ``root`` user, but
- it will drop privileges to this user during startup.
-
.. _admin_sitewide_variables:
.. attr:: variables
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 677e958..6e1b52e 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -12,6 +12,14 @@
:doc:`admin/index` useful. If you want help make Zuul itself better,
take a look at the :doc:`developer/index`.
+If you are looking for the Edge routing service named Zuul that is
+related to Netflix, it can be found here:
+https://github.com/Netflix/zuul
+
+If you are looking for the Javascript testing tool named Zuul, it
+can be found here:
+https://github.com/defunctzombie/zuul
+
Contents:
.. toctree::
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index fff673b..525cb38 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -539,6 +539,13 @@
specified in a project's pipeline, set this attribute to
``true``.
+ .. attr:: protected
+ :default: false
+
+ When set to ``true`` only jobs defined in the same project may inherit
+ from this job. Once this is set to ``true`` it cannot be reset to
+ ``false``.
+
.. attr:: success-message
:default: SUCCESS
diff --git a/requirements.txt b/requirements.txt
index 39a2b02..193c64e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,5 +25,6 @@
cachecontrol
pyjwt
iso8601
+yarl>=0.11,<1.0
aiohttp
uvloop;python_version>='3.5'
diff --git a/tests/base.py b/tests/base.py
index 64f6657..c449242 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1619,6 +1619,8 @@
data['username'] = 'fakeuser'
if 'windows' in node_type:
data['connection_type'] = 'winrm'
+ if 'network' in node_type:
+ data['connection_type'] = 'network_cli'
data = json.dumps(data).encode('utf8')
path = self.client.create(path, data,
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index 36789a3..f592eb4 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -40,6 +40,8 @@
label: fakeuser-label
- name: windows
label: windows-label
+ - name: network
+ label: network-label
- job:
name: base
diff --git a/tests/fixtures/config/protected/git/common-config/zuul.yaml b/tests/fixtures/config/protected/git/common-config/zuul.yaml
new file mode 100644
index 0000000..c941573
--- /dev/null
+++ b/tests/fixtures/config/protected/git/common-config/zuul.yaml
@@ -0,0 +1,16 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- job:
+ name: base
+ parent: null
diff --git a/tests/fixtures/config/protected/git/org_project/playbooks/job-protected.yaml b/tests/fixtures/config/protected/git/org_project/playbooks/job-protected.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/protected/git/org_project/playbooks/job-protected.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/protected/git/org_project/zuul.yaml b/tests/fixtures/config/protected/git/org_project/zuul.yaml
new file mode 100644
index 0000000..95f33df
--- /dev/null
+++ b/tests/fixtures/config/protected/git/org_project/zuul.yaml
@@ -0,0 +1,9 @@
+- job:
+ name: job-protected
+ protected: true
+ run: playbooks/job-protected.yaml
+
+- project:
+ name: org/project
+ check:
+ jobs: []
diff --git a/tests/fixtures/config/protected/git/org_project1/README b/tests/fixtures/config/protected/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/protected/git/org_project1/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/protected/git/org_project1/playbooks/job-child-notok.yaml b/tests/fixtures/config/protected/git/org_project1/playbooks/job-child-notok.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/protected/git/org_project1/playbooks/job-child-notok.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/protected/git/org_project1/playbooks/placeholder b/tests/fixtures/config/protected/git/org_project1/playbooks/placeholder
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/fixtures/config/protected/git/org_project1/playbooks/placeholder
diff --git a/tests/fixtures/config/protected/main.yaml b/tests/fixtures/config/protected/main.yaml
new file mode 100644
index 0000000..5f57245
--- /dev/null
+++ b/tests/fixtures/config/protected/main.yaml
@@ -0,0 +1,9 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project
+ - org/project1
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index 197b525..c45da94 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -115,7 +115,7 @@
self.assertEqual('check', buildset0['pipeline'])
self.assertEqual('org/project', buildset0['project'])
self.assertEqual(1, buildset0['change'])
- self.assertEqual(1, buildset0['patchset'])
+ self.assertEqual('1', buildset0['patchset'])
self.assertEqual('SUCCESS', buildset0['result'])
self.assertEqual('Build succeeded.', buildset0['message'])
self.assertEqual('tenant-one', buildset0['tenant'])
@@ -141,7 +141,7 @@
self.assertEqual('check', buildset1['pipeline'])
self.assertEqual('org/project', buildset1['project'])
self.assertEqual(2, buildset1['change'])
- self.assertEqual(1, buildset1['patchset'])
+ self.assertEqual('1', buildset1['patchset'])
self.assertEqual('FAILURE', buildset1['result'])
self.assertEqual('Build failed.', buildset1['message'])
@@ -194,7 +194,7 @@
self.assertEqual('check', buildsets_resultsdb[0]['pipeline'])
self.assertEqual('org/project', buildsets_resultsdb[0]['project'])
self.assertEqual(1, buildsets_resultsdb[0]['change'])
- self.assertEqual(1, buildsets_resultsdb[0]['patchset'])
+ self.assertEqual('1', buildsets_resultsdb[0]['patchset'])
self.assertEqual('SUCCESS', buildsets_resultsdb[0]['result'])
self.assertEqual('Build succeeded.', buildsets_resultsdb[0]['message'])
@@ -215,7 +215,7 @@
self.assertEqual(
'org/project', buildsets_resultsdb_failures[0]['project'])
self.assertEqual(2, buildsets_resultsdb_failures[0]['change'])
- self.assertEqual(1, buildsets_resultsdb_failures[0]['patchset'])
+ self.assertEqual('1', buildsets_resultsdb_failures[0]['patchset'])
self.assertEqual('FAILURE', buildsets_resultsdb_failures[0]['result'])
self.assertEqual(
'Build failed.', buildsets_resultsdb_failures[0]['message'])
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 6ab1a26..3942b0b 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -50,6 +50,12 @@
self.assertEqual(str(A.head_sha), zuulvars['patchset'])
self.assertEqual('master', zuulvars['branch'])
self.assertEqual(1, len(A.comments))
+ self.assertThat(
+ A.comments[0],
+ MatchesRegex('.*\[project-test1 \]\(.*\).*', re.DOTALL))
+ self.assertThat(
+ A.comments[0],
+ MatchesRegex('.*\[project-test2 \]\(.*\).*', re.DOTALL))
self.assertEqual(2, len(self.history))
# test_pull_unmatched_branch_event(self):
diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py
index be50447..b7e35eb 100644
--- a/tests/unit/test_inventory.py
+++ b/tests/unit/test_inventory.py
@@ -37,6 +37,12 @@
inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
return yaml.safe_load(open(inv_path, 'r'))
+ def _get_setup_inventory(self, name):
+ build = self.getBuildByName(name)
+ setup_inv_path = os.path.join(build.jobdir.root, 'ansible',
+ 'setup-inventory.yaml')
+ return yaml.safe_load(open(setup_inv_path, 'r'))
+
def test_single_inventory(self):
inventory = self._get_build_inventory('single-inventory')
@@ -131,3 +137,23 @@
self.executor_server.release()
self.waitUntilSettled()
+
+ def test_setup_inventory(self):
+
+ setup_inventory = self._get_setup_inventory('hostvars-inventory')
+ inventory = self._get_build_inventory('hostvars-inventory')
+
+ self.assertIn('all', inventory)
+ self.assertIn('hosts', inventory['all'])
+
+ self.assertIn('default', setup_inventory['all']['hosts'])
+ self.assertIn('fakeuser', setup_inventory['all']['hosts'])
+ self.assertIn('windows', setup_inventory['all']['hosts'])
+ self.assertNotIn('network', setup_inventory['all']['hosts'])
+ self.assertIn('default', inventory['all']['hosts'])
+ self.assertIn('fakeuser', inventory['all']['hosts'])
+ self.assertIn('windows', inventory['all']['hosts'])
+ self.assertIn('network', inventory['all']['hosts'])
+
+ self.executor_server.release()
+ self.waitUntilSettled()
diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py
index 59dd8b0..b999106 100644
--- a/tests/unit/test_streaming.py
+++ b/tests/unit/test_streaming.py
@@ -41,13 +41,13 @@
def startStreamer(self, port, root=None):
if not root:
root = tempfile.gettempdir()
- return zuul.lib.log_streamer.LogStreamer(None, self.host, port, root)
+ return zuul.lib.log_streamer.LogStreamer(self.host, port, root)
def test_start_stop(self):
- port = 7900
- streamer = self.startStreamer(port)
+ streamer = self.startStreamer(0)
self.addCleanup(streamer.stop)
+ port = streamer.server.socket.getsockname()[1]
s = socket.create_connection((self.host, port))
s.close()
@@ -77,8 +77,9 @@
def startStreamer(self, port, build_uuid, root=None):
if not root:
root = tempfile.gettempdir()
- self.streamer = zuul.lib.log_streamer.LogStreamer(None, self.host,
+ self.streamer = zuul.lib.log_streamer.LogStreamer(self.host,
port, root)
+ port = self.streamer.server.socket.getsockname()[1]
s = socket.create_connection((self.host, port))
self.addCleanup(s.close)
@@ -129,10 +130,9 @@
# Create a thread to stream the log. We need this to be happening
# before we create the flag file to tell the job to complete.
- port = 7901
streamer_thread = threading.Thread(
target=self.startStreamer,
- args=(port, build.uuid, self.executor_server.jobdir_root,)
+ args=(0, build.uuid, self.executor_server.jobdir_root,)
)
streamer_thread.start()
self.addCleanup(self.stopStreamer)
@@ -209,7 +209,7 @@
def test_websocket_streaming(self):
# Start the finger streamer daemon
streamer = zuul.lib.log_streamer.LogStreamer(
- None, self.host, 0, self.executor_server.jobdir_root)
+ self.host, 0, self.executor_server.jobdir_root)
self.addCleanup(streamer.stop)
# Need to set the streaming port before submitting the job
@@ -294,7 +294,7 @@
def test_finger_gateway(self):
# Start the finger streamer daemon
streamer = zuul.lib.log_streamer.LogStreamer(
- None, self.host, 0, self.executor_server.jobdir_root)
+ self.host, 0, self.executor_server.jobdir_root)
self.addCleanup(streamer.stop)
finger_port = streamer.server.socket.getsockname()[1]
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 2779e6e..163a58b 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -73,6 +73,110 @@
"not affect tenant one")
+class TestProtected(ZuulTestCase):
+
+ tenant_config_file = 'config/protected/main.yaml'
+
+ def test_protected_ok(self):
+ # test clean usage of final parent job
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: job-protected
+ protected: true
+ run: playbooks/job-protected.yaml
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - job-child-ok
+
+ - job:
+ name: job-child-ok
+ parent: job-protected
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - job-child-ok
+
+ """)
+
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '1')
+
+ def test_protected_reset(self):
+ # try to reset protected flag
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: job-protected
+ protected: true
+ run: playbooks/job-protected.yaml
+
+ - job:
+ name: job-child-reset-protected
+ parent: job-protected
+ protected: false
+
+ - project:
+ name: org/project
+ check:
+ jobs:
+ - job-child-reset-protected
+
+ """)
+
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ # The second patch tried to override some variables.
+ # Thus it should fail.
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '-1')
+ self.assertIn('Unable to reset protected attribute', A.messages[0])
+
+ def test_protected_inherit_not_ok(self):
+ # try to inherit from a protected job in different project
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: job-child-notok
+ run: playbooks/job-child-notok.yaml
+ parent: job-protected
+
+ - project:
+ name: org/project1
+ check:
+ jobs:
+ - job-child-notok
+
+ """)
+
+ file_dict = {'zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.reported, 1)
+ self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '-1')
+ self.assertIn(
+ "which is defined in review.example.com/org/project is protected "
+ "and cannot be inherited from other projects.", A.messages[0])
+
+
class TestFinal(ZuulTestCase):
tenant_config_file = 'config/final/main.yaml'
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index 236fd9f..07d4a8d 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -181,8 +181,9 @@
else:
# Exercise the pidfile before we do anything else (including
# logging or daemonizing)
- with daemon.DaemonContext(pidfile=pid):
+ with pid:
pass
+
with daemon.DaemonContext(pidfile=pid):
self.run()
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index ade9715..ad7aaa8 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -14,10 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import grp
import logging
import os
-import pwd
import sys
import signal
import tempfile
@@ -64,7 +62,7 @@
self.log.info("Starting log streamer")
streamer = zuul.lib.log_streamer.LogStreamer(
- self.user, '::', self.finger_port, self.job_dir)
+ '::', self.finger_port, self.job_dir)
# Keep running until the parent dies:
pipe_read = os.fdopen(pipe_read)
@@ -76,22 +74,6 @@
os.close(pipe_read)
self.log_streamer_pid = child_pid
- def change_privs(self):
- '''
- Drop our privileges to the zuul user.
- '''
- if os.getuid() != 0:
- return
- pw = pwd.getpwnam(self.user)
- # get a list of supplementary groups for the target user, and make sure
- # we set them when dropping privileges.
- groups = [g.gr_gid for g in grp.getgrall() if self.user in g.gr_mem]
- os.setgroups(groups)
- os.setgid(pw.pw_gid)
- os.setuid(pw.pw_uid)
- os.chdir(pw.pw_dir)
- os.umask(0o022)
-
def run(self):
if self.args.command in zuul.executor.server.COMMANDS:
self.send_command(self.args.command)
@@ -99,8 +81,6 @@
self.configure_connections(source_only=True)
- self.user = get_default(self.config, 'executor', 'user', 'zuul')
-
if self.config.has_option('executor', 'job_dir'):
self.job_dir = os.path.expanduser(
self.config.get('executor', 'job_dir'))
@@ -120,7 +100,6 @@
)
self.start_log_streamer()
- self.change_privs()
ExecutorServer = zuul.executor.server.ExecutorServer
self.executor = ExecutorServer(self.config, self.connections,
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 3a7e9b9..d622370 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -474,6 +474,7 @@
# Attributes of a job that can also be used in Project and ProjectTemplate
job_attributes = {'parent': vs.Any(str, None),
'final': bool,
+ 'protected': bool,
'failure-message': str,
'success-message': str,
'failure-url': str,
@@ -513,6 +514,7 @@
simple_attributes = [
'final',
+ 'protected',
'timeout',
'workspace',
'voting',
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
index 505757f..848ae1b 100644
--- a/zuul/driver/github/githubreporter.py
+++ b/zuul/driver/github/githubreporter.py
@@ -75,6 +75,14 @@
msg = self._formatItemReportMergeFailure(item)
self.addPullComment(item, msg)
+ def _formatItemReportJobs(self, item):
+ # Return the list of jobs portion of the report
+ ret = ''
+ jobs_fields = self._getItemReportJobsFields(item)
+ for job_fields in jobs_fields:
+ ret += '- [%s](%s) : %s%s%s%s\n' % job_fields
+ return ret
+
def addPullComment(self, item, comment=None):
message = comment or self._formatItemReport(item)
project = item.change.project.name
diff --git a/zuul/driver/sql/alembic.ini b/zuul/driver/sql/alembic.ini
new file mode 100644
index 0000000..e94d496
--- /dev/null
+++ b/zuul/driver/sql/alembic.ini
@@ -0,0 +1,2 @@
+[alembic]
+script_location = alembic
diff --git a/zuul/driver/sql/alembic/versions/19d3a3ebfe1d_change_patchset_to_string.py b/zuul/driver/sql/alembic/versions/19d3a3ebfe1d_change_patchset_to_string.py
new file mode 100644
index 0000000..505a1ed
--- /dev/null
+++ b/zuul/driver/sql/alembic/versions/19d3a3ebfe1d_change_patchset_to_string.py
@@ -0,0 +1,29 @@
+"""Change patchset to string
+
+Revision ID: 19d3a3ebfe1d
+Revises: cfc0dc45f341
+Create Date: 2018-01-10 07:42:16.546751
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '19d3a3ebfe1d'
+down_revision = 'cfc0dc45f341'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+BUILDSET_TABLE = 'zuul_buildset'
+
+
+def upgrade(table_prefix=''):
+ op.alter_column(table_prefix + BUILDSET_TABLE,
+ 'patchset',
+ type_=sa.String(255),
+ existing_nullable=True)
+
+
+def downgrade():
+ raise Exception("Downgrades not supported")
diff --git a/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py
new file mode 100644
index 0000000..3fde8e5
--- /dev/null
+++ b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py
@@ -0,0 +1,30 @@
+"""Change patchset to string
+
+Revision ID: cfc0dc45f341
+Revises: ba4cdce9b18c
+Create Date: 2018-01-09 16:44:31.506958
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'cfc0dc45f341'
+down_revision = 'ba4cdce9b18c'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+BUILDSET_TABLE = 'zuul_buildset'
+
+
+def upgrade(table_prefix=''):
+ op.alter_column(table_prefix + BUILDSET_TABLE,
+ 'patchset',
+ sa.String(255),
+ existing_nullable=True,
+ existing_type=sa.Integer)
+
+
+def downgrade():
+ raise Exception("Downgrades not supported")
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index 285d0c2..715d72b 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -92,7 +92,7 @@
sa.Column('pipeline', sa.String(255)),
sa.Column('project', sa.String(255)),
sa.Column('change', sa.Integer, nullable=True),
- sa.Column('patchset', sa.Integer, nullable=True),
+ sa.Column('patchset', sa.String(255), nullable=True),
sa.Column('ref', sa.String(255)),
sa.Column('oldrev', sa.String(255)),
sa.Column('newrev', sa.String(255)),
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 5a710a6..a8ab8c4 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -44,7 +44,8 @@
BUFFER_LINES_FOR_SYNTAX = 200
COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
'unverbose', 'keep', 'nokeep']
-DEFAULT_FINGER_PORT = 79
+DEFAULT_FINGER_PORT = 7900
+BLACKLISTED_ANSIBLE_CONNECTION_TYPES = ['network_cli']
class StopException(Exception):
@@ -347,6 +348,8 @@
pass
self.known_hosts = os.path.join(ssh_dir, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
+ self.setup_inventory = os.path.join(self.ansible_root,
+ 'setup-inventory.yaml')
self.logging_json = os.path.join(self.ansible_root, 'logging.json')
self.playbooks = [] # The list of candidate playbooks
self.playbook = None # A pointer to the candidate we have chosen
@@ -493,6 +496,26 @@
shutil.copy(os.path.join(library_path, fn), target_dir)
+def make_setup_inventory_dict(nodes):
+
+ hosts = {}
+ for node in nodes:
+ if (node['host_vars']['ansible_connection'] in
+ BLACKLISTED_ANSIBLE_CONNECTION_TYPES):
+ continue
+
+ for name in node['name']:
+ hosts[name] = node['host_vars']
+
+ inventory = {
+ 'all': {
+ 'hosts': hosts,
+ }
+ }
+
+ return inventory
+
+
def make_inventory_dict(nodes, groups, all_vars):
hosts = {}
@@ -1157,8 +1180,13 @@
result_data_file=self.jobdir.result_data_file)
nodes = self.getHostList(args)
+ setup_inventory = make_setup_inventory_dict(nodes)
inventory = make_inventory_dict(nodes, args['groups'], all_vars)
+ with open(self.jobdir.setup_inventory, 'w') as setup_inventory_yaml:
+ setup_inventory_yaml.write(
+ yaml.safe_dump(setup_inventory, default_flow_style=False))
+
with open(self.jobdir.inventory, 'w') as inventory_yaml:
inventory_yaml.write(
yaml.safe_dump(inventory, default_flow_style=False))
@@ -1423,6 +1451,7 @@
verbose = '-v'
cmd = ['ansible', '*', verbose, '-m', 'setup',
+ '-i', self.jobdir.setup_inventory,
'-a', 'gather_subset=!all']
result, code = self.runAnsible(
diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py
index c778812..f96f442 100644
--- a/zuul/lib/log_streamer.py
+++ b/zuul/lib/log_streamer.py
@@ -157,12 +157,11 @@
Class implementing log streaming over the finger daemon port.
'''
- def __init__(self, user, host, port, jobdir_root):
+ def __init__(self, host, port, jobdir_root):
self.log = logging.getLogger('zuul.log_streamer')
self.log.debug("LogStreamer starting on port %s", port)
self.server = LogStreamerServer((host, port),
RequestHandler,
- user=user,
jobdir_root=jobdir_root)
# We start the actual serving within a thread so we can return to
diff --git a/zuul/lib/streamer_utils.py b/zuul/lib/streamer_utils.py
index 43bc286..3d2d561 100644
--- a/zuul/lib/streamer_utils.py
+++ b/zuul/lib/streamer_utils.py
@@ -74,7 +74,7 @@
address_family = socket.AF_INET6
def __init__(self, *args, **kwargs):
- self.user = kwargs.pop('user')
+ self.user = kwargs.pop('user', None)
self.pid_file = kwargs.pop('pid_file', None)
socketserver.ThreadingTCPServer.__init__(self, *args, **kwargs)
diff --git a/zuul/model.py b/zuul/model.py
index bac9e4c..29c5a9d 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -845,6 +845,7 @@
semaphore=None,
attempts=3,
final=False,
+ protected=None,
roles=(),
required_projects={},
allowed_projects=None,
@@ -862,6 +863,7 @@
inheritance_path=(),
parent_data=None,
description=None,
+ protected_origin=None,
)
self.inheritable_attributes = {}
@@ -1039,12 +1041,21 @@
for k in self.execution_attributes:
if (other._get(k) is not None and
- k not in set(['final'])):
+ k not in set(['final', 'protected'])):
if self.final:
raise Exception("Unable to modify final job %s attribute "
"%s=%s with variant %s" % (
repr(self), k, other._get(k),
repr(other)))
+ if self.protected_origin:
+ # this is a protected job, check origin of job definition
+ this_origin = self.protected_origin
+ other_origin = other.source_context.project.canonical_name
+ if this_origin != other_origin:
+ raise Exception("Job %s which is defined in %s is "
+ "protected and cannot be inherited "
+ "from other projects."
+ % (repr(self), this_origin))
if k not in set(['pre_run', 'run', 'post_run', 'roles',
'variables', 'required_projects']):
# TODO(jeblair): determine if deepcopy is required
@@ -1055,6 +1066,17 @@
if other.final != self.attributes['final']:
self.final = other.final
+ # Protected may only be set to true
+ if other.protected is not None:
+ # don't allow to reset protected flag
+ if not other.protected and self.protected_origin:
+ raise Exception("Unable to reset protected attribute of job"
+ " %s by job %s" % (
+ repr(self), repr(other)))
+ if not self.protected_origin:
+ self.protected_origin = \
+ other.source_context.project.canonical_name
+
# We must update roles before any playbook contexts
if other._get('roles') is not None:
self.addRoles(other.roles)
diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py
index ecf8855..1bff5cb 100644
--- a/zuul/reporter/__init__.py
+++ b/zuul/reporter/__init__.py
@@ -109,12 +109,10 @@
else:
return self._formatItemReport(item)
- def _formatItemReportJobs(self, item):
- # Return the list of jobs portion of the report
- ret = ''
-
+ def _getItemReportJobsFields(self, item):
+ # Extract the report elements from an item
config = self.connection.sched.config
-
+ jobs_fields = []
for job in item.getJobs():
build = item.current_build_set.getBuild(job.name)
(result, url) = item.formatJobResult(job)
@@ -147,6 +145,13 @@
else:
error = ''
name = job.name + ' '
- ret += '- %s%s : %s%s%s%s\n' % (name, url, result, error,
- elapsed, voting)
+ jobs_fields.append((name, url, result, error, elapsed, voting))
+ return jobs_fields
+
+ def _formatItemReportJobs(self, item):
+ # Return the list of jobs portion of the report
+ ret = ''
+ jobs_fields = self._getItemReportJobsFields(item)
+ for job_fields in jobs_fields:
+ ret += '- %s%s : %s%s%s%s\n' % job_fields
return ret