Merge "Switch to build-openstack-infra-sphinx-docs for doc build jobs" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index d4fc1e9..b0fe135 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -28,6 +28,17 @@
- "zuul/ansible/callback/.*"
- "playbooks/zuul-stream/.*"
+- job:
+ name: zuul-migrate
+ parent: unittests
+ run: playbooks/zuul-migrate
+ # We're adding zuul to the required-projects so that we can also trigger
+ # this from project-config changes
+ required-projects:
+ - openstack-infra/project-config
+ - name: openstack-infra/zuul
+ override-branch: feature/zuulv3
+
- project:
name: openstack-infra/zuul
check:
@@ -39,15 +50,20 @@
- tox-pep8
- tox-py35
- zuul-stream-functional
+ - zuul-migrate:
+ files:
+ - zuul/cmd/migrate.py
+ - playbooks/zuul-migrate.yaml
gate:
jobs:
- build-openstack-infra-sphinx-docs:
success-url: 'html/feature/zuulv3/'
- tox-pep8
- tox-py35
+ - zuul-stream-functional
post:
jobs:
- - publish-openstack-python-docs:
+ - publish-openstack-python-docs-infra:
vars:
afs_publisher_target: 'infra/zuul'
- publish-openstack-python-branch-tarball
diff --git a/doc/source/admin/tenants.rst b/doc/source/admin/tenants.rst
index 54bc10a..4722750 100644
--- a/doc/source/admin/tenants.rst
+++ b/doc/source/admin/tenants.rst
@@ -163,6 +163,11 @@
The maximum number of nodes a job can request. A value of
'-1' value removes the limit.
+ .. attr:: max-job-timeout
+ :default: 10800
+
+ The maximum timeout for jobs. A value of '-1' value removes the limit.
+
.. attr:: exclude-unprotected-branches
:default: false
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 973470d..025ea71 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -497,9 +497,10 @@
- job:
name: run-tests
parent: base
- nodes:
- - name: test-node
- label: fedora
+ nodeset:
+ nodes:
+ - name: test-node
+ label: fedora
.. attr:: job
@@ -630,12 +631,12 @@
- job:
name: run-tests
- nodes: current-release
+ nodeset: current-release
- job:
name: run-tests
branch: stable/2.0
- nodes: old-release
+ nodeset: old-release
In some cases, Zuul uses an implied value for the branch
specifier if none is supplied:
@@ -722,18 +723,19 @@
ssh_key:
key: descrypted-secret-key-data
- .. attr:: nodes
+ .. attr:: nodeset
- A list of nodes which should be supplied to the job. This
- parameter may be supplied either as a string, in which case it
- references a :ref:`nodeset` definition which appears elsewhere
- in the configuration, or a list, in which case it is interpreted
- in the same way as a Nodeset definition (in essence, it is an
- anonymous Node definition unique to this job). See the
- :ref:`nodeset` reference for the syntax to use in that case.
+ The nodes which should be supplied to the job. This parameter
+ may be supplied either as a string, in which case it references
+ a :ref:`nodeset` definition which appears elsewhere in the
+ configuration, or a dictionary, in which case it is interpreted
+ in the same way as a Nodeset definition, though the ``name``
+ attribute should be omitted (in essence, it is an anonymous
+ Nodeset definition unique to this job). See the :ref:`nodeset`
+ reference for the syntax to use in that case.
- If a job has an empty or no node definition, it will still run
- and may be able to perform actions on the Zuul executor.
+ If a job has an empty or no nodeset definition, it will still
+ run and may be able to perform actions on the Zuul executor.
.. attr:: override-branch
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 837fb17..3d24f5d 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -303,11 +303,24 @@
The identifier for the change.
+ .. var:: change_url
+
+ The URL to the source location of the given change.
+ E.g., `https://review.example.org/#/c/123456/` or
+ `https://github.com/example/example/pull/1234`.
+
.. var:: patchset
The patchset identifier for the change. If a change is
revised, this will have a different value.
+.. var:: zuul_success
+
+ Post run playbook(s) will be passed this variable to indicate if the run
+ phase of the job was successful or not. This variable is meant to be used
+ with the `boolean` filter.
+
+
Change Items
++++++++++++
@@ -332,6 +345,12 @@
The patchset identifier for the change. If a change is revised,
this will have a different value.
+ .. var:: change_url
+
+ The URL to the source location of the given change.
+ E.g., `https://review.example.org/#/c/123456/` or
+ `https://github.com/example/example/pull/1234`.
+
Branch Items
++++++++++++
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index 1937cd5..70e999e 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -569,6 +569,11 @@
// Toggle showing/hiding the patchset when the header is
// clicked.
+ if (e.target.nodeName.toLowerCase() === 'a') {
+ // Ignore clicks from gerrit patch set link
+ return;
+ }
+
// Grab the patchset panel
var $panel = $(e.target).parents('.zuul-change');
var $body = $panel.children('.zuul-patchset-body');
@@ -652,7 +657,7 @@
setTimeout(function() {app.schedule(app);}, 5000);
return;
}
- app.update().complete(function () {
+ app.update().always(function () {
setTimeout(function() {app.schedule(app);}, 5000);
});
@@ -722,7 +727,7 @@
.removeClass('zuul-msg-wrap-off')
.show();
})
- .complete(function () {
+ .always(function () {
xhr = undefined;
app.emit('update-end');
});
diff --git a/playbooks/zuul-migrate.yaml b/playbooks/zuul-migrate.yaml
new file mode 100644
index 0000000..7ed6626
--- /dev/null
+++ b/playbooks/zuul-migrate.yaml
@@ -0,0 +1,26 @@
+- hosts: all
+ tasks:
+
+ - name: Install migration dependencies
+ command: "python3 -m pip install --user src/git.openstack.org/openstack-infra/zuul[migrate]"
+
+ - name: Migrate the data
+ command: "python3 ../zuul/zuul/cmd/migrate.py zuul/layout.yaml jenkins/jobs nodepool/nodepool.yaml . --mapping=zuul/mapping.yaml -v -m"
+ args:
+ chdir: src/git.openstack.org/openstack-infra/project-config
+
+ - name: Collect generated job config
+ synchronize:
+ dest: "{{ zuul.executor.log_root }}"
+ mode: pull
+ src: "src/git.openstack.org/openstack-infra/project-config/zuul.d"
+ verify_host: true
+ no_log: true
+
+ - name: Collect generated playbooks
+ synchronize:
+ dest: "{{ zuul.executor.log_root }}/playbooks"
+ mode: pull
+ src: "src/git.openstack.org/openstack-infra/project-config/playbooks/legacy"
+ verify_host: true
+ no_log: true
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml
index 6a31ff8..fd28757 100644
--- a/playbooks/zuul-stream/fixtures/test-stream.yaml
+++ b/playbooks/zuul-stream/fixtures/test-stream.yaml
@@ -10,6 +10,10 @@
debug:
var: setupvar
+ - name: Output a debug sentence
+ debug:
+ msg: This is a debug message
+
- name: Run a shell task
command: ip addr show
diff --git a/playbooks/zuul-stream/functional.yaml b/playbooks/zuul-stream/functional.yaml
index 9275037..6b67b05 100644
--- a/playbooks/zuul-stream/functional.yaml
+++ b/playbooks/zuul-stream/functional.yaml
@@ -22,8 +22,8 @@
- name: Validate output - shell task
shell: |
- egrep "^.*\| node1 \| link/loopback" job-output.txt
- egrep "^.*\| node2 \| link/loopback" job-output.txt
+ egrep "^.*\| node1 \| link/loopback" job-output.txt
+ egrep "^.*\| node2 \| link/loopback" job-output.txt
- name: Validate output - loop with items
shell: |
diff --git a/requirements.txt b/requirements.txt
index eadb508..cdffda2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,6 +17,7 @@
PrettyTable>=0.6,<0.8
babel>=1.0
ansible>=2.3.0.0,<2.4
+netaddr
kazoo
sqlalchemy
alembic
diff --git a/setup.cfg b/setup.cfg
index ce7a40e..63ff562 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -27,6 +27,7 @@
zuul-executor = zuul.cmd.executor:main
zuul-bwrap = zuul.driver.bubblewrap:main
zuul-web = zuul.cmd.web:main
+ zuul-migrate = zuul.cmd.migrate:main
[build_sphinx]
source-dir = doc/source
@@ -37,3 +38,5 @@
[extras]
mysql_reporter=
PyMySQL
+migrate=
+ jenkins-job-builder==1.6.2
diff --git a/tests/base.py b/tests/base.py
index 4214809..c159865 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1012,8 +1012,7 @@
def getGithubClient(self,
project=None,
- user_id=None,
- use_app=True):
+ user_id=None):
return self.github_client
def openFakePullRequest(self, project, branch, subject, files=[],
@@ -1195,7 +1194,7 @@
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
- self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ self.sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
self.sock.bind(('', 0))
self.port = self.sock.getsockname()[1]
self.wake_read, self.wake_write = os.pipe()
@@ -2145,6 +2144,7 @@
def getGithubConnection(driver, name, config):
con = FakeGithubConnection(driver, name, config,
upstream_root=self.upstream_root)
+ self.event_queues.append(con.event_queue)
setattr(self, 'fake_' + name, con)
return con
diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
index 9bfeb0e..e6bd5ef 100644
--- a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml
@@ -24,6 +24,12 @@
- zuul.project.canonical_name == 'review.example.com/org/project'
- zuul.project.src_dir == 'src/review.example.com/org/project'
+ - name: Assert legacy zuul vars are valid
+ assert:
+ that:
+ - zuul.project.name == '{{ (zuul | zuul_legacy_vars).ZUUL_PROJECT }}'
+ - zuul.branch == '{{ (zuul | zuul_legacy_vars).ZUUL_BRANCH }}'
+
- debug:
msg: "vartest secret {{ vartest_secret }}"
diff --git a/tests/fixtures/config/ansible/git/common-config/zuul.yaml b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
index d34d5c4..67d1c70 100644
--- a/tests/fixtures/config/ansible/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/ansible/git/common-config/zuul.yaml
@@ -98,9 +98,10 @@
- job:
parent: python27
name: check-vars
- nodes:
- - name: ubuntu-xenial
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: ubuntu-xenial
+ label: ubuntu-xenial
vars:
vartest_job: vartest_job
vartest_secret: vartest_job
@@ -112,9 +113,10 @@
- job:
parent: python27
name: check-secret-names
- nodes:
- - name: ubuntu-xenial
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: ubuntu-xenial
+ label: ubuntu-xenial
secrets:
- secret: vartest_secret
name: renamed_secret
diff --git a/tests/fixtures/config/in-repo-join/git/common-config/playbooks/common-config-test.yaml b/tests/fixtures/config/in-repo-join/git/common-config/playbooks/common-config-test.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/in-repo-join/git/common-config/playbooks/common-config-test.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
new file mode 100644
index 0000000..561fc39
--- /dev/null
+++ b/tests/fixtures/config/in-repo-join/git/common-config/zuul.yaml
@@ -0,0 +1,46 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (tenant-one-gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - Approved: 1
+ success:
+ gerrit:
+ Verified: 2
+ submit: true
+ failure:
+ gerrit:
+ Verified: -2
+ start:
+ gerrit:
+ Verified: 0
+ precedence: high
+
+- job:
+ name: base
+ parent: null
+
+- job:
+ name: common-config-test
+
+- project:
+ name: org/project
+ check:
+ jobs:
+ - common-config-test
diff --git a/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml b/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
new file mode 100644
index 0000000..280342c
--- /dev/null
+++ b/tests/fixtures/config/in-repo-join/git/org_project/.zuul.yaml
@@ -0,0 +1,2 @@
+- job:
+ name: project-test1
diff --git a/tests/fixtures/config/in-repo-join/git/org_project/README b/tests/fixtures/config/in-repo-join/git/org_project/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/in-repo-join/git/org_project/README
@@ -0,0 +1 @@
+test
diff --git a/tests/fixtures/config/in-repo-join/git/org_project/playbooks/project-test1.yaml b/tests/fixtures/config/in-repo-join/git/org_project/playbooks/project-test1.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/in-repo-join/git/org_project/playbooks/project-test1.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/in-repo-join/main.yaml b/tests/fixtures/config/in-repo-join/main.yaml
new file mode 100644
index 0000000..208e274
--- /dev/null
+++ b/tests/fixtures/config/in-repo-join/main.yaml
@@ -0,0 +1,8 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project
diff --git a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
index ff4268b..5623467 100644
--- a/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml
@@ -78,6 +78,8 @@
- project:
name: common-config
+ check:
+ jobs: []
tenant-one-gate:
jobs:
- common-config-test
diff --git a/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
index 60cd434..e1c27bb 100644
--- a/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
+++ b/tests/fixtures/config/in-repo/git/org_project/.zuul.yaml
@@ -3,6 +3,8 @@
- project:
name: org/project
+ check:
+ jobs: []
tenant-one-gate:
jobs:
- project-test1
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index 7809c5d..e5727a2 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -37,10 +37,11 @@
- job:
name: single-inventory
- nodes:
- - name: ubuntu-xenial
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: ubuntu-xenial
+ label: ubuntu-xenial
- job:
name: group-inventory
- nodes: nodeset1
+ nodeset: nodeset1
diff --git a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
index 27f2fd5..273469c 100644
--- a/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml
@@ -17,6 +17,7 @@
- job:
name: python27
- nodes:
- - name: controller
- label: ubuntu-trusty
+ nodeset:
+ nodes:
+ - name: controller
+ label: ubuntu-trusty
diff --git a/tests/fixtures/config/multi-tenant/main.yaml b/tests/fixtures/config/multi-tenant/main.yaml
index 4916905..e667588 100644
--- a/tests/fixtures/config/multi-tenant/main.yaml
+++ b/tests/fixtures/config/multi-tenant/main.yaml
@@ -1,5 +1,6 @@
- tenant:
name: tenant-one
+ max-job-timeout: 1800
source:
gerrit:
config-projects:
diff --git a/tests/fixtures/config/openstack/git/project-config/zuul.yaml b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
index 2506db0..de6321d 100644
--- a/tests/fixtures/config/openstack/git/project-config/zuul.yaml
+++ b/tests/fixtures/config/openstack/git/project-config/zuul.yaml
@@ -37,9 +37,10 @@
name: base
parent: null
timeout: 30
- nodes:
- - name: controller
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: controller
+ label: ubuntu-xenial
- job:
name: python27
@@ -49,9 +50,10 @@
name: python27
parent: base
branches: stable/mitaka
- nodes:
- - name: controller
- label: ubuntu-trusty
+ nodeset:
+ nodes:
+ - name: controller
+ label: ubuntu-trusty
- job:
name: python35
diff --git a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
index 9796fe2..14f43f4 100644
--- a/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/single-tenant/git/common-config/zuul.yaml
@@ -47,41 +47,47 @@
- job:
name: project-merge
hold-following-changes: true
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
attempts: 4
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
branches: stable
- nodes:
- - name: controller
- label: label2
+ nodeset:
+ nodes:
+ - name: controller
+ label: label2
- job:
name: project-post
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- job:
name: project-test2
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project1-project2-integration
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-testfile
diff --git a/tests/fixtures/layouts/autohold.yaml b/tests/fixtures/layouts/autohold.yaml
index 515f79d..578f886 100644
--- a/tests/fixtures/layouts/autohold.yaml
+++ b/tests/fixtures/layouts/autohold.yaml
@@ -17,9 +17,10 @@
- job:
name: project-test2
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- project:
name: org/project
diff --git a/tests/fixtures/layouts/disable_at.yaml b/tests/fixtures/layouts/disable_at.yaml
index 7b1b8c8..8c24c1b 100644
--- a/tests/fixtures/layouts/disable_at.yaml
+++ b/tests/fixtures/layouts/disable_at.yaml
@@ -21,9 +21,10 @@
- job:
name: project-test1
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- project:
name: org/project
diff --git a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
index 6a92deb..bb98b57 100644
--- a/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
+++ b/tests/fixtures/layouts/dont-ignore-ref-deletes.yaml
@@ -13,9 +13,10 @@
- job:
name: project-post
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/fixtures/layouts/idle.yaml b/tests/fixtures/layouts/idle.yaml
index ec31408..4cc07ae 100644
--- a/tests/fixtures/layouts/idle.yaml
+++ b/tests/fixtures/layouts/idle.yaml
@@ -11,9 +11,10 @@
- job:
name: project-bitrot
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/fixtures/layouts/no-timer.yaml b/tests/fixtures/layouts/no-timer.yaml
index 3790ea7..7aaa1ed 100644
--- a/tests/fixtures/layouts/no-timer.yaml
+++ b/tests/fixtures/layouts/no-timer.yaml
@@ -29,9 +29,10 @@
- job:
name: project-bitrot
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/fixtures/layouts/repo-deleted.yaml b/tests/fixtures/layouts/repo-deleted.yaml
index 6e6c301..3a7f6b3 100644
--- a/tests/fixtures/layouts/repo-deleted.yaml
+++ b/tests/fixtures/layouts/repo-deleted.yaml
@@ -42,16 +42,18 @@
- job:
name: project-test1
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
branches: stable
- nodes:
- - name: controller
- label: label2
+ nodeset:
+ nodes:
+ - name: controller
+ label: label2
- job:
name: project-test2
diff --git a/tests/fixtures/layouts/smtp.yaml b/tests/fixtures/layouts/smtp.yaml
index 5ea75ce..0654448 100644
--- a/tests/fixtures/layouts/smtp.yaml
+++ b/tests/fixtures/layouts/smtp.yaml
@@ -48,16 +48,18 @@
- job:
name: project-test1
- nodes:
- - name: controller
- label: label1
+ nodeset:
+ nodes:
+ - name: controller
+ label: label1
- job:
name: project-test1
branches: stable
- nodes:
- - name: controller
- label: label2
+ nodeset:
+ nodes:
+ - name: controller
+ label: label2
- job:
name: project-test2
diff --git a/tests/fixtures/layouts/timer.yaml b/tests/fixtures/layouts/timer.yaml
index e1c4e77..8c0cc2b 100644
--- a/tests/fixtures/layouts/timer.yaml
+++ b/tests/fixtures/layouts/timer.yaml
@@ -30,9 +30,10 @@
- job:
name: project-bitrot
- nodes:
- - name: static
- label: ubuntu-xenial
+ nodeset:
+ nodes:
+ - name: static
+ label: ubuntu-xenial
- project:
name: org/project
diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py
index 3b5c206..f7d580c 100644
--- a/tests/unit/test_configloader.py
+++ b/tests/unit/test_configloader.py
@@ -304,7 +304,7 @@
def test_conflict_config(self):
tenant = self.sched.abide.tenants.get('tenant-one')
jobs = sorted(tenant.layout.jobs.keys())
- self.assertEquals(
+ self.assertEqual(
['base', 'noop', 'trusted-zuul.yaml-job',
'untrusted-zuul.yaml-job'],
jobs)
diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py
index 3793edc..9c45645 100755
--- a/tests/unit/test_executor.py
+++ b/tests/unit/test_executor.py
@@ -35,17 +35,17 @@
'Project %s commit for build %s #%s should '
'not have a detached HEAD' % (
project, build, number))
- self.assertEquals(repo.active_branch.name,
- state['branch'],
- 'Project %s commit for build %s #%s should '
- 'be on the correct branch' % (
- project, build, number))
+ self.assertEqual(repo.active_branch.name,
+ state['branch'],
+ 'Project %s commit for build %s #%s should '
+ 'be on the correct branch' % (
+ project, build, number))
if 'commit' in state:
- self.assertEquals(state['commit'],
- str(repo.commit('HEAD')),
- 'Project %s commit for build %s #%s should '
- 'be correct' % (
- project, build, number))
+ self.assertEqual(state['commit'],
+ str(repo.commit('HEAD')),
+ 'Project %s commit for build %s #%s should '
+ 'be correct' % (
+ project, build, number))
ref = repo.commit('HEAD')
repo_messages = set(
[c.message.strip() for c in repo.iter_commits(ref)])
@@ -93,7 +93,7 @@
self.waitUntilSettled()
- self.assertEquals(2, len(self.builds), "Two builds are running")
+ self.assertEqual(2, len(self.builds), "Two builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
@@ -133,7 +133,7 @@
self.waitUntilSettled()
- self.assertEquals(3, len(self.builds), "Three builds are running")
+ self.assertEqual(3, len(self.builds), "Three builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
@@ -194,7 +194,7 @@
self.waitUntilSettled()
- self.assertEquals(4, len(self.builds), "Four builds are running")
+ self.assertEqual(4, len(self.builds), "Four builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
@@ -283,7 +283,7 @@
time.sleep(1)
self.waitUntilSettled()
- self.assertEquals(1, len(self.builds), "One build is running")
+ self.assertEqual(1, len(self.builds), "One build is running")
upstream = self.getUpstreamRepos(projects)
states = [
@@ -326,7 +326,7 @@
time.sleep(1)
self.waitUntilSettled()
- self.assertEquals(2, len(self.builds), "Two builds are running")
+ self.assertEqual(2, len(self.builds), "Two builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
diff --git a/tests/unit/test_gerrit.py b/tests/unit/test_gerrit.py
index a369aff..5ce1aa3 100644
--- a/tests/unit/test_gerrit.py
+++ b/tests/unit/test_gerrit.py
@@ -63,11 +63,11 @@
result = gerrit.simpleQuery('project:openstack-infra/zuul')
_ssh_mock.assert_has_calls(calls)
- self.assertEquals(len(calls), _ssh_mock.call_count,
- '_ssh should be called %d times' % len(calls))
+ self.assertEqual(len(calls), _ssh_mock.call_count,
+ '_ssh should be called %d times' % len(calls))
self.assertIsNotNone(result, 'Result is not none')
- self.assertEquals(len(result), expected_patches,
- 'There must be %d patches.' % expected_patches)
+ self.assertEqual(len(result), expected_patches,
+ 'There must be %d patches.' % expected_patches)
def test_simple_query_pagination_new(self):
files = ['simple_query_pagination_new_1',
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index a088236..ebb5e1c 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -17,6 +17,7 @@
from testtools.matchers import MatchesRegex, StartsWith
import urllib
import time
+from unittest import skip
import git
@@ -685,6 +686,8 @@
# New timestamp should be greater than the old timestamp
self.assertLess(old, new)
+ # TODO(jlk): Make this a more generic test for unknown project
+ @skip("Skipped for rewrite of webhook handler")
@simple_layout('layouts/basic-github.yaml', driver='github')
def test_ping_event(self):
# Test valid ping
diff --git a/tests/unit/test_log_streamer.py b/tests/unit/test_log_streamer.py
index f47a8c8..c808540 100644
--- a/tests/unit/test_log_streamer.py
+++ b/tests/unit/test_log_streamer.py
@@ -22,6 +22,7 @@
import os.path
import socket
import tempfile
+import testtools
import threading
import time
@@ -34,7 +35,7 @@
def setUp(self):
super(TestLogStreamer, self).setUp()
- self.host = '0.0.0.0'
+ self.host = '::'
def startStreamer(self, port, root=None):
if not root:
@@ -46,16 +47,13 @@
streamer = self.startStreamer(port)
self.addCleanup(streamer.stop)
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- self.addCleanup(s.close)
- self.assertEqual(0, s.connect_ex((self.host, port)))
+ s = socket.create_connection((self.host, port))
s.close()
streamer.stop()
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- self.addCleanup(s.close)
- self.assertNotEqual(0, s.connect_ex((self.host, port)))
+ with testtools.ExpectedException(ConnectionRefusedError):
+ s = socket.create_connection((self.host, port))
s.close()
@@ -66,7 +64,7 @@
def setUp(self):
super(TestStreaming, self).setUp()
- self.host = '0.0.0.0'
+ self.host = '::'
self.streamer = None
self.stop_streamer = False
self.streaming_data = ''
@@ -80,8 +78,7 @@
root = tempfile.gettempdir()
self.streamer = zuul.lib.log_streamer.LogStreamer(None, self.host,
port, root)
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- s.connect((self.host, port))
+ s = socket.create_connection((self.host, port))
self.addCleanup(s.close)
req = '%s\n' % build_uuid
@@ -161,7 +158,7 @@
def runWSClient(self, build_uuid, event):
async def client(loop, build_uuid, event):
- uri = 'http://127.0.0.1:9000/console-stream'
+ uri = 'http://[::1]:9000/console-stream'
try:
session = aiohttp.ClientSession(loop=loop)
async with session.ws_connect(uri) as ws:
@@ -226,7 +223,7 @@
# Start the web server
web_server = zuul.web.ZuulWeb(
- listen_address='127.0.0.1', listen_port=9000,
+ listen_address='::', listen_port=9000,
gear_server='127.0.0.1', gear_port=self.gearman_server.port)
loop = asyncio.new_event_loop()
loop.set_debug(True)
@@ -237,8 +234,11 @@
self.addCleanup(web_server.stop)
# Wait until web server is started
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
- while s.connect_ex((self.host, 9000)):
+ while True:
+ try:
+ with socket.create_connection((self.host, 9000)):
+ break
+ except ConnectionRefusedError:
time.sleep(0.1)
# Start a thread with the websocket client
diff --git a/tests/unit/test_merger_repo.py b/tests/unit/test_merger_repo.py
index f815344..8aafabf 100644
--- a/tests/unit/test_merger_repo.py
+++ b/tests/unit/test_merger_repo.py
@@ -65,12 +65,12 @@
os.path.join(self.workspace_root, 'subdir', '.git')),
msg='Cloned over the submodule placeholder')
- self.assertEquals(
+ self.assertEqual(
os.path.join(self.upstream_root, 'org/project1'),
work_repo.createRepoObject().remotes[0].url,
message="Parent clone still point to upstream project1")
- self.assertEquals(
+ self.assertEqual(
os.path.join(self.upstream_root, 'org/project2'),
sub_repo.createRepoObject().remotes[0].url,
message="Sub repository points to upstream project2")
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 6dd8333..c457ff0 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -190,10 +190,12 @@
'timeout': 30,
'pre-run': 'base-pre',
'post-run': 'base-post',
- 'nodes': [{
- 'name': 'controller',
- 'label': 'base',
- }],
+ 'nodeset': {
+ 'nodes': [{
+ 'name': 'controller',
+ 'label': 'base',
+ }],
+ },
})
layout.addJob(base)
python27 = configloader.JobParser.fromYaml(tenant, layout, {
@@ -203,10 +205,12 @@
'parent': 'base',
'pre-run': 'py27-pre',
'post-run': ['py27-post-a', 'py27-post-b'],
- 'nodes': [{
- 'name': 'controller',
- 'label': 'new',
- }],
+ 'nodeset': {
+ 'nodes': [{
+ 'name': 'controller',
+ 'label': 'new',
+ }],
+ },
'timeout': 40,
})
layout.addJob(python27)
@@ -220,10 +224,12 @@
'pre-run': 'py27-diablo-pre',
'run': 'py27-diablo',
'post-run': 'py27-diablo-post',
- 'nodes': [{
- 'name': 'controller',
- 'label': 'old',
- }],
+ 'nodeset': {
+ 'nodes': [{
+ 'name': 'controller',
+ 'label': 'old',
+ }],
+ },
'timeout': 50,
})
layout.addJob(python27diablo)
@@ -800,14 +806,22 @@
self.db = model.TimeDataBase(self.tmp_root)
def test_timedatabase(self):
- self.assertEqual(self.db.getEstimatedTime('job-name'), 0)
- self.db.update('job-name', 50, 'SUCCESS')
- self.assertEqual(self.db.getEstimatedTime('job-name'), 50)
- self.db.update('job-name', 100, 'SUCCESS')
- self.assertEqual(self.db.getEstimatedTime('job-name'), 75)
+ pipeline = Dummy(layout=Dummy(tenant=Dummy(name='test-tenant')))
+ change = Dummy(project=Dummy(canonical_name='git.example.com/foo/bar'))
+ job = Dummy(name='job-name')
+ item = Dummy(pipeline=pipeline,
+ change=change)
+ build = Dummy(build_set=Dummy(item=item),
+ job=job)
+
+ self.assertEqual(self.db.getEstimatedTime(build), 0)
+ self.db.update(build, 50, 'SUCCESS')
+ self.assertEqual(self.db.getEstimatedTime(build), 50)
+ self.db.update(build, 100, 'SUCCESS')
+ self.assertEqual(self.db.getEstimatedTime(build), 75)
for x in range(10):
- self.db.update('job-name', 100, 'SUCCESS')
- self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
+ self.db.update(build, 100, 'SUCCESS')
+ self.assertEqual(self.db.getEstimatedTime(build), 100)
class TestGraph(BaseTestCase):
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index f33d964..2dcd9bf 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -617,7 +617,6 @@
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
- @skip("Disabled for early v3 development")
def _test_time_database(self, iteration):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
@@ -626,7 +625,7 @@
self.waitUntilSettled()
time.sleep(2)
- data = json.loads(self.sched.formatStatusJSON())
+ data = json.loads(self.sched.formatStatusJSON('tenant-one'))
found_job = None
for pipeline in data['pipelines']:
if pipeline['name'] != 'gate':
@@ -652,7 +651,6 @@
self.executor_server.release()
self.waitUntilSettled()
- @skip("Disabled for early v3 development")
def test_time_database(self):
"Test the time database"
@@ -2436,6 +2434,35 @@
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
+ def test_live_reconfiguration_abort(self):
+ # Raise an exception during reconfiguration and verify we
+ # still function.
+ self.executor_server.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ pipeline = tenant.layout.pipelines['gate']
+ change = pipeline.getAllItems()[0].change
+ # Set this to an invalid value to cause an exception during
+ # reconfiguration.
+ change.branch = None
+
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+
+ self.waitUntilSettled()
+ self.assertEqual(self.getJobFromHistory('project-merge').result,
+ 'ABORTED')
+ self.assertEqual(A.data['status'], 'NEW')
+ # The final report fails because of the invalid value set above.
+ self.assertEqual(A.reported, 1)
+
def test_live_reconfiguration_merge_conflict(self):
# A real-world bug: a change in a gate queue has a merge
# conflict and a job is added to its project while it's
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index c681305..94f169a 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -371,55 +371,6 @@
dict(name='project-test1', result='SUCCESS', changes='1,2'),
dict(name='project-test2', result='SUCCESS', changes='1,2')])
- def test_dynamic_dependent_pipeline(self):
- # Test dynamically adding a project to a
- # dependent pipeline for the first time
- self.executor_server.hold_jobs_in_build = True
-
- tenant = self.sched.abide.tenants.get('tenant-one')
- gate_pipeline = tenant.layout.pipelines['gate']
-
- in_repo_conf = textwrap.dedent(
- """
- - job:
- name: project-test1
-
- - job:
- name: project-test2
-
- - project:
- name: org/project
- gate:
- jobs:
- - project-test2
- """)
-
- in_repo_playbook = textwrap.dedent(
- """
- - hosts: all
- tasks: []
- """)
-
- file_dict = {'.zuul.yaml': in_repo_conf,
- 'playbooks/project-test2.yaml': in_repo_playbook}
- A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
- files=file_dict)
- A.addApproval('Approved', 1)
- self.fake_gerrit.addEvent(A.addApproval('Code-Review', 2))
- self.waitUntilSettled()
-
- items = gate_pipeline.getAllItems()
- self.assertEqual(items[0].change.number, '1')
- self.assertEqual(items[0].change.patchset, '1')
- self.assertTrue(items[0].live)
-
- self.executor_server.hold_jobs_in_build = False
- self.executor_server.release()
- self.waitUntilSettled()
-
- # Make sure the dynamic queue got cleaned up
- self.assertEqual(gate_pipeline.queues, [])
-
def test_in_repo_branch(self):
in_repo_conf = textwrap.dedent(
"""
@@ -544,6 +495,84 @@
dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'),
])
+ def test_yaml_list_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ job: foo
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1,
+ "A should report failure")
+ self.assertIn('not a list', A.messages[0],
+ "A should have a syntax error reported")
+
+ def test_yaml_dict_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1,
+ "A should report failure")
+ self.assertIn('not a dictionary', A.messages[0],
+ "A should have a syntax error reported")
+
+ def test_yaml_key_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test2
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1,
+ "A should report failure")
+ self.assertIn('has more than one key', A.messages[0],
+ "A should have a syntax error reported")
+
+ def test_yaml_unknown_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - foobar:
+ foo: bar
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1,
+ "A should report failure")
+ self.assertIn('not recognized', A.messages[0],
+ "A should have a syntax error reported")
+
def test_untrusted_syntax_error(self):
in_repo_conf = textwrap.dedent(
"""
@@ -719,6 +748,48 @@
self.assertIn('appears multiple times', A.messages[0],
"A should have a syntax error reported")
+ def test_secret_not_found_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: test
+ secrets: does-not-exist
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1,
+ "A should report failure")
+ self.assertIn('secret "does-not-exist" was not found', A.messages[0],
+ "A should have a syntax error reported")
+
+ def test_nodeset_not_found_error(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: test
+ nodeset: does-not-exist
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(A.reported, 1,
+ "A should report failure")
+ self.assertIn('nodeset "does-not-exist" was not found', A.messages[0],
+ "A should have a syntax error reported")
+
def test_multi_repo(self):
downstream_repo_conf = textwrap.dedent(
"""
@@ -846,6 +917,124 @@
"C should have an error reported")
+class TestInRepoJoin(ZuulTestCase):
+ # In this config, org/project is not a member of any pipelines, so
+ # that we may test the changes that cause it to join them.
+
+ tenant_config_file = 'config/in-repo-join/main.yaml'
+
+ def test_dynamic_dependent_pipeline(self):
+ # Test dynamically adding a project to a
+ # dependent pipeline for the first time
+ self.executor_server.hold_jobs_in_build = True
+
+ tenant = self.sched.abide.tenants.get('tenant-one')
+ gate_pipeline = tenant.layout.pipelines['gate']
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test1
+
+ - job:
+ name: project-test2
+
+ - project:
+ name: org/project
+ gate:
+ jobs:
+ - project-test2
+ """)
+
+ in_repo_playbook = textwrap.dedent(
+ """
+ - hosts: all
+ tasks: []
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf,
+ 'playbooks/project-test2.yaml': in_repo_playbook}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ items = gate_pipeline.getAllItems()
+ self.assertEqual(items[0].change.number, '1')
+ self.assertEqual(items[0].change.patchset, '1')
+ self.assertTrue(items[0].live)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ # Make sure the dynamic queue got cleaned up
+ self.assertEqual(gate_pipeline.queues, [])
+
+ def test_dynamic_dependent_pipeline_failure(self):
+ # Test that a change behind a failing change adding a project
+ # to a dependent pipeline is dequeued.
+ self.executor_server.hold_jobs_in_build = True
+
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: project-test1
+
+ - project:
+ name: org/project
+ gate:
+ jobs:
+ - project-test1
+ """)
+
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
+ files=file_dict)
+ self.executor_server.failJob('project-test1', A)
+ A.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ B.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+
+ self.orderedRelease()
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 2,
+ "A should report start and failure")
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.reported, 1,
+ "B should report start")
+ self.assertHistory([
+ dict(name='project-test1', result='FAILURE', changes='1,1'),
+ dict(name='project-test1', result='ABORTED', changes='1,1 2,1'),
+ ], ordered=False)
+
+ def test_dynamic_dependent_pipeline_absent(self):
+ # Test that a series of dependent changes don't report merge
+ # failures to a pipeline they aren't in.
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
+ B.setDependsOn(A, 1)
+
+ A.addApproval('Code-Review', 2)
+ A.addApproval('Approved', 1)
+ B.addApproval('Code-Review', 2)
+ self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 0,
+ "A should not report")
+ self.assertEqual(A.data['status'], 'NEW')
+ self.assertEqual(B.reported, 0,
+ "B should not report")
+ self.assertEqual(B.data['status'], 'NEW')
+ self.assertHistory([])
+
+
class TestAnsible(AnsibleZuulTestCase):
# A temporary class to hold new tests while others are disabled
@@ -1255,24 +1444,25 @@
class TestMaxNodesPerJob(AnsibleZuulTestCase):
tenant_config_file = 'config/multi-tenant/main.yaml'
- def test_max_nodes_reached(self):
+ def test_max_timeout_exceeded(self):
in_repo_conf = textwrap.dedent(
"""
- job:
name: test-job
- nodes:
- - name: node01
- label: fake
- - name: node02
- label: fake
- - name: node03
- label: fake
- - name: node04
- label: fake
- - name: node05
- label: fake
- - name: node06
- label: fake
+ nodeset:
+ nodes:
+ - name: node01
+ label: fake
+ - name: node02
+ label: fake
+ - name: node03
+ label: fake
+ - name: node04
+ label: fake
+ - name: node05
+ label: fake
+ - name: node06
+ label: fake
""")
file_dict = {'.zuul.yaml': in_repo_conf}
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
@@ -1290,6 +1480,32 @@
"B should not fail because of nodes limit")
+class TestMaxTimeout(AnsibleZuulTestCase):
+ tenant_config_file = 'config/multi-tenant/main.yaml'
+
+ def test_max_nodes_reached(self):
+ in_repo_conf = textwrap.dedent(
+ """
+ - job:
+ name: test-job
+ timeout: 3600
+ """)
+ file_dict = {'.zuul.yaml': in_repo_conf}
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertIn('The job "test-job" exceeds tenant max-job-timeout',
+ A.messages[0], "A should fail because of timeout limit")
+
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
+ files=file_dict)
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertNotIn("exceeds tenant max-job-timeout", B.messages[0],
+ "B should not fail because of timeout limit")
+
+
class TestBaseJobs(ZuulTestCase):
tenant_config_file = 'config/base-jobs/main.yaml'
diff --git a/tools/run-migration.sh b/tools/run-migration.sh
new file mode 100755
index 0000000..6c7e250
--- /dev/null
+++ b/tools/run-migration.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright (c) 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stupid script I'm using to test migration script locally
+# Assumes project-config is adjacent to zuul and has the mapping file
+
+BASE_DIR=$(cd $(dirname $0)/../..; pwd)
+cd $BASE_DIR/project-config
+python3 $BASE_DIR/zuul/zuul/cmd/migrate.py --mapping=zuul/mapping.yaml \
+ zuul/layout.yaml jenkins/jobs nodepool/nodepool.yaml .
diff --git a/zuul/ansible/action/normal.py b/zuul/ansible/action/normal.py
index 34df21d..152f13f 100644
--- a/zuul/ansible/action/normal.py
+++ b/zuul/ansible/action/normal.py
@@ -40,7 +40,7 @@
or self._play_context.remote_addr.startswith('127.')
or self._task.delegate_to == 'localhost'
or (self._task.delegate_to
- and self._task.delegate_to.startswtih('127.'))):
+ and self._task.delegate_to.startswith('127.'))):
if not self.dispatch_handler():
raise AnsibleError("Executing local code is prohibited")
return super(ActionModule, self).run(tmp, task_vars)
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index 6aec86b..0a266df 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -36,23 +36,6 @@
LOG_STREAM_PORT = 19885
-def linesplit(socket):
- buff = socket.recv(4096).decode("utf-8")
- buffering = True
- while buffering:
- if "\n" in buff:
- (line, buff) = buff.split("\n", 1)
- yield line + "\n"
- else:
- more = socket.recv(4096).decode("utf-8")
- if not more:
- buffering = False
- else:
- buff += more
- if buff:
- yield buff
-
-
def zuul_filter_result(result):
"""Remove keys from shell/command output.
@@ -122,6 +105,7 @@
self._logger = logging.getLogger('zuul.executor.ansible')
def _log(self, msg, ts=None, job=True, executor=False, debug=False):
+ msg = msg.rstrip()
if job:
now = ts or datetime.datetime.now()
self._logger.info("{now} | {msg}".format(now=now, msg=msg))
@@ -144,19 +128,36 @@
continue
msg = "%s\n" % log_id
s.send(msg.encode("utf-8"))
- for line in linesplit(s):
- if "[Zuul] Task exit code" in line:
- return
- elif self._streamers_stop and "[Zuul] Log not found" in line:
- return
- elif "[Zuul] Log not found" in line:
- # don't output this line
- pass
+ buff = s.recv(4096).decode("utf-8")
+ buffering = True
+ while buffering:
+ if "\n" in buff:
+ (line, buff) = buff.split("\n", 1)
+ done = self._log_streamline(host, line)
+ if done:
+ return
else:
- ts, ln = line.split(' | ', 1)
- ln = ln.strip()
+ more = s.recv(4096).decode("utf-8")
+ if not more:
+ buffering = False
+ else:
+ buff += more
+ if buff:
+ self._log_streamline(host, line)
- self._log("%s | %s " % (host, ln), ts=ts)
+ def _log_streamline(self, host, line):
+ if "[Zuul] Task exit code" in line:
+ return True
+ elif self._streamers_stop and "[Zuul] Log not found" in line:
+ return True
+ elif "[Zuul] Log not found" in line:
+ # don't output this line
+ return False
+ else:
+ ts, ln = line.split(' | ', 1)
+
+ self._log("%s | %s " % (host, ln), ts=ts)
+ return False
def v2_playbook_on_start(self, playbook):
self._playbook_name = os.path.splitext(playbook._file_name)[0]
@@ -201,10 +202,11 @@
msg = u"PLAY [{name}]".format(name=name)
self._log(msg)
- # Log an extra blank line to get space after each play
- self._log("")
def v2_playbook_on_task_start(self, task, is_conditional):
+ # Log an extra blank line to get space before each task
+ self._log("")
+
self._task = task
if self._play.strategy != 'free':
@@ -275,7 +277,7 @@
if is_localhost:
for line in stdout_lines:
hostname = self._get_hostname(result)
- self._log("%s | %s " % (hostname, line.strip()))
+ self._log("%s | %s " % (hostname, line))
def v2_runner_on_failed(self, result, ignore_errors=False):
result_dict = dict(result._result)
@@ -292,18 +294,24 @@
if result._task.loop and 'results' in result_dict:
# items have their own events
pass
- elif (result_dict.get('msg') == 'MODULE FAILURE' and
- 'module_stdout' in result_dict):
- self._log_message(
- result, status='MODULE FAILURE',
- msg=result_dict['module_stdout'])
+ elif (result_dict.get('msg') == 'MODULE FAILURE'):
+ if 'module_stdout' in result_dict:
+ self._log_message(
+ result, status='MODULE FAILURE',
+ msg=result_dict['module_stdout'])
+ elif 'exception' in result_dict:
+ self._log_message(
+ result, status='MODULE FAILURE',
+ msg=result_dict['exception'])
+ elif 'module_stderr' in result_dict:
+ self._log_message(
+ result, status='MODULE FAILURE',
+ msg=result_dict['module_stderr'])
else:
self._log_message(
result=result, status='ERROR', result_dict=result_dict)
if ignore_errors:
self._log_message(result, "Ignoring Errors", status="ERROR")
- # Log an extra blank line to get space after each task
- self._log("")
def v2_runner_on_skipped(self, result):
if result._task.loop:
@@ -314,8 +322,6 @@
if reason:
# No reason means it's an item, which we'll log differently
self._log_message(result, status='skipping', msg=reason)
- # Log an extra blank line to get space after each skip
- self._log("")
def v2_runner_item_on_skipped(self, result):
reason = result._result.get('skip_reason')
@@ -365,20 +371,39 @@
# items have their own events
pass
- elif (result_dict.get('msg') == 'MODULE FAILURE' and
- 'module_stdout' in result_dict):
- self._log_message(
- result, status='MODULE FAILURE',
- msg=result_dict['module_stdout'])
+ elif (result_dict.get('msg') == 'MODULE FAILURE'):
+ if 'module_stdout' in result_dict:
+ self._log_message(
+ result, status='MODULE FAILURE',
+ msg=result_dict['module_stdout'])
+ elif 'exception' in result_dict:
+ self._log_message(
+ result, status='MODULE FAILURE',
+ msg=result_dict['exception'])
+ elif 'module_stderr' in result_dict:
+ self._log_message(
+ result, status='MODULE FAILURE',
+ msg=result_dict['module_stderr'])
elif (len([key for key in result_dict.keys()
if not key.startswith('_ansible')]) == 1):
# this is a debug statement, handle it special
for key in [k for k in result_dict.keys()
if k.startswith('_ansible')]:
del result_dict[key]
- self._log_message(
- msg=json.dumps(result_dict, indent=2, sort_keys=True),
- status=status, result=result)
+ keyname = next(iter(result_dict.keys()))
+ # If it has msg, that means it was like:
+ #
+ # debug:
+ # msg: Some debug text the user was looking for
+ #
+ # So we log it with self._log to get just the raw string the
+ # user provided.
+ if keyname == 'msg':
+ self._log(msg=result_dict['msg'])
+ else:
+ self._log_message(
+ msg=json.dumps(result_dict, indent=2, sort_keys=True),
+ status=status, result=result)
elif result._task.action not in ('command', 'shell'):
if 'msg' in result_dict:
self._log_message(msg=result_dict['msg'],
@@ -391,16 +416,14 @@
for res in result_dict['results']:
self._log_message(
result,
- "Runtime: {delta} Start: {start} End: {end}".format(**res))
+ "Runtime: {delta}".format(**res))
elif result_dict.get('msg') == 'All items completed':
self._log_message(result, result_dict['msg'])
else:
self._log_message(
result,
- "Runtime: {delta} Start: {start} End: {end}".format(
+ "Runtime: {delta}".format(
**result_dict))
- # Log an extra blank line to get space after each task
- self._log("")
def v2_runner_item_on_ok(self, result):
result_dict = dict(result._result)
@@ -430,12 +453,11 @@
if isinstance(result_dict['item'], str):
self._log_message(
result,
- "Item: {item} Runtime: {delta}"
- " Start: {start} End: {end}".format(**result_dict))
+ "Item: {item} Runtime: {delta}".format(**result_dict))
else:
self._log_message(
result,
- "Item: Runtime: {delta} Start: {start} End: {end}".format(
+ "Item: Runtime: {delta}".format(
**result_dict))
if self._deferred_result:
@@ -462,10 +484,11 @@
if self._deferred_result:
self._process_deferred(result)
- # Log an extra blank line to get space after each task
- self._log("")
def v2_playbook_on_stats(self, stats):
+ # Add a spacer line before the stats so that there will be a line
+ # between the last task and the recap
+ self._log("")
self._log("PLAY RECAP")
@@ -554,7 +577,7 @@
msg = result_dict['msg']
result_dict = None
if msg:
- msg_lines = msg.strip().split('\n')
+ msg_lines = msg.rstrip().split('\n')
if len(msg_lines) > 1:
self._log("{host} | {status}:".format(
host=hostname, status=status))
diff --git a/zuul/ansible/filter/zuul_filters.py b/zuul/ansible/filter/zuul_filters.py
new file mode 100644
index 0000000..4304d51
--- /dev/null
+++ b/zuul/ansible/filter/zuul_filters.py
@@ -0,0 +1,63 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+def zuul_legacy_vars(zuul):
+ # omitted:
+ # ZUUL_URL
+ # ZUUL_REF
+ # ZUUL_COMMIT
+
+ short_name = zuul['project']['name'].split('/')[-1]
+ params = dict(ZUUL_UUID=zuul['build'],
+ ZUUL_PROJECT=zuul['project']['name'],
+ ZUUL_SHORT_PROJECT_NAME=short_name,
+ ZUUL_PIPELINE=zuul['pipeline'],
+ ZUUL_VOTING=zuul['voting'],
+ WORKSPACE='/home/zuul')
+ if 'branch' in zuul:
+ params['ZUUL_BRANCH'] = zuul['branch']
+
+ if 'change' in zuul:
+ changes_str = '^'.join(
+ ['%s:%s:refs/changes/%s/%s/%s' % (
+ i['project']['name'],
+ i['branch'],
+ str(i['change'])[:-2:],
+ i['change'],
+ i['patchset'])
+ for i in zuul['items']])
+ params['ZUUL_CHANGES'] = changes_str
+
+ change_ids = ' '.join(['%s,%s' % (i['change'], i['patchset'])
+ for i in zuul['items']])
+ params['ZUUL_CHANGE_IDS'] = change_ids
+ params['ZUUL_CHANGE'] = str(zuul['change'])
+ params['ZUUL_PATCHSET'] = str(zuul['patchset'])
+
+ if 'newrev' in zuul or 'oldrev' in zuul:
+ params['ZUUL_REFNAME'] = zuul['ref']
+ params['ZUUL_OLDREV'] = zuul.get('oldrev', '0' * 40)
+ params['ZUUL_NEWREV'] = zuul.get('newrev', '0' * 40)
+
+ params['TOX_TESTENV_PASSENV'] = ' '.join(params.keys())
+ return params
+
+
+class FilterModule(object):
+
+ def filters(self):
+ return {
+ 'zuul_legacy_vars': zuul_legacy_vars,
+ }
diff --git a/zuul/ansible/library/command.py b/zuul/ansible/library/command.py
index f701b48..0fc6129 100644
--- a/zuul/ansible/library/command.py
+++ b/zuul/ansible/library/command.py
@@ -159,9 +159,14 @@
# Jenkins format but with microsecond resolution instead of
# millisecond. It is kept so log parsing/formatting remains
# consistent.
- ts = datetime.datetime.now()
- outln = '%s | %s' % (ts, ln)
- self.logfile.write(outln.encode('utf-8'))
+ ts = str(datetime.datetime.now()).encode('utf-8')
+ if not isinstance(ln, bytes):
+ try:
+ ln = ln.encode('utf-8')
+ except Exception:
+ ln = repr(ln).encode('utf-8') + b'\n'
+ outln = b'%s | %s' % (ts, ln)
+ self.logfile.write(outln)
def follow(fd, log_uuid):
diff --git a/zuul/ansible/logconfig.py b/zuul/ansible/logconfig.py
index 7c3507b..7ef43a8 100644
--- a/zuul/ansible/logconfig.py
+++ b/zuul/ansible/logconfig.py
@@ -13,6 +13,7 @@
# under the License.
import abc
+import copy
import logging.config
import json
import os
@@ -161,14 +162,15 @@
logging.config.dictConfig(self._config)
def writeJson(self, filename: str):
- open(filename, 'w').write(json.dumps(self._config, indent=2))
+ with open(filename, 'w') as f:
+ f.write(json.dumps(self._config, indent=2))
class JobLoggingConfig(DictLoggingConfig):
def __init__(self, config=None, job_output_file=None):
if not config:
- config = _DEFAULT_JOB_LOGGING_CONFIG.copy()
+ config = copy.deepcopy(_DEFAULT_JOB_LOGGING_CONFIG)
super(JobLoggingConfig, self).__init__(config=config)
if job_output_file:
self.job_output_file = job_output_file
@@ -190,7 +192,7 @@
def __init__(self, config=None, server=None):
if not config:
- config = _DEFAULT_SERVER_LOGGING_CONFIG.copy()
+ config = copy.deepcopy(_DEFAULT_SERVER_LOGGING_CONFIG)
super(ServerLoggingConfig, self).__init__(config=config)
if server:
self.server = server
@@ -206,7 +208,7 @@
# config above because we're templating out the filename. Also, we
# only want to add the handler if we're actually going to use it.
for name, handler in _DEFAULT_SERVER_FILE_HANDLERS.items():
- server_handler = handler.copy()
+ server_handler = copy.deepcopy(handler)
server_handler['filename'] = server_handler['filename'].format(
server=server)
self._config['handlers'][name] = server_handler
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index 06ef0ba..63c621d 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -82,7 +82,7 @@
self.log.info("Starting log streamer")
streamer = zuul.lib.log_streamer.LogStreamer(
- self.user, '0.0.0.0', self.finger_port, self.job_dir)
+ self.user, '::', self.finger_port, self.job_dir)
# Keep running until the parent dies:
pipe_read = os.fdopen(pipe_read)
diff --git a/zuul/cmd/migrate.py b/zuul/cmd/migrate.py
new file mode 100644
index 0000000..92b5963
--- /dev/null
+++ b/zuul/cmd/migrate.py
@@ -0,0 +1,1496 @@
+#!/usr/bin/env python
+
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# TODO(mordred):
+# * Read and apply filters from the jobs: section
+# * Figure out shared job queues
+# * Emit job definitions
+# * figure out from builders whether or not it's a normal job or a
+# a devstack-legacy job
+# * Handle emitting arbitrary tox jobs (see tox-py27dj18)
+
+import argparse
+import collections
+import copy
+import itertools
+import getopt
+import logging
+import os
+import subprocess
+import tempfile
+import re
+from typing import Any, Dict, List, Optional # flake8: noqa
+
+import jenkins_jobs.builder
+from jenkins_jobs.formatter import deep_format
+import jenkins_jobs.formatter
+from jenkins_jobs.parser import matches
+import jenkins_jobs.parser
+import yaml
+
+JOBS_BY_ORIG_TEMPLATE = {} # type: ignore
+SUFFIXES = [] # type: ignore
+ENVIRONMENT = '{{ zuul | zuul_legacy_vars }}'
+DESCRIPTION = """Migrate zuul v2 and Jenkins Job Builder to Zuul v3.
+
+This program takes a zuul v2 layout.yaml and a collection of Jenkins Job
+Builder job definitions and transforms them into a Zuul v3 config. An
+optional mapping config can be given that defines how to map old jobs
+to new jobs.
+"""
+
+def deal_with_shebang(data):
+ # Ansible shell blocks do not honor shebang lines. That's fine - but
+ # we do have a bunch of scripts that have either nothing, -x, -xe,
+ # -ex or -eux. Transform those into leading set commands
+ if not data.startswith('#!'):
+ return (None, data)
+ data_lines = data.split('\n')
+ data_lines.reverse()
+ shebang = data_lines.pop()
+ split_line = shebang.split()
+ # Strip the # and the !
+ executable = split_line[0][2:]
+ if executable == '/bin/sh':
+ # Ansible default
+ executable = None
+ if len(split_line) > 1:
+ flag_x = False
+ flag_e = False
+ flag_u = False
+ optlist, args = getopt.getopt(split_line[1:], 'uex')
+ for opt, _ in optlist:
+ if opt == '-x':
+ flag_x = True
+ elif opt == '-e':
+ flag_e = True
+ elif opt == '-u':
+ flag_u = True
+
+ if flag_x:
+ data_lines.append('set -x')
+ if flag_e:
+ data_lines.append('set -e')
+ if flag_u:
+ data_lines.append('set -u')
+ data_lines.reverse()
+ data = '\n'.join(data_lines).lstrip()
+ return (executable, data)
+
+
+def _extract_from_vars(line):
+ # export PROJECTS="openstack/blazar $PROJECTS"
+ # export DEVSTACK_PROJECT_FROM_GIT=python-swiftclient
+ # export DEVSTACK_PROJECT_FROM_GIT="python-octaviaclient"
+ # export DEVSTACK_PROJECT_FROM_GIT+=",glean"
+ projects = []
+ line = line.replace('"', '').replace('+', '').replace(',', ' ')
+ if (line.startswith('export PROJECTS') or
+ line.startswith('export DEVSTACK_PROJECT_FROM_GIT')):
+ nothing, project_string = line.split('=')
+ project_string = project_string.replace('$PROJECTS', '').strip()
+ projects = project_string.split()
+ return projects
+
+
+def extract_projects(data):
+ # clonemap:
+ # - name: openstack/windmill
+ # dest: .
+ # EOF
+ projects = []
+ data_lines = data.split('\n')
+ in_clonemap = False
+ in_clonemap_cli = False
+ for line in data_lines:
+ line = line.strip()
+ if line == 'clonemap:':
+ in_clonemap = True
+ continue
+ elif line == 'EOF':
+ in_clonemap = False
+ continue
+ elif line.startswith('/usr/zuul-env/bin/zuul-cloner'):
+ in_clonemap_cli = True
+ continue
+ elif in_clonemap_cli and not line.endswith('\\'):
+ in_clonemap_cli = False
+ continue
+ if in_clonemap:
+ if line.startswith('- name:'):
+ garbage, project = line.split(':')
+ project = project.strip().replace("'", '').replace('"', '')
+ if project == '$ZUUL_PROJECT':
+ continue
+ projects.append(project)
+ elif in_clonemap_cli and line.startswith('openstack/'):
+ line = line.replace('\\', '').strip()
+ projects.append(line)
+ elif in_clonemap_cli:
+ continue
+ else:
+ projects.extend(_extract_from_vars(line))
+ return projects
+
+
+def expand_project_names(required, full):
+ projects = []
+ for name in full:
+ org, repo = name.split('/')
+ if repo in required or name in required:
+ projects.append(name)
+ return projects
+
+
+# from :
+# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data flake8: noqa
+def should_use_block(value):
+ for c in u"\u000a\u000d\u001c\u001d\u001e\u0085\u2028\u2029":
+ if c in value:
+ return True
+ return False
+
+
+def my_represent_scalar(self, tag, value, style=None):
+ if style is None:
+ if should_use_block(value):
+ style='|'
+ else:
+ style = self.default_style
+
+ node = yaml.representer.ScalarNode(tag, value, style=style)
+ if self.alias_key is not None:
+ self.represented_objects[self.alias_key] = node
+ return node
+
+def project_representer(dumper, data):
+ return dumper.represent_mapping('tag:yaml.org,2002:map',
+ data.items())
+
+
+def construct_yaml_map(self, node):
+ data = collections.OrderedDict()
+ yield data
+ value = self.construct_mapping(node)
+
+ if isinstance(node, yaml.MappingNode):
+ self.flatten_mapping(node)
+ else:
+ raise yaml.constructor.ConstructorError(
+ None, None,
+ 'expected a mapping node, but found %s' % node.id,
+ node.start_mark)
+
+ mapping = collections.OrderedDict()
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=False)
+ try:
+ hash(key)
+ except TypeError as exc:
+ raise yaml.constructor.ConstructorError(
+ 'while constructing a mapping', node.start_mark,
+ 'found unacceptable key (%s)' % exc, key_node.start_mark)
+ value = self.construct_object(value_node, deep=False)
+ mapping[key] = value
+ data.update(mapping)
+
+
+class IndentedEmitter(yaml.emitter.Emitter):
+ def expect_block_sequence(self):
+ self.increase_indent(flow=False, indentless=False)
+ self.state = self.expect_first_block_sequence_item
+
+
+class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer,
+ yaml.representer.Representer, yaml.resolver.Resolver):
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ IndentedEmitter.__init__(
+ self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode,
+ line_break=line_break)
+ yaml.serializer.Serializer.__init__(
+ self, encoding=encoding,
+ explicit_start=explicit_start,
+ explicit_end=explicit_end,
+ version=version, tags=tags)
+ yaml.representer.Representer.__init__(
+ self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ yaml.resolver.Resolver.__init__(self)
+
+
+def ordered_load(stream, *args, **kwargs):
+ return yaml.load(stream=stream, *args, **kwargs)
+
+
+def ordered_dump(data, stream=None, *args, **kwargs):
+ dumper = IndentedDumper
+ # We need to do this because of how template expasion into a project
+ # works. Without it, we end up with YAML references to the expanded jobs.
+ dumper.ignore_aliases = lambda self, data: True
+
+ output = yaml.dump(
+ data, default_flow_style=False,
+ Dumper=dumper, width=80, *args, **kwargs).replace(
+ '\n -', '\n\n -')
+ if stream:
+ stream.write(output)
+ else:
+ return output
+
+
+def get_single_key(var):
+ if isinstance(var, str):
+ return var
+ elif isinstance(var, list):
+ return var[0]
+ return list(var.keys())[0]
+
+
+def has_single_key(var):
+ if isinstance(var, list):
+ return len(var) == 1
+ if isinstance(var, str):
+ return True
+ dict_keys = list(var.keys())
+ if len(dict_keys) != 1:
+ return False
+ if var[get_single_key(var)]:
+ return False
+ return True
+
+
+def combination_matches(combination, match_combinations):
+ """
+ Checks if the given combination is matches for any of the given combination
+ globs, being those a set of combinations where if a key is missing, it's
+ considered matching
+
+ (key1=2, key2=3)
+
+ would match the combination match:
+ (key2=3)
+
+ but not:
+ (key1=2, key2=2)
+ """
+ for cmatch in match_combinations:
+ for key, val in combination.items():
+ if cmatch.get(key, val) != val:
+ break
+ else:
+ return True
+ return False
+
+
+def expandYamlForTemplateJob(self, project, template, jobs_glob=None):
+ dimensions = []
+ template_name = template['name']
+ orig_template = copy.deepcopy(template)
+
+ # reject keys that are not useful during yaml expansion
+ for k in ['jobs']:
+ project.pop(k)
+ excludes = project.pop('exclude', [])
+ for (k, v) in project.items():
+ tmpk = '{{{0}}}'.format(k)
+ if tmpk not in template_name:
+ continue
+ if type(v) == list:
+ dimensions.append(zip([k] * len(v), v))
+ # XXX somewhat hackish to ensure we actually have a single
+ # pass through the loop
+ if len(dimensions) == 0:
+ dimensions = [(("", ""),)]
+
+ for values in itertools.product(*dimensions):
+ params = copy.deepcopy(project)
+ params = self.applyDefaults(params, template)
+
+ expanded_values = {}
+ for (k, v) in values:
+ if isinstance(v, dict):
+ inner_key = next(iter(v))
+ expanded_values[k] = inner_key
+ expanded_values.update(v[inner_key])
+ else:
+ expanded_values[k] = v
+
+ params.update(expanded_values)
+ params = deep_format(params, params)
+ if combination_matches(params, excludes):
+ log = logging.getLogger("zuul.Migrate.YamlParser")
+ log.debug('Excluding combination %s', str(params))
+ continue
+
+ allow_empty_variables = self.config \
+ and self.config.has_section('job_builder') \
+ and self.config.has_option(
+ 'job_builder', 'allow_empty_variables') \
+ and self.config.getboolean(
+ 'job_builder', 'allow_empty_variables')
+
+ for key in template.keys():
+ if key not in params:
+ params[key] = template[key]
+
+ params['template-name'] = template_name
+ project_name = params['name']
+ params['name'] = '$ZUUL_SHORT_PROJECT_NAME'
+ expanded = deep_format(template, params, allow_empty_variables)
+
+ job_name = expanded.get('name')
+ templated_job_name = job_name
+ if job_name:
+ job_name = job_name.replace(
+ '$ZUUL_SHORT_PROJECT_NAME', project_name)
+ expanded['name'] = job_name
+ if jobs_glob and not matches(job_name, jobs_glob):
+ continue
+
+ self.formatDescription(expanded)
+ expanded['orig_template'] = orig_template
+ expanded['template_name'] = template_name
+ self.jobs.append(expanded)
+ JOBS_BY_ORIG_TEMPLATE[templated_job_name] = expanded
+
+jenkins_jobs.parser.YamlParser.expandYamlForTemplateJob = \
+ expandYamlForTemplateJob
+
+
+class JJB(jenkins_jobs.builder.Builder):
+ def __init__(self):
+ self.global_config = None
+ self._plugins_list = []
+
+ def expandComponent(self, component_type, component, template_data):
+ component_list_type = component_type + 's'
+ new_components = []
+ if isinstance(component, dict):
+ name, component_data = next(iter(component.items()))
+ if template_data:
+ component_data = jenkins_jobs.formatter.deep_format(
+ component_data, template_data, True)
+ else:
+ name = component
+ component_data = {}
+
+ new_component = self.parser.data.get(component_type, {}).get(name)
+ if new_component:
+ for new_sub_component in new_component[component_list_type]:
+ new_components.extend(
+ self.expandComponent(component_type,
+ new_sub_component, component_data))
+ else:
+ new_components.append({name: component_data})
+ return new_components
+
+ def expandMacros(self, job):
+ for component_type in ['builder', 'publisher', 'wrapper']:
+ component_list_type = component_type + 's'
+ new_components = []
+ for new_component in job.get(component_list_type, []):
+ new_components.extend(self.expandComponent(component_type,
+ new_component, {}))
+ job[component_list_type] = new_components
+
+
+class OldProject:
+ def __init__(self, name, gate_jobs):
+ self.name = name
+ self.gate_jobs = gate_jobs
+
+
+class OldJob:
+ def __init__(self, name):
+ self.name = name
+ self.queue_name = None
+
+ def __repr__(self):
+ return self.name
+
+
+class Job:
+
+ log = logging.getLogger("zuul.Migrate")
+
+ def __init__(self,
+ orig: str,
+ name: str=None,
+ content: Dict[str, Any]=None,
+ vars: Dict[str, str]=None,
+ nodes: List[str]=None,
+ parent=None) -> None:
+ self.orig = orig
+ self.voting = True
+ self.name = name
+ self.content = content.copy() if content else None
+ self.vars = vars or {}
+ self.required_projects = [] # type: ignore
+ self.nodes = nodes or []
+ self.parent = parent
+ self.branch = None
+ self.files = None
+ self.jjb_job = None
+ self.emit = True
+
+ if self.content and not self.name:
+ self.name = get_single_key(content)
+ if not self.name:
+ self.name = self.orig
+ self.name = self.name.replace('-{name}', '').replace('{name}-', '')
+
+ for suffix in SUFFIXES:
+ suffix = '-{suffix}'.format(suffix=suffix)
+
+ if self.name.endswith(suffix):
+ self.name = self.name.replace(suffix, '')
+
+ def _stripNodeName(self, node):
+ node_key = '-{node}'.format(node=node)
+ self.name = self.name.replace(node_key, '')
+
+ def setNoEmit(self):
+ self.emit = False
+
+ def setVars(self, vars):
+ self.vars = vars
+
+ def setParent(self, parent):
+ self.parent = parent
+
+ def extractNode(self, default_node, labels):
+ matching_label = None
+ for label in labels:
+ if label in self.orig:
+ if not matching_label:
+ matching_label = label
+ elif len(label) > len(matching_label):
+ matching_label = label
+
+ if matching_label:
+ if matching_label == default_node:
+ self._stripNodeName(matching_label)
+ else:
+ self.nodes.append(matching_label)
+
+ def getDepends(self):
+ return [self.parent.name]
+
+ def getNodes(self):
+ return self.nodes
+
+ def addJJBJob(self, jobs):
+ if '{name}' in self.orig:
+ self.jjb_job = JOBS_BY_ORIG_TEMPLATE[self.orig.format(
+ name='$ZUUL_SHORT_PROJECT_NAME')]
+ else:
+ self.jjb_job = jobs[self.orig]
+
+ def getTimeout(self):
+ if self.jjb_job:
+ for wrapper in self.jjb_job.get('wrappers', []):
+ if isinstance(wrapper, dict):
+ build_timeout = wrapper.get('timeout')
+ if isinstance(build_timeout, dict):
+ timeout = build_timeout.get('timeout')
+ if timeout is not None:
+ timeout = int(timeout) * 60
+
+ @property
+ def short_name(self):
+ return self.name.replace('legacy-', '')
+
+ @property
+ def job_path(self):
+ return 'playbooks/legacy/{name}'.format(name=self.short_name)
+
+ def _getRsyncOptions(self, source):
+ # If the source starts with ** then we want to match any
+ # number of directories, so don't anchor the include filter.
+ # If it does not start with **, then the intent is likely to
+ # at least start by matching an immediate file or subdirectory
+ # (even if later we have a ** in the middle), so in this case,
+ # anchor it to the root of the transfer (the workspace).
+ if not source.startswith('**'):
+ source = os.path.join('/', source)
+ # These options mean: include the thing we want, include any
+ # directories (so that we continue to search for the thing we
+ # want no matter how deep it is), exclude anything that
+ # doesn't match the thing we want or is a directory, then get
+ # rid of empty directories left over at the end.
+ rsync_opts = ['--include="%s"' % source,
+ '--include="*/"',
+ '--exclude="*"',
+ '--prune-empty-dirs']
+ return rsync_opts
+
+ def _makeSCPTask(self, publisher):
+ # NOTE(mordred) About docs-draft manipulation:
+ # The target of html/ was chosen to put the node contents into the
+ # html dir inside of logs such that if the node's contents have an
+ # index.html in them setting the success-url to html/ will render
+ # things as expected. Existing builder macros look like:
+ #
+ # - publisher:
+ # name: upload-sphinx-draft
+ # publishers:
+ # - scp:
+ # site: 'static.openstack.org'
+ # files:
+ # - target: 'docs-draft/$LOG_PATH'
+ # source: 'doc/build/html/**'
+ # keep-hierarchy: true
+ # copy-after-failure: true
+ #
+ # Which is pulling the tree of the remote html directory starting with
+ # doc/build/html and putting that whole thing into
+ # docs-draft/$LOG_PATH.
+ #
+ # Then there is a success-pattern in layout.yaml that looks like:
+ #
+ # http://{url}/{log_path}/doc/build/html/
+ #
+ # Which gets reports. There are many variations on that URL. So rather
+ # than needing to figure out varying success-urls to report in v3,
+ # we'll remote the ** and not process this through the rsync_opts
+ # processing we use for the other publishers, but instead will just
+ # pass doc/build/html/ to get the contents of doc/build/html/ and we'll
+ # put those in {{ log_root }}/html/ locally meaning the success-url
+ # can always be html/. This should work for all values of source
+ # from v2.
+ tasks = []
+ artifacts = False
+ draft = False
+ site = publisher['scp']['site']
+ for scpfile in publisher['scp']['files']:
+ if 'ZUUL_PROJECT' in scpfile.get('source', ''):
+ self.log.error(
+ "Job {name} uses ZUUL_PROJECT in source".format(
+ name=self.name))
+ continue
+
+ if scpfile.get('copy-console'):
+ continue
+ else:
+ src = "{{ ansible_user_dir }}/"
+ rsync_opts = self._getRsyncOptions(scpfile['source'])
+
+ target = scpfile['target']
+ # TODO(mordred) Generalize this next section, it's SUPER
+ # openstack specific. We can likely do this in mapping.yaml
+ if site == 'static.openstack.org':
+ for f in ('service-types', 'specs'):
+ if target.startswith(f):
+ self.log.error(
+ "Job {name} uses {f} publishing".format(
+ name=self.name, f=f))
+ continue
+ if target.startswith('docs-draft'):
+ target = "{{ zuul.executor.log_root }}/html/"
+ src = scpfile['source'].replace('**', '')
+ rsync_opts = None
+ draft = True
+ else:
+ target = target.replace(
+ 'logs/$LOG_PATH',
+ "{{ zuul.executor.log_root }}")
+ elif site == 'tarballs.openstack.org':
+ if not target.startswith('tarballs'):
+ self.log.error(
+ 'Job {name} wants to publish artifacts to non'
+ ' tarballs dir'.format(name=self.name))
+ continue
+ if target.startswith('tarballs/ci'):
+ target = target.split('/', 3)[-1]
+ else:
+ target = target.split('/', 2)[-1]
+ target = "{{ zuul.executor.work_root }}/artifacts/" + target
+ artifacts = True
+ elif site == 'yaml2ical':
+ self.log.error('Job {name} uses yaml2ical publisher')
+ continue
+
+ syncargs = collections.OrderedDict()
+ syncargs['src'] = src
+ syncargs['dest'] = target
+ syncargs['mode'] = 'pull'
+ syncargs['copy_links'] = True
+ syncargs['verify_host'] = True
+ if rsync_opts:
+ syncargs['rsync_opts'] = rsync_opts
+ task = collections.OrderedDict()
+ task['name'] = 'Copy files from {src} on node'.format(src=src)
+ task['synchronize'] = syncargs
+ # We don't use retry_args here because there is a bug in
+ # the synchronize module that breaks subsequent attempts at
+ # retrying. Better to try once and get an accurate error
+ # message if it fails.
+ # https://github.com/ansible/ansible/issues/18281
+ tasks.append(task)
+
+ if artifacts:
+ ensure_task = collections.OrderedDict()
+ ensure_task['name'] = 'Ensure artifacts directory exists'
+ ensure_task['file'] = collections.OrderedDict(
+ path="{{ zuul.executor.work_root }}/artifacts",
+ state='directory')
+ ensure_task['delegate_to'] = 'localhost'
+ tasks.insert(0, ensure_task)
+ return dict(tasks=tasks, artifacts=artifacts, draft=draft)
+
+ def _emitShellTask(self, data, syntax_check):
+ shell, data = deal_with_shebang(data)
+ task = collections.OrderedDict()
+ # Putting the data directly into shell: causes here docs to break.
+ task['shell'] = collections.OrderedDict()
+ task['shell']['cmd'] = data
+ if shell:
+ task['shell']['executable'] = shell
+
+ if syntax_check:
+ # Emit a test playbook with this shell task in it then run
+ # ansible-playbook --syntax-check on it. This will fail if there
+ # are embedding issues, such as with unbalanced single quotes
+ # The end result should be less scripts and more shell
+ play = dict(hosts='all', tasks=[task])
+ (fd, tmp_path) = tempfile.mkstemp()
+ try:
+ f = os.fdopen(fd, 'w')
+ ordered_dump([play], f)
+ f.close()
+ proc = subprocess.run(
+ ['ansible-playbook', '--syntax-check', tmp_path],
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ if proc.returncode != 0:
+ # Return of None means we must emit a script
+ self.log.error(
+ "Job {name} had an ansible syntax error, using script"
+ " instead of shell".format(name=self.name))
+ return None
+ finally:
+ os.unlink(tmp_path)
+ return task
+
+ def _emitScriptContent(self, data, playbook_dir, seq):
+ script_fn = '%s-%02d.sh' % (self.short_name, seq)
+ script_path = os.path.join(playbook_dir, script_fn)
+
+ with open(script_path, 'w') as script:
+ if not data.startswith('#!'):
+ data = '#!/bin/bash -x\n %s' % (data,)
+ script.write(data)
+
+ task = collections.OrderedDict()
+ task['name'] = 'Running playbooks/legacy/{playbook}'.format(
+ playbook=script_fn)
+ task['script'] = script_fn
+ return task
+
+ def _makeBuilderTask(self, playbook_dir, builder, sequence, syntax_check):
+ # Don't write a script to echo the template line
+ # TODO(mordred) Put these into mapping.yaml
+ if builder['shell'].startswith('echo JJB template: '):
+ return
+ if 'echo "Detailed logs:' in builder['shell']:
+ return
+ if ('cat /etc/dib-builddate.txt' in builder['shell'] and
+ 'echo "Network configuration data..."' in builder['shell']):
+ return
+
+ task = self._emitShellTask(builder['shell'], syntax_check)
+ if not task:
+ task = self._emitScriptContent(
+ builder['shell'], playbook_dir, sequence)
+ task['environment'] = ENVIRONMENT
+ return task
+
+ def _transformPublishers(self, jjb_job):
+ early_publishers = []
+ late_publishers = []
+ old_publishers = jjb_job.get('publishers', [])
+ for publisher in old_publishers:
+ early_scpfiles = []
+ late_scpfiles = []
+ if 'scp' not in publisher:
+ early_publishers.append(publisher)
+ continue
+ copy_console = False
+ for scpfile in publisher['scp']['files']:
+ if scpfile.get('copy-console'):
+ scpfile['keep-hierarchy'] = True
+ late_scpfiles.append(scpfile)
+ copy_console = True
+ else:
+ early_scpfiles.append(scpfile)
+ publisher['scp']['files'] = early_scpfiles + late_scpfiles
+ if copy_console:
+ late_publishers.append(publisher)
+ else:
+ early_publishers.append(publisher)
+ publishers = early_publishers + late_publishers
+ if old_publishers != publishers:
+ self.log.debug("Transformed job publishers")
+ return early_publishers, late_publishers
+
+ def emitPlaybooks(self, jobsdir, syntax_check=False):
+ has_artifacts = False
+ has_draft = False
+ if not self.jjb_job:
+ if self.emit:
+ self.log.error(
+ 'Job {name} has no job content'.format(name=self.name))
+ return False, False, False
+
+ playbook_dir = os.path.join(jobsdir, self.job_path)
+ if not os.path.exists(playbook_dir):
+ os.makedirs(playbook_dir)
+
+ run_playbook = os.path.join(self.job_path, 'run.yaml')
+ post_playbook = os.path.join(self.job_path, 'post.yaml')
+ tasks = []
+ sequence = 0
+ for builder in self.jjb_job.get('builders', []):
+ if 'shell' in builder:
+ self.required_projects.extend(
+ extract_projects(builder['shell']))
+ task = self._makeBuilderTask(
+ playbook_dir, builder, sequence, syntax_check)
+ if task:
+ if 'script' in task:
+ sequence += 1
+ tasks.append(task)
+ play = collections.OrderedDict()
+ play['hosts'] = 'all'
+ play['name'] = 'Autoconverted job {name} from old job {old}'.format(
+ name=self.name, old=self.orig)
+ play['tasks'] = tasks
+
+ with open(run_playbook, 'w') as run_playbook_out:
+ ordered_dump([play], run_playbook_out)
+
+ has_post = False
+ tasks = []
+ early_publishers, late_publishers = self._transformPublishers(
+ self.jjb_job)
+ for publishers in [early_publishers, late_publishers]:
+ for publisher in publishers:
+ if 'scp' in publisher:
+ ret = self._makeSCPTask(publisher)
+ if ret['artifacts']:
+ has_artifacts = True
+ if ret['draft']:
+ has_draft = True
+ tasks.extend(ret['tasks'])
+ if 'afs' in builder:
+ self.log.error(
+ "Job {name} uses AFS publisher".format(name=self.name))
+ if tasks:
+ has_post = True
+ play = collections.OrderedDict()
+ play['hosts'] = 'all'
+ play['tasks'] = tasks
+ with open(post_playbook, 'w') as post_playbook_out:
+ ordered_dump([play], post_playbook_out)
+ return has_artifacts, has_post, has_draft
+
+ def toJobDict(
+ self, has_artifacts=False, has_post=False, has_draft=False,
+ project_names=[]):
+ output = collections.OrderedDict()
+ output['name'] = self.name
+
+ expanded_projects = []
+ if self.required_projects:
+ expanded_projects = expand_project_names(
+ self.required_projects, project_names)
+ # Look for project names in the job name. Lookie there - the
+ # python in operator works on lists and strings.
+ expanded_projects.extend(expand_project_names(
+ self.name, project_names))
+
+ output['parent'] = 'legacy-base'
+ if 'dsvm' in self.name:
+ output['parent'] = 'legacy-dsvm-base'
+ elif 'puppet-openstack-integration' in self.name:
+ output['parent'] = 'legacy-puppet-openstack-integration'
+ elif 'openstack/puppet-openstack-integration' in expanded_projects:
+ output['parent'] = 'legacy-puppet-openstack-integration'
+ elif has_artifacts:
+ output['parent'] = 'publish-openstack-artifacts'
+ elif has_draft:
+ output['success-url'] = 'html/'
+ output['run'] = os.path.join(self.job_path, 'run.yaml')
+ if has_post:
+ output['post-run'] = os.path.join(self.job_path, 'post.yaml')
+
+ if self.vars:
+ output['vars'] = self.vars.copy()
+ timeout = self.getTimeout()
+ if timeout:
+ output['timeout'] = timeout
+ output['vars']['BUILD_TIMEOUT'] = str(timeout * 1000)
+
+ if self.nodes:
+ if len(self.nodes) == 1:
+ output['nodeset'] = self.getNodes()[0]
+ else:
+ output['nodeset'] = dict(nodes=self.getNodes())
+
+ if expanded_projects:
+ output['required-projects'] = list(set(expanded_projects))
+
+ return output
+
+ def toPipelineDict(self):
+ if self.content:
+ output = self.content
+ else:
+ output = collections.OrderedDict()
+ output[self.name] = collections.OrderedDict()
+
+ if self.parent:
+ output[self.name].setdefault('dependencies', self.getDepends())
+
+ if not self.voting:
+ output[self.name].setdefault('voting', False)
+
+ if self.vars:
+ job_vars = output[self.name].get('vars', collections.OrderedDict())
+ job_vars.update(self.vars)
+
+ if self.branch:
+ output[self.name]['branch'] = self.branch
+
+ if self.files:
+ output[self.name]['files'] = self.files
+
+ if not output[self.name]:
+ return self.name
+
+ return output
+
+
+class JobMapping:
+ log = logging.getLogger("zuul.Migrate.JobMapping")
+
+ def __init__(self, nodepool_config, layout, mapping_file=None):
+ self.layout = layout
+ self.job_direct = {}
+ self.labels = []
+ self.job_mapping = []
+ self.template_mapping = {}
+ self.jjb_jobs = {}
+ self.seen_new_jobs = []
+ self.unshare = []
+ nodepool_data = ordered_load(open(nodepool_config, 'r'))
+ for label in nodepool_data['labels']:
+ self.labels.append(label['name'])
+ if not mapping_file:
+ self.default_node = 'ubuntu-xenial'
+ else:
+ mapping_data = ordered_load(open(mapping_file, 'r'))
+ self.default_node = mapping_data['default-node']
+ global SUFFIXES
+ SUFFIXES = mapping_data.get('strip-suffixes', [])
+ self.unshare = mapping_data.get('unshare', [])
+ for map_info in mapping_data.get('job-mapping', []):
+ if map_info['old'].startswith('^'):
+ map_info['pattern'] = re.compile(map_info['old'])
+ self.job_mapping.append(map_info)
+ else:
+ self.job_direct[map_info['old']] = map_info['new']
+
+ for map_info in mapping_data.get('template-mapping', []):
+ self.template_mapping[map_info['old']] = map_info['new']
+
+ def makeNewName(self, new_name, match_dict):
+ return new_name.format(**match_dict)
+
+ def hasProjectTemplate(self, old_name):
+ return old_name in self.template_mapping
+
+ def setJJBJobs(self, jjb_jobs):
+ self.jjb_jobs = jjb_jobs
+
+ def getNewTemplateName(self, old_name):
+ return self.template_mapping.get(old_name, old_name)
+
+ def mapNewJob(self, name, info) -> Optional[Job]:
+ matches = info['pattern'].search(name)
+ if not matches:
+ return None
+ match_dict = matches.groupdict()
+ if isinstance(info['new'], dict):
+ new_job = info['new']
+ old_name = get_single_key(new_job)
+ new_name = old_name.format(**match_dict)
+ job = Job(orig=name, content={new_name: new_job[old_name]})
+ else:
+ job = Job(orig=name, name=info['new'].format(**match_dict))
+
+ if 'vars' in info:
+ job.setVars(self._expandVars(info, match_dict))
+
+ return job
+
+ def _expandVars(self, info, match_dict):
+ job_vars = info['vars'].copy()
+ for key in job_vars.keys():
+ job_vars[key] = job_vars[key].format(**match_dict)
+ return job_vars
+
+ def getNewJob(self, job_name, remove_gate):
+ if job_name in self.job_direct:
+ if isinstance(self.job_direct[job_name], dict):
+ job = Job(job_name, content=self.job_direct[job_name])
+ else:
+ job = Job(job_name, name=self.job_direct[job_name])
+ if job_name not in self.seen_new_jobs:
+ self.seen_new_jobs.append(self.job_direct[job_name])
+ job.setNoEmit()
+ return job
+
+ new_job = None
+ for map_info in self.job_mapping:
+ new_job = self.mapNewJob(job_name, map_info)
+ if new_job:
+ if job_name not in self.seen_new_jobs:
+ self.seen_new_jobs.append(new_job.name)
+ new_job.setNoEmit()
+ break
+ if not new_job:
+ orig_name = job_name
+ if remove_gate:
+ job_name = job_name.replace('gate-', '', 1)
+ job_name = 'legacy-{job_name}'.format(job_name=job_name)
+ new_job = Job(orig=orig_name, name=job_name)
+
+ new_job.extractNode(self.default_node, self.labels)
+
+ # Handle matchers
+ for layout_job in self.layout.get('jobs', []):
+ if re.search(layout_job['name'], new_job.orig):
+ # Matchers that can apply to templates must be processed first
+ # since project-specific matchers can cause the template to
+ # be expanded into a project.
+ if not layout_job.get('voting', True):
+ new_job.voting = False
+ if layout_job.get('branch'):
+ new_job.branch = layout_job['branch']
+ if layout_job.get('files'):
+ new_job.files = layout_job['files']
+
+ new_job.addJJBJob(self.jjb_jobs)
+ return new_job
+
+
+class ChangeQueue:
+ def __init__(self):
+ self.name = ''
+ self.assigned_name = None
+ self.generated_name = None
+ self.projects = []
+ self._jobs = set()
+
+ def getJobs(self):
+ return self._jobs
+
+ def getProjects(self):
+ return [p.name for p in self.projects]
+
+ def addProject(self, project):
+ if project not in self.projects:
+ self.projects.append(project)
+ self._jobs |= project.gate_jobs
+
+ names = [x.name for x in self.projects]
+ names.sort()
+ self.generated_name = names[0].split('/')[-1]
+
+ for job in self._jobs:
+ if job.queue_name:
+ if (self.assigned_name and
+ job.queue_name != self.assigned_name):
+ raise Exception("More than one name assigned to "
+ "change queue: %s != %s" %
+ (self.assigned_name,
+ job.queue_name))
+ self.assigned_name = job.queue_name
+ self.name = self.assigned_name or self.generated_name
+
+ def mergeChangeQueue(self, other):
+ for project in other.projects:
+ self.addProject(project)
+
+
+class ZuulMigrate:
+
+ log = logging.getLogger("zuul.Migrate")
+
+ def __init__(self, layout, job_config, nodepool_config,
+ outdir, mapping, move, syntax_check):
+ self.layout = ordered_load(open(layout, 'r'))
+ self.job_config = job_config
+ self.outdir = outdir
+ self.mapping = JobMapping(nodepool_config, self.layout, mapping)
+ self.move = move
+ self.syntax_check = syntax_check
+
+ self.jobs = {}
+ self.old_jobs = {}
+ self.job_objects = []
+ self.new_templates = {}
+
+ def run(self):
+ self.loadJobs()
+ self.buildChangeQueues()
+ self.convertJobs()
+ self.writeJobs()
+
+ def loadJobs(self):
+ self.log.debug("Loading jobs")
+ builder = JJB()
+ builder.load_files([self.job_config])
+ builder.parser.expandYaml()
+ unseen = set(self.jobs.keys())
+ for job in builder.parser.jobs:
+ builder.expandMacros(job)
+ self.jobs[job['name']] = job
+ unseen.discard(job['name'])
+ for name in unseen:
+ del self.jobs[name]
+ self.mapping.setJJBJobs(self.jobs)
+
+ def getOldJob(self, name):
+ if name not in self.old_jobs:
+ self.old_jobs[name] = OldJob(name)
+ return self.old_jobs[name]
+
+ def flattenOldJobs(self, tree, name=None):
+ if isinstance(tree, str):
+ n = tree.format(name=name)
+ if n in self.mapping.unshare:
+ return []
+ return [self.getOldJob(n)]
+
+ new_list = [] # type: ignore
+ if isinstance(tree, list):
+ for job in tree:
+ new_list.extend(self.flattenOldJobs(job, name))
+ elif isinstance(tree, dict):
+ parent_name = get_single_key(tree)
+ jobs = self.flattenOldJobs(tree[parent_name], name)
+ for job in jobs:
+ if job not in self.mapping.unshare:
+ new_list.append(self.getOldJob(job))
+ if parent_name not in self.mapping.unshare:
+ new_list.append(self.getOldJob(parent_name))
+ return new_list
+
+ def buildChangeQueues(self):
+ self.log.debug("Building shared change queues")
+
+ for j in self.layout['jobs']:
+ if '^' in j['name'] or '$' in j['name']:
+ continue
+ job = self.getOldJob(j['name'])
+ job.queue_name = j.get('queue-name')
+
+ change_queues = []
+
+ for project in self.layout.get('projects'):
+ if 'gate' not in project:
+ continue
+ gate_jobs = set()
+ for template in project['template']:
+ for pt in self.layout.get('project-templates'):
+ if pt['name'] != template['name']:
+ continue
+ if 'gate' not in pt['name']:
+ continue
+ gate_jobs |= set(self.flattenOldJobs(pt['gate'],
+ project['name']))
+ gate_jobs |= set(self.flattenOldJobs(project['gate']))
+ old_project = OldProject(project['name'], gate_jobs)
+ change_queue = ChangeQueue()
+ change_queue.addProject(old_project)
+ change_queues.append(change_queue)
+ self.log.debug("Created queue: %s" % change_queue)
+
+ # Iterate over all queues trying to combine them, and keep doing
+ # so until they can not be combined further.
+ last_change_queues = change_queues
+ while True:
+ new_change_queues = self.combineChangeQueues(last_change_queues)
+ if len(last_change_queues) == len(new_change_queues):
+ break
+ last_change_queues = new_change_queues
+
+ self.log.debug(" Shared change queues:")
+ for queue in new_change_queues:
+ self.log.debug(" %s containing %s" % (
+ queue, queue.generated_name))
+ self.change_queues = new_change_queues
+
+ def combineChangeQueues(self, change_queues):
+ self.log.debug("Combining shared queues")
+ new_change_queues = []
+ for a in change_queues:
+ merged_a = False
+ for b in new_change_queues:
+ if not a.getJobs().isdisjoint(b.getJobs()):
+ self.log.debug("Merging queue %s into %s" % (a, b))
+ b.mergeChangeQueue(a)
+ merged_a = True
+ break # this breaks out of 'for b' and continues 'for a'
+ if not merged_a:
+ self.log.debug("Keeping queue %s" % (a))
+ new_change_queues.append(a)
+ return new_change_queues
+
+ def convertJobs(self):
+ pass
+
+ def setupDir(self):
+ zuul_yaml = os.path.join(self.outdir, 'zuul.yaml')
+ zuul_d = os.path.join(self.outdir, 'zuul.d')
+ orig = os.path.join(zuul_d, '01zuul.yaml')
+ job_outfile = os.path.join(zuul_d, '99converted-jobs.yaml')
+ project_outfile = os.path.join(zuul_d, '99converted-projects.yaml')
+ if not os.path.exists(self.outdir):
+ os.makedirs(self.outdir)
+ if not os.path.exists(zuul_d):
+ os.makedirs(zuul_d)
+ if os.path.exists(zuul_yaml) and self.move:
+ os.rename(zuul_yaml, orig)
+ return job_outfile, project_outfile
+
+ def makeNewJobs(self, old_job, parent: Job=None):
+ self.log.debug("makeNewJobs(%s)", old_job)
+ if isinstance(old_job, str):
+ remove_gate = True
+ if old_job.startswith('gate-'):
+ # Check to see if gate- and bare versions exist
+ if old_job.replace('gate-', '', 1) in self.jobs:
+ remove_gate = False
+ job = self.mapping.getNewJob(old_job, remove_gate)
+ if parent:
+ job.setParent(parent)
+ return [job]
+
+ new_list = [] # type: ignore
+ if isinstance(old_job, list):
+ for job in old_job:
+ new_list.extend(self.makeNewJobs(job, parent=parent))
+
+ elif isinstance(old_job, dict):
+ parent_name = get_single_key(old_job)
+ parent = self.makeNewJobs(parent_name, parent=parent)[0]
+
+ jobs = self.makeNewJobs(old_job[parent_name], parent=parent)
+ for job in jobs:
+ new_list.append(job)
+ new_list.append(parent)
+ return new_list
+
+ def writeProjectTemplate(self, template):
+ new_template = collections.OrderedDict()
+ if 'name' in template:
+ new_template['name'] = template['name']
+ for key, value in template.items():
+ if key == 'name':
+ continue
+
+ # keep a cache of the Job objects so we can use it to get old
+ # job name to new job name when expanding templates into projects.
+ tmp = [job for job in self.makeNewJobs(value)]
+ self.job_objects.extend(tmp)
+ jobs = [job.toPipelineDict() for job in tmp]
+ new_template[key] = dict(jobs=jobs)
+
+ return new_template
+
+ def scanForProjectMatchers(self, project_name):
+ ''' Get list of job matchers that reference the given project name '''
+ job_matchers = []
+ for matcher in self.layout.get('jobs', []):
+ for skipper in matcher.get('skip-if', []):
+ if skipper.get('project'):
+ if re.search(skipper['project'], project_name):
+ job_matchers.append(matcher)
+ return job_matchers
+
+ def findReferencedTemplateNames(self, job_matchers, project_name):
+ ''' Search templates in the layout file for matching jobs '''
+ template_names = []
+
+ def search_jobs(template):
+ def _search(job):
+ if isinstance(job, str):
+ for matcher in job_matchers:
+ if re.search(matcher['name'],
+ job.format(name=project_name)):
+ template_names.append(template['name'])
+ return True
+ elif isinstance(job, list):
+ for i in job:
+ if _search(i):
+ return True
+ elif isinstance(job, dict):
+ for k, v in job.items():
+ if _search(k) or _search(v):
+ return True
+ return False
+
+ for key, value in template.items():
+ if key == 'name':
+ continue
+ for job in template[key]:
+ if _search(job):
+ return
+
+ for template in self.layout.get('project-templates', []):
+ search_jobs(template)
+ return template_names
+
+ def expandTemplateIntoProject(self, template_name, project):
+ self.log.debug("EXPAND template %s into project %s",
+ template_name, project['name'])
+ # find the new template since that's the thing we're expanding
+ if template_name not in self.new_templates:
+ self.log.error(
+ "Template %s not found for expansion into project %s",
+ template_name, project['name'])
+ return
+
+ template = self.new_templates[template_name]
+
+ for pipeline, value in template.items():
+ if pipeline == 'name':
+ continue
+ if pipeline not in project:
+ project[pipeline] = dict(jobs=[])
+ project[pipeline]['jobs'].extend(value['jobs'])
+
+ def getOldJobName(self, new_job_name):
+ for job in self.job_objects:
+ if job.name == new_job_name:
+ return job.orig
+ return None
+
+ def applyProjectMatchers(self, matchers, project):
+ '''
+ Apply per-project job matchers to the given project.
+
+ :param matchers: Job matchers that referenced the given project.
+ :param project: The new project object.
+ '''
+
+ def processPipeline(pipeline_jobs, job_name_regex, files):
+ new_jobs = []
+ for job in pipeline_jobs:
+ if isinstance(job, str):
+ old_job_name = self.getOldJobName(job)
+ if old_job_name and re.search(
+ job_name_regex, old_job_name):
+ self.log.debug(
+ "Applied irrelevant-files to job %s in project %s",
+ job, project['name'])
+ job = {job: {'irrelevant-files': list(set(files))}}
+ elif isinstance(job, dict):
+ job = job.copy()
+ job_name = get_single_key(job)
+ extras = job[job_name]
+ old_job_name = self.getOldJobName(job_name)
+ if old_job_name and re.search(
+ job_name_regex, old_job_name):
+ self.log.debug(
+ "Applied irrelevant-files to complex job "
+ "%s in project %s", job_name, project['name'])
+ if 'irrelevant-files' not in extras:
+ extras['irrelevant-files'] = []
+ extras['irrelevant-files'].extend(files)
+ extras['irrelevant-files'] = list(
+ set(extras['irrelevant-files']))
+ job[job_name] = extras
+ new_jobs.append(job)
+ return new_jobs
+
+ def applyIrrelevantFiles(job_name_regex, files):
+ for k, v in project.items():
+ if k in ('template', 'name'):
+ continue
+ project[k]['jobs'] = processPipeline(
+ project[k]['jobs'], job_name_regex, files)
+
+ for matcher in matchers:
+ # find the project-specific section
+ for skipper in matcher.get('skip-if', []):
+ if skipper.get('project'):
+ if re.search(skipper['project'], project['name']):
+ if 'all-files-match-any' in skipper:
+ applyIrrelevantFiles(
+ matcher['name'],
+ skipper['all-files-match-any'])
+
+ def writeProject(self, project):
+ '''
+ Create a new v3 project definition.
+
+ As part of creating the project, scan for project-specific job matchers
+ referencing this project and remove the templates matching the job
+ regex for that matcher. Expand the matched template(s) into the project
+ so we can apply the project-specific matcher to the job(s).
+ '''
+ new_project = collections.OrderedDict()
+ if 'name' in project:
+ new_project['name'] = project['name']
+
+ job_matchers = self.scanForProjectMatchers(project['name'])
+ if job_matchers:
+ exp_template_names = self.findReferencedTemplateNames(
+ job_matchers, project['name'])
+ else:
+ exp_template_names = []
+
+ templates_to_expand = []
+ if 'template' in project:
+ new_project['template'] = []
+ for template in project['template']:
+ if template['name'] in exp_template_names:
+ templates_to_expand.append(template['name'])
+ continue
+ new_project['template'].append(dict(
+ name=self.mapping.getNewTemplateName(template['name'])))
+
+ for key, value in project.items():
+ if key in ('name', 'template'):
+ continue
+ else:
+ new_project[key] = collections.OrderedDict()
+ if key == 'gate':
+ for queue in self.change_queues:
+ if project['name'] not in queue.getProjects():
+ continue
+ if len(queue.getProjects()) == 1:
+ continue
+ new_project[key]['queue'] = queue.name
+ tmp = [job for job in self.makeNewJobs(value)]
+ self.job_objects.extend(tmp)
+ jobs = [job.toPipelineDict() for job in tmp]
+ new_project[key]['jobs'] = jobs
+
+ for name in templates_to_expand:
+ self.expandTemplateIntoProject(name, new_project)
+
+ # Need a deep copy after expansion, else our templates end up
+ # also getting this change.
+ new_project = copy.deepcopy(new_project)
+ self.applyProjectMatchers(job_matchers, new_project)
+
+ return new_project
+
+ def writeJobs(self):
+ job_outfile, project_outfile = self.setupDir()
+ job_config = []
+ project_config = []
+
+ for template in self.layout.get('project-templates', []):
+ self.log.debug("Processing template: %s", template)
+ new_template = self.writeProjectTemplate(template)
+ self.new_templates[new_template['name']] = new_template
+ if not self.mapping.hasProjectTemplate(template['name']):
+ job_config.append({'project-template': new_template})
+
+ project_names = []
+ for project in self.layout.get('projects', []):
+ project_names.append(project['name'])
+ project_config.append(
+ {'project': self.writeProject(project)})
+
+ seen_jobs = []
+ for job in sorted(self.job_objects, key=lambda job: job.name):
+ if (job.name not in seen_jobs and
+ job.name not in self.mapping.seen_new_jobs and
+ job.emit):
+ has_artifacts, has_post, has_draft = job.emitPlaybooks(
+ self.outdir, self.syntax_check)
+ job_config.append({'job': job.toJobDict(
+ has_artifacts, has_post, has_draft, project_names)})
+ seen_jobs.append(job.name)
+
+ with open(job_outfile, 'w') as yamlout:
+ # Insert an extra space between top-level list items
+ yamlout.write(ordered_dump(job_config).replace('\n-', '\n\n-'))
+
+ with open(project_outfile, 'w') as yamlout:
+ # Insert an extra space between top-level list items
+ yamlout.write(ordered_dump(project_config).replace('\n-', '\n\n-'))
+
+
+def main():
+ yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+ construct_yaml_map)
+
+ yaml.add_representer(collections.OrderedDict, project_representer,
+ Dumper=IndentedDumper)
+ yaml.representer.BaseRepresenter.represent_scalar = my_represent_scalar
+
+ parser = argparse.ArgumentParser(description=DESCRIPTION)
+ parser.add_argument(
+ 'layout',
+ help="The Zuul v2 layout.yaml file to read.")
+ parser.add_argument(
+ 'job_config',
+ help="Directory containing Jenkins Job Builder job definitions.")
+ parser.add_argument(
+ 'nodepool_config',
+ help="Nodepool config file containing complete set of node names")
+ parser.add_argument(
+ 'outdir',
+ help="A directory into which the Zuul v3 config will be written.")
+ parser.add_argument(
+ '--mapping',
+ default=None,
+ help="A filename with a yaml mapping of old name to new name.")
+ parser.add_argument(
+ '-v', dest='verbose', action='store_true', help='verbose output')
+ parser.add_argument(
+ '--syntax-check', dest='syntax_check', action='store_true',
+ help='Run ansible-playbook --syntax-check on generated playbooks')
+ parser.add_argument(
+ '-m', dest='move', action='store_true',
+ help='Move zuul.yaml to zuul.d if it exists')
+
+ args = parser.parse_args()
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+
+ ZuulMigrate(args.layout, args.job_config, args.nodepool_config,
+ args.outdir, args.mapping, args.move, args.syntax_check).run()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 3b09623..b70ea59 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -76,6 +76,15 @@
super(MaxNodeError, self).__init__(message)
+class MaxTimeoutError(Exception):
+ def __init__(self, job, tenant):
+ message = textwrap.dedent("""\
+ The job "{job}" exceeds tenant max-job-timeout {maxtimeout}.""")
+ message = textwrap.fill(message.format(
+ job=job.name, maxtimeout=tenant.max_job_timeout))
+ super(MaxTimeoutError, self).__init__(message)
+
+
class DuplicateGroupError(Exception):
def __init__(self, nodeset, group):
message = textwrap.dedent("""\
@@ -97,6 +106,24 @@
super(ProjectNotFoundError, self).__init__(message)
+class SecretNotFoundError(Exception):
+ def __init__(self, secret):
+ message = textwrap.dedent("""\
+ The secret "{secret}" was not found.
+ """)
+ message = textwrap.fill(message.format(secret=secret))
+ super(SecretNotFoundError, self).__init__(message)
+
+
+class NodesetNotFoundError(Exception):
+ def __init__(self, nodeset):
+ message = textwrap.dedent("""\
+ The nodeset "{nodeset}" was not found.
+ """)
+ message = textwrap.fill(message.format(nodeset=nodeset))
+ super(NodesetNotFoundError, self).__init__(message)
+
+
class PipelineNotPermittedError(Exception):
def __init__(self):
message = textwrap.dedent("""\
@@ -120,6 +147,30 @@
@contextmanager
+def early_configuration_exceptions(context):
+ try:
+ yield
+ except ConfigurationSyntaxError:
+ raise
+ except Exception as e:
+ intro = textwrap.fill(textwrap.dedent("""\
+ Zuul encountered a syntax error while parsing its configuration in the
+ repo {repo} on branch {branch}. The error was:""".format(
+ repo=context.project.name,
+ branch=context.branch,
+ )))
+
+ m = textwrap.dedent("""\
+ {intro}
+
+ {error}""")
+
+ m = m.format(intro=intro,
+ error=indent(str(e)))
+ raise ConfigurationSyntaxError(m)
+
+
+@contextmanager
def configuration_exceptions(stanza, conf):
try:
yield
@@ -254,7 +305,7 @@
class NodeSetParser(object):
@staticmethod
- def getSchema():
+ def getSchema(anonymous=False):
node = {vs.Required('name'): str,
vs.Required('label'): str,
}
@@ -263,19 +314,20 @@
vs.Required('nodes'): to_list(str),
}
- nodeset = {vs.Required('name'): str,
- vs.Required('nodes'): to_list(node),
+ nodeset = {vs.Required('nodes'): to_list(node),
'groups': to_list(group),
'_source_context': model.SourceContext,
'_start_mark': ZuulMark,
}
+ if not anonymous:
+ nodeset[vs.Required('name')] = str
return vs.Schema(nodeset)
@staticmethod
- def fromYaml(layout, conf):
- NodeSetParser.getSchema()(conf)
- ns = model.NodeSet(conf['name'])
+ def fromYaml(conf, anonymous=False):
+ NodeSetParser.getSchema(anonymous)(conf)
+ ns = model.NodeSet(conf.get('name'))
node_names = set()
group_names = set()
for conf_node in as_list(conf['nodes']):
@@ -324,10 +376,6 @@
@staticmethod
def getSchema():
- node = {vs.Required('name'): str,
- vs.Required('label'): str,
- }
-
zuul_role = {vs.Required('zuul'): str,
'name': str}
@@ -357,7 +405,8 @@
'files': to_list(str),
'secrets': to_list(vs.Any(secret, str)),
'irrelevant-files': to_list(str),
- 'nodes': vs.Any([node], str),
+ # validation happens in NodeSetParser
+ 'nodeset': vs.Any(dict, str),
'timeout': int,
'attempts': int,
'pre-run': to_list(str),
@@ -453,13 +502,15 @@
# Secrets are part of the playbook context so we must establish
# them earlier than playbooks.
secrets = []
- for secret_config in conf.get('secrets', []):
+ for secret_config in as_list(conf.get('secrets', [])):
if isinstance(secret_config, str):
secret_name = secret_config
- secret = layout.secrets[secret_name]
+ secret = layout.secrets.get(secret_name)
else:
secret_name = secret_config['name']
- secret = layout.secrets[secret_config['secret']]
+ secret = layout.secrets.get(secret_config['secret'])
+ if secret is None:
+ raise SecretNotFoundError(secret_name)
if secret_name == 'zuul':
raise Exception("Secrets named 'zuul' are not allowed.")
if secret.source_context != job.source_context:
@@ -481,6 +532,10 @@
if secrets and not conf['_source_context'].trusted:
job.post_review = True
+ if conf.get('timeout') and tenant.max_job_timeout != -1 and \
+ int(conf['timeout']) > tenant.max_job_timeout:
+ raise MaxTimeoutError(job, tenant)
+
if 'post-review' in conf:
if conf['post-review']:
job.post_review = True
@@ -532,16 +587,15 @@
a = k.replace('-', '_')
if k in conf:
setattr(job, a, conf[k])
- if 'nodes' in conf:
- conf_nodes = conf['nodes']
- if isinstance(conf_nodes, str):
+ if 'nodeset' in conf:
+ conf_nodeset = conf['nodeset']
+ if isinstance(conf_nodeset, str):
# This references an existing named nodeset in the layout.
- ns = layout.nodesets[conf_nodes]
+ ns = layout.nodesets.get(conf_nodeset)
+ if ns is None:
+ raise NodesetNotFoundError(conf_nodeset)
else:
- ns = model.NodeSet()
- for conf_node in conf_nodes:
- node = model.Node(conf_node['name'], conf_node['label'])
- ns.addNode(node)
+ ns = NodeSetParser.fromYaml(conf_nodeset, anonymous=True)
if tenant.max_nodes_per_job != -1 and \
len(ns) > tenant.max_nodes_per_job:
raise MaxNodeError(job, tenant)
@@ -1035,6 +1089,7 @@
def getSchema(connections=None):
tenant = {vs.Required('name'): str,
'max-nodes-per-job': int,
+ 'max-job-timeout': int,
'source': TenantParser.validateTenantSources(connections),
'exclude-unprotected-branches': bool,
'default-parent': str,
@@ -1048,6 +1103,8 @@
tenant = model.Tenant(conf['name'])
if conf.get('max-nodes-per-job') is not None:
tenant.max_nodes_per_job = conf['max-nodes-per-job']
+ if conf.get('max-job-timeout') is not None:
+ tenant.max_job_timeout = int(conf['max-job-timeout'])
if conf.get('exclude-unprotected-branches') is not None:
tenant.exclude_unprotected_branches = \
conf['exclude-unprotected-branches']
@@ -1318,6 +1375,8 @@
continue
TenantParser.log.debug("Waiting for cat job %s" % (job,))
job.wait()
+ if not job.updated:
+ raise Exception("Cat job %s failed" % (job,))
TenantParser.log.debug("Cat job %s got files %s" %
(job, job.files))
loaded = False
@@ -1367,13 +1426,15 @@
def _parseConfigProjectLayout(data, source_context):
# This is the top-level configuration for a tenant.
config = model.UnparsedTenantConfig()
- config.extend(safe_load_yaml(data, source_context))
+ with early_configuration_exceptions(source_context):
+ config.extend(safe_load_yaml(data, source_context))
return config
@staticmethod
def _parseUntrustedProjectLayout(data, source_context):
config = model.UnparsedTenantConfig()
- config.extend(safe_load_yaml(data, source_context))
+ with early_configuration_exceptions(source_context):
+ config.extend(safe_load_yaml(data, source_context))
if config.pipelines:
with configuration_exceptions('pipeline', config.pipelines[0]):
raise PipelineNotPermittedError()
@@ -1404,7 +1465,7 @@
continue
with configuration_exceptions('nodeset', config_nodeset):
layout.addNodeSet(NodeSetParser.fromYaml(
- layout, config_nodeset))
+ config_nodeset))
for config_secret in data.secrets:
classes = TenantParser._getLoadClasses(tenant, config_secret)
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index ecf5f94..83871e3 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -340,7 +340,7 @@
change.newrev = event.newrev
change.url = self._getGitwebUrl(project, sha=event.newrev)
elif event.ref and not event.ref.startswith('refs/'):
- # Gerrit ref-updated events don't have branch prefixes.
+ # Pre 2.13 Gerrit ref-updated events don't have branch prefixes.
project = self.source.getProject(event.project_name)
change = Branch(project)
change.branch = event.ref
@@ -349,11 +349,11 @@
change.newrev = event.newrev
change.url = self._getGitwebUrl(project, sha=event.newrev)
elif event.ref and event.ref.startswith('refs/heads/'):
- # From the timer trigger
+ # From the timer trigger or Post 2.13 Gerrit
project = self.source.getProject(event.project_name)
change = Branch(project)
change.ref = event.ref
- change.branch = event.branch
+ change.branch = event.ref[len('refs/heads/'):]
change.oldrev = event.oldrev
change.newrev = event.newrev
change.url = self._getGitwebUrl(project, sha=event.newrev)
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 0ce6ef5..3d0eb37 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -17,6 +17,8 @@
import logging
import hmac
import hashlib
+import queue
+import threading
import time
import re
@@ -80,11 +82,10 @@
delivery=delivery))
self._validate_signature(request)
+ # TODO(jlk): Validate project in the request is a project we know
try:
self.__dispatch_event(request)
- except webob.exc.HTTPNotFound:
- raise
except:
self.log.exception("Exception handling Github event:")
@@ -98,20 +99,58 @@
'header.')
try:
- method = getattr(self, '_event_' + event)
- except AttributeError:
- message = "Unhandled X-Github-Event: {0}".format(event)
- self.log.debug(message)
- # Returns empty 200 on unhandled events
- raise webob.exc.HTTPOk()
-
- try:
json_body = request.json_body
+ self.connection.addEvent(json_body, event)
except:
message = 'Exception deserializing JSON body'
self.log.exception(message)
raise webob.exc.HTTPBadRequest(message)
+ def _validate_signature(self, request):
+ secret = self.connection.connection_config.get('webhook_token', None)
+ if secret is None:
+ raise RuntimeError("webhook_token is required")
+
+ body = request.body
+ try:
+ request_signature = request.headers['X-Hub-Signature']
+ except KeyError:
+ raise webob.exc.HTTPUnauthorized(
+ 'Please specify a X-Hub-Signature header with secret.')
+
+ payload_signature = _sign_request(body, secret)
+
+ self.log.debug("Payload Signature: {0}".format(str(payload_signature)))
+ self.log.debug("Request Signature: {0}".format(str(request_signature)))
+ if not hmac.compare_digest(
+ str(payload_signature), str(request_signature)):
+ raise webob.exc.HTTPUnauthorized(
+ 'Request signature does not match calculated payload '
+ 'signature. Check that secret is correct.')
+
+ return True
+
+
+class GithubEventConnector(threading.Thread):
+ """Move events from GitHub into the scheduler"""
+
+ log = logging.getLogger("zuul.GithubEventConnector")
+
+ def __init__(self, connection):
+ super(GithubEventConnector, self).__init__()
+ self.daemon = True
+ self.connection = connection
+ self._stopped = False
+
+ def stop(self):
+ self._stopped = True
+ self.connection.addEvent(None)
+
+ def _handleEvent(self):
+ json_body, event_type = self.connection.getEvent()
+ if self._stopped:
+ return
+
# If there's any installation mapping information in the body then
# update the project mapping before any requests are made.
installation_id = json_body.get('installation', {}).get('id')
@@ -127,9 +166,17 @@
self.connection.installation_map[project_name] = installation_id
try:
+ method = getattr(self, '_event_' + event_type)
+ except AttributeError:
+ # TODO(jlk): Gracefully handle event types we don't care about
+ # instead of logging an exception.
+ message = "Unhandled X-Github-Event: {0}".format(event_type)
+ self.log.debug(message)
+ # Returns empty on unhandled events
+ return
+
+ try:
event = method(json_body)
- except webob.exc.HTTPNotFound:
- raise
except:
self.log.exception('Exception when handling event:')
event = None
@@ -240,14 +287,6 @@
event.action = body.get('action')
return event
- def _event_ping(self, body):
- project_name = body['repository']['full_name']
- if not self.connection.getProject(project_name):
- self.log.warning("Ping received for unknown project %s" %
- project_name)
- raise webob.exc.HTTPNotFound("Sorry, this project is not "
- "registered")
-
def _event_status(self, body):
action = body.get('action')
if action == 'pending':
@@ -277,30 +316,6 @@
(number, project_name))
return pr_body
- def _validate_signature(self, request):
- secret = self.connection.connection_config.get('webhook_token', None)
- if secret is None:
- raise RuntimeError("webhook_token is required")
-
- body = request.body
- try:
- request_signature = request.headers['X-Hub-Signature']
- except KeyError:
- raise webob.exc.HTTPUnauthorized(
- 'Please specify a X-Hub-Signature header with secret.')
-
- payload_signature = _sign_request(body, secret)
-
- self.log.debug("Payload Signature: {0}".format(str(payload_signature)))
- self.log.debug("Request Signature: {0}".format(str(request_signature)))
- if not hmac.compare_digest(
- str(payload_signature), str(request_signature)):
- raise webob.exc.HTTPUnauthorized(
- 'Request signature does not match calculated payload '
- 'signature. Check that secret is correct.')
-
- return True
-
def _pull_request_to_event(self, pr_body):
event = GithubTriggerEvent()
event.trigger_name = 'github'
@@ -327,6 +342,17 @@
if login:
return self.connection.getUser(login)
+ def run(self):
+ while True:
+ if self._stopped:
+ return
+ try:
+ self._handleEvent()
+ except:
+ self.log.exception("Exception moving GitHub event:")
+ finally:
+ self.connection.eventDone()
+
class GithubUser(collections.Mapping):
log = logging.getLogger('zuul.GithubUser')
@@ -376,6 +402,7 @@
self.canonical_hostname = self.connection_config.get(
'canonical_hostname', self.server)
self.source = driver.getSource(self)
+ self.event_queue = queue.Queue()
# ssl verification must default to true
verify_ssl = self.connection_config.get('verify_ssl', 'true')
@@ -408,9 +435,20 @@
self.registerHttpHandler(self.payload_path,
webhook_listener.handle_request)
self._authenticateGithubAPI()
+ self._start_event_connector()
def onStop(self):
self.unregisterHttpHandler(self.payload_path)
+ self._stop_event_connector()
+
+ def _start_event_connector(self):
+ self.github_event_connector = GithubEventConnector(self)
+ self.github_event_connector.start()
+
+ def _stop_event_connector(self):
+ if self.github_event_connector:
+ self.github_event_connector.stop()
+ self.github_event_connector.join()
def _createGithubClient(self):
if self.server != 'github.com':
@@ -504,15 +542,21 @@
return token
+ def addEvent(self, data, event=None):
+ return self.event_queue.put((data, event))
+
+ def getEvent(self):
+ return self.event_queue.get()
+
+ def eventDone(self):
+ self.event_queue.task_done()
+
def getGithubClient(self,
project=None,
- user_id=None,
- use_app=True):
+ user_id=None):
# if you're authenticating for a project and you're an integration then
- # you need to use the installation specific token. There are some
- # operations that are not yet supported by integrations so
- # use_app lets you use api_key auth.
- if use_app and project and self.app_id:
+ # you need to use the installation specific token.
+ if project and self.app_id:
github = self._createGithubClient()
github.login(token=self._get_installation_key(project, user_id))
github._zuul_project = project
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
index 3b8f518..505757f 100644
--- a/zuul/driver/github/githubreporter.py
+++ b/zuul/driver/github/githubreporter.py
@@ -112,10 +112,10 @@
description = 'status: %s' % self._commit_status
self.log.debug(
- 'Reporting change %s, params %s, status:\n'
+ 'Reporting change %s, params %s, '
'context: %s, state: %s, description: %s, url: %s' %
- (item.change, self.config, context, state,
- description, url))
+ (item.change, self.config,
+ context, state, description, url))
self.connection.setCommitStatus(
project, sha, state, url, description, context)
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index be41186..f97d286 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -175,6 +175,8 @@
zuul_params['tag'] = item.change.tag
if hasattr(item.change, 'number'):
zuul_params['change'] = str(item.change.number)
+ if hasattr(item.change, 'url'):
+ zuul_params['change_url'] = item.change.url
if hasattr(item.change, 'patchset'):
zuul_params['patchset'] = str(item.change.patchset)
if (hasattr(item.change, 'oldrev') and item.change.oldrev
@@ -196,6 +198,8 @@
)
if hasattr(i.change, 'number'):
d['change'] = str(i.change.number)
+ if hasattr(i.change, 'url'):
+ d['change_url'] = i.change.url
if hasattr(i.change, 'patchset'):
d['patchset'] = str(i.change.patchset)
if hasattr(i.change, 'branch'):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index de928db..27fd85f 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -36,7 +36,7 @@
import gear
import zuul.merger.merger
-import zuul.ansible
+import zuul.ansible.logconfig
from zuul.lib import commandsocket
BUFFER_LINES_FOR_SYNTAX = 200
@@ -203,7 +203,7 @@
except OSError:
self.log.exception(
'Problem sending SIGTERM to agent {}'.format(self.env))
- self.log.info('Sent SIGTERM to SSH Agent, {}'.format(self.env))
+ self.log.debug('Sent SIGTERM to SSH Agent, {}'.format(self.env))
self.env = {}
def add(self, key_path):
@@ -567,6 +567,7 @@
self.action_dir = os.path.join(plugin_dir, 'action')
self.callback_dir = os.path.join(plugin_dir, 'callback')
self.lookup_dir = os.path.join(plugin_dir, 'lookup')
+ self.filter_dir = os.path.join(plugin_dir, 'filter')
_copy_ansible_files(zuul.ansible, plugin_dir)
@@ -1162,6 +1163,7 @@
ansible_user=self.executor_server.default_username,
ansible_port=port,
nodepool=dict(
+ label=node.get('label'),
az=node.get('az'),
cloud=node.get('cloud'),
provider=node.get('provider'),
@@ -1445,6 +1447,8 @@
config.write('command_warnings = False\n')
config.write('callback_plugins = %s\n' % callback_path)
config.write('stdout_callback = zuul_stream\n')
+ config.write('filter_plugins = %s\n'
+ % self.executor_server.filter_dir)
# bump the timeout because busy nodes may take more than
# 10s to respond
config.write('timeout = 30\n')
@@ -1617,7 +1621,22 @@
now=datetime.datetime.now()))
for line in syntax_buffer:
job_output.write("{now} | {line}\n".format(
- now=datetime.datetime.now(), line=line))
+ now=datetime.datetime.now(),
+ line=line.decode('utf-8').rstrip()))
+ elif ret == 250:
+ # Unexpected error from ansible
+ with open(self.jobdir.job_output_file, 'a') as job_output:
+ job_output.write("{now} | UNEXPECTED ANSIBLE ERROR\n".format(
+ now=datetime.datetime.now()))
+ found_marker = False
+ for line in syntax_buffer:
+ if line.startswith('ERROR! Unexpected Exception'):
+ found_marker = True
+ if not found_marker:
+ continue
+ job_output.write("{now} | {line}\n".format(
+ now=datetime.datetime.now(),
+ line=line.decode('utf-8').rstrip()))
return (self.RESULT_NORMAL, ret)
@@ -1633,7 +1652,7 @@
cmd.extend(['-e', '@' + playbook.secrets])
if success is not None:
- cmd.extend(['-e', 'success=%s' % str(bool(success))])
+ cmd.extend(['-e', 'zuul_success=%s' % str(bool(success))])
if phase:
cmd.extend(['-e', 'zuul_execution_phase=%s' % phase])
diff --git a/zuul/lib/commandsocket.py b/zuul/lib/commandsocket.py
index 901291a..2836999 100644
--- a/zuul/lib/commandsocket.py
+++ b/zuul/lib/commandsocket.py
@@ -44,9 +44,9 @@
# First, wake up our listener thread with a connection and
# tell it to stop running.
self.running = False
- s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
- s.connect(self.path)
- s.sendall(b'_stop\n')
+ with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
+ s.connect(self.path)
+ s.sendall(b'_stop\n')
# The command '_stop' will be ignored by our listener, so
# directly inject it into the queue so that consumers of this
# class which are waiting in .get() are awakened. They can
diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py
index 57afef9..3ecaf4d 100644
--- a/zuul/lib/log_streamer.py
+++ b/zuul/lib/log_streamer.py
@@ -168,6 +168,8 @@
'''
Custom version that allows us to drop privileges after port binding.
'''
+ address_family = socket.AF_INET6
+
def __init__(self, *args, **kwargs):
self.user = kwargs.pop('user')
self.jobdir_root = kwargs.pop('jobdir_root')
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index efd86eb..98c7350 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -529,11 +529,12 @@
if not item.job_graph:
try:
+ self.log.debug("Freezing job graph for %s" % (item,))
item.freezeJobGraph()
except Exception as e:
# TODOv3(jeblair): nicify this exception as it will be reported
self.log.exception("Error freezing job graph for %s" %
- item)
+ (item,))
item.setConfigError("Unable to freeze job graph: %s" %
(str(e)))
return False
@@ -752,9 +753,12 @@
layout = (item.current_build_set.layout or
self.pipeline.layout)
- if not layout.hasProject(item.change.project):
+ project_in_pipeline = True
+ if not layout.getProjectPipelineConfig(item.change.project,
+ self.pipeline):
self.log.debug("Project %s not in pipeline %s for change %s" % (
item.change.project, self.pipeline, item.change))
+ project_in_pipeline = False
actions = []
elif item.getConfigError():
self.log.debug("Invalid config for change %s" % item.change)
@@ -780,7 +784,7 @@
actions = self.pipeline.failure_actions
item.setReportedResult('FAILURE')
self.pipeline._consecutive_failures += 1
- if layout.hasProject(item.change.project) and self.pipeline._disabled:
+ if project_in_pipeline and self.pipeline._disabled:
actions = self.pipeline.disabled_actions
# Check here if we should disable so that we only use the disabled
# reporters /after/ the last disable_at failure is still reported as
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 5191a44..2614e58 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -134,18 +134,18 @@
def onBuildCompleted(self, job):
data = getJobData(job)
merged = data.get('merged', False)
- updated = data.get('updated', False)
+ job.updated = data.get('updated', False)
commit = data.get('commit')
files = data.get('files', {})
repo_state = data.get('repo_state', {})
job.files = files
self.log.info("Merge %s complete, merged: %s, updated: %s, "
"commit: %s" %
- (job, merged, updated, commit))
+ (job, merged, job.updated, commit))
job.setComplete()
if job.build_set:
self.sched.onMergeCompleted(job.build_set,
- merged, updated, commit, files,
+ merged, job.updated, commit, files,
repo_state)
# The test suite expects the job to be removed from the
# internal account after the wake flag is set.
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index ed98696..8b98bfb 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -191,11 +191,14 @@
def checkout(self, ref):
repo = self.createRepoObject()
self.log.debug("Checking out %s" % ref)
- repo.head.reference = ref
+ # Perform a hard reset before checking out so that we clean up
+ # anything that might be left over from a merge.
reset_repo_to_head(repo)
+ repo.git.checkout(ref)
return repo.head.commit
def checkoutLocalBranch(self, branch):
+ # TODO(jeblair): retire in favor of checkout
repo = self.createRepoObject()
# Perform a hard reset before checking out so that we clean up
# anything that might be left over from a merge.
@@ -341,10 +344,6 @@
return self._addProject(hostname, project_name, url, sshkey)
def updateRepo(self, connection_name, project_name):
- # TODOv3(jhesketh): Reimplement
- # da90a50b794f18f74de0e2c7ec3210abf79dda24 after merge..
- # Likely we'll handle connection context per projects differently.
- # self._setGitSsh()
repo = self.getRepo(connection_name, project_name)
try:
self.log.info("Updating local repository %s/%s",
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index fc599c1..881209d 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -111,7 +111,7 @@
def refstate(self, job):
args = json.loads(job.arguments)
- success, repo_state = self.merger.getItemRepoState(args['items'])
+ success, repo_state = self.merger.getRepoState(args['items'])
result = dict(updated=success,
repo_state=repo_state)
job.sendWorkComplete(json.dumps(result))
diff --git a/zuul/model.py b/zuul/model.py
index 850bbe2..0e42368 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -21,6 +21,7 @@
import time
from uuid import uuid4
import urllib.parse
+import textwrap
MERGER_MERGE = 1 # "git merge"
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
@@ -1374,6 +1375,7 @@
self.quiet = False
self.active = False # Whether an item is within an active window
self.live = True # Whether an item is intended to be processed at all
+ # TODO(jeblair): move job_graph to buildset
self.job_graph = None
def __repr__(self):
@@ -1391,6 +1393,7 @@
old.next_build_set = self.current_build_set
self.current_build_set.previous_build_set = old
self.build_sets.append(self.current_build_set)
+ self.job_graph = None
def addBuild(self, build):
self.current_build_set.addBuild(build)
@@ -2093,6 +2096,83 @@
self.private_key_file = None
+class ConfigItemNotListError(Exception):
+ def __init__(self):
+ message = textwrap.dedent("""\
+ Configuration file is not a list. Each zuul.yaml configuration
+ file must be a list of items, for example:
+
+ - job:
+ name: foo
+
+ - project:
+ name: bar
+
+ Ensure that every item starts with "- " so that it is parsed as a
+ YAML list.
+ """)
+ super(ConfigItemNotListError, self).__init__(message)
+
+
+class ConfigItemNotDictError(Exception):
+ def __init__(self):
+ message = textwrap.dedent("""\
+ Configuration item is not a dictionary. Each zuul.yaml
+ configuration file must be a list of dictionaries, for
+ example:
+
+ - job:
+ name: foo
+
+ - project:
+ name: bar
+
+ Ensure that every item in the list is a dictionary with one
+ key (in this example, 'job' and 'project').
+ """)
+ super(ConfigItemNotDictError, self).__init__(message)
+
+
+class ConfigItemMultipleKeysError(Exception):
+ def __init__(self):
+ message = textwrap.dedent("""\
+ Configuration item has more than one key. Each zuul.yaml
+ configuration file must be a list of dictionaries with a
+ single key, for example:
+
+ - job:
+ name: foo
+
+ - project:
+ name: bar
+
+ Ensure that every item in the list is a dictionary with only
+ one key (in this example, 'job' and 'project'). This error
+ may be caused by insufficient indentation of the keys under
+ the configuration item ('name' in this example).
+ """)
+ super(ConfigItemMultipleKeysError, self).__init__(message)
+
+
+class ConfigItemUnknownError(Exception):
+ def __init__(self):
+ message = textwrap.dedent("""\
+ Configuration item not recognized. Each zuul.yaml
+ configuration file must be a list of dictionaries, for
+ example:
+
+ - job:
+ name: foo
+
+ - project:
+ name: bar
+
+ The dictionary keys must match one of the configuration item
+ types recognized by zuul (for example, 'job' or 'project').
+ """)
+ super(ConfigItemUnknownError, self).__init__(message)
+
+
class UnparsedAbideConfig(object):
"""A collection of yaml lists that has not yet been parsed into objects.
@@ -2109,25 +2189,18 @@
return
if not isinstance(conf, list):
- raise Exception("Configuration items must be in the form of "
- "a list of dictionaries (when parsing %s)" %
- (conf,))
+ raise ConfigItemNotListError()
+
for item in conf:
if not isinstance(item, dict):
- raise Exception("Configuration items must be in the form of "
- "a list of dictionaries (when parsing %s)" %
- (conf,))
+ raise ConfigItemNotDictError()
if len(item.keys()) > 1:
- raise Exception("Configuration item dictionaries must have "
- "a single key (when parsing %s)" %
- (conf,))
+ raise ConfigItemMultipleKeysError()
key, value = list(item.items())[0]
if key == 'tenant':
self.tenants.append(value)
else:
- raise Exception("Configuration item not recognized "
- "(when parsing %s)" %
- (conf,))
+ raise ConfigItemUnknownError()
class UnparsedTenantConfig(object):
@@ -2166,19 +2239,13 @@
return
if not isinstance(conf, list):
- raise Exception("Configuration items must be in the form of "
- "a list of dictionaries (when parsing %s)" %
- (conf,))
+ raise ConfigItemNotListError()
for item in conf:
if not isinstance(item, dict):
- raise Exception("Configuration items must be in the form of "
- "a list of dictionaries (when parsing %s)" %
- (conf,))
+ raise ConfigItemNotDictError()
if len(item.keys()) > 1:
- raise Exception("Configuration item dictionaries must have "
- "a single key (when parsing %s)" %
- (conf,))
+ raise ConfigItemMultipleKeysError()
key, value = list(item.items())[0]
if key == 'project':
name = value['name']
@@ -2196,9 +2263,7 @@
elif key == 'semaphore':
self.semaphores.append(value)
else:
- raise Exception("Configuration item `%s` not recognized "
- "(when parsing %s)" %
- (item, conf,))
+ raise ConfigItemUnknownError()
class Layout(object):
@@ -2331,19 +2396,21 @@
job_graph.addJob(frozen_job)
def createJobGraph(self, item):
- project_config = self.project_configs.get(
- item.change.project.canonical_name, None)
- ret = JobGraph()
# NOTE(pabelanger): It is possible for a foreign project not to have a
# configured pipeline, if so return an empty JobGraph.
- if project_config and item.pipeline.name in project_config.pipelines:
- project_job_list = \
- project_config.pipelines[item.pipeline.name].job_list
- self._createJobGraph(item, project_job_list, ret)
+ ret = JobGraph()
+ ppc = self.getProjectPipelineConfig(item.change.project,
+ item.pipeline)
+ if ppc:
+ self._createJobGraph(item, ppc.job_list, ret)
return ret
- def hasProject(self, project):
- return project.canonical_name in self.project_configs
+ def getProjectPipelineConfig(self, project, pipeline):
+ project_config = self.project_configs.get(
+ project.canonical_name, None)
+ if not project_config:
+ return None
+ return project_config.pipelines.get(pipeline.name, None)
class Semaphore(object):
@@ -2439,6 +2506,7 @@
def __init__(self, name):
self.name = name
self.max_nodes_per_job = 5
+ self.max_job_timeout = 10800
self.exclude_unprotected_branches = False
self.default_base_job = None
self.layout = None
@@ -2600,20 +2668,30 @@
class TimeDataBase(object):
def __init__(self, root):
self.root = root
- self.jobs = {}
- def _getTD(self, name):
- td = self.jobs.get(name)
- if not td:
- td = JobTimeData(os.path.join(self.root, name))
- self.jobs[name] = td
- td.load()
+ def _getTD(self, build):
+ if hasattr(build.build_set.item.change, 'branch'):
+ branch = build.build_set.item.change.branch
+ else:
+ branch = ''
+
+ dir_path = os.path.join(
+ self.root,
+ build.build_set.item.pipeline.layout.tenant.name,
+ build.build_set.item.change.project.canonical_name,
+ branch)
+ if not os.path.exists(dir_path):
+ os.makedirs(dir_path)
+ path = os.path.join(dir_path, build.job.name)
+
+ td = JobTimeData(path)
+ td.load()
return td
def getEstimatedTime(self, name):
return self._getTD(name).getEstimatedTime()
- def update(self, name, elapsed, result):
- td = self._getTD(name)
+ def update(self, build, elapsed, result):
+ td = self._getTD(build)
td.add(elapsed, result)
td.save()
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 52b34ec..5432661 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -543,9 +543,16 @@
tenant, item)
item.item_ahead = None
item.items_behind = []
- if (item.change.project and
- new_pipeline.manager.reEnqueueItem(item,
- last_head)):
+ reenqueued = False
+ if item.change.project:
+ try:
+ reenqueued = new_pipeline.manager.reEnqueueItem(
+ item, last_head)
+ except Exception:
+ self.log.exception(
+ "Exception while re-enqueing item %s",
+ item)
+ if reenqueued:
for build in item.current_build_set.getBuilds():
new_job = item.getJob(build.job.name)
if new_job:
@@ -826,7 +833,7 @@
return
try:
build.estimated_time = float(self.time_database.getEstimatedTime(
- build.job.name))
+ build))
except Exception:
self.log.exception("Exception estimating build time:")
pipeline.manager.onBuildStarted(event.build)
@@ -865,8 +872,7 @@
if build.end_time and build.start_time and build.result:
duration = build.end_time - build.start_time
try:
- self.time_database.update(
- build.job.name, duration, build.result)
+ self.time_database.update(build, duration, build.result)
except Exception:
self.log.exception("Exception recording build time:")
pipeline.manager.onBuildCompleted(event.build)