Merge "Write inventory as yaml not ini" into feature/zuulv3
diff --git a/doc/source/reporters.rst b/doc/source/reporters.rst
index e3ab947..dd053fa 100644
--- a/doc/source/reporters.rst
+++ b/doc/source/reporters.rst
@@ -44,6 +44,10 @@
set as the commit status on github.
``status: 'success'``
+ **status-url**
+ String value for a link url to set in the github status. Defaults to the zuul
+ server status_url, or the empty string if that is unset.
+
**comment**
Boolean value (``true`` or ``false``) that determines if the reporter should
add a comment to the pipeline status to the github pull request. Defaults
diff --git a/tests/base.py b/tests/base.py
index c6f1a4b..48716e2 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1221,6 +1221,25 @@
self.log.debug(" OK")
return True
+ def getWorkspaceRepos(self, projects):
+ """Return workspace git repo objects for the listed projects
+
+ :arg list projects: A list of strings, each the canonical name
+ of a project.
+
+ :returns: A dictionary of {name: repo} for every listed
+ project.
+ :rtype: dict
+
+ """
+
+ repos = {}
+ for project in projects:
+ path = os.path.join(self.jobdir.src_root, project)
+ repo = git.Repo(path)
+ repos[project] = repo
+ return repos
+
class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
"""An Ansible executor to be used in tests.
@@ -1307,10 +1326,10 @@
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
- def doMergeChanges(self, items, repo_state):
+ def doMergeChanges(self, merger, items, repo_state):
# Get a merger in order to update the repos involved in this job.
commit = super(RecordingAnsibleJob, self).doMergeChanges(
- items, repo_state)
+ merger, items, repo_state)
if not commit: # merge conflict
self.recordResult('MERGER_FAILURE')
return commit
@@ -2190,6 +2209,8 @@
def shutdown(self):
self.log.debug("Shutting down after tests")
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
self.executor_client.stop()
self.merge_client.stop()
self.executor_server.stop()
@@ -2675,6 +2696,29 @@
specified_conn.server == conn.server):
conn.addEvent(event)
+ def getUpstreamRepos(self, projects):
+ """Return upstream git repo objects for the listed projects
+
+ :arg list projects: A list of strings, each the canonical name
+ of a project.
+
+ :returns: A dictionary of {name: repo} for every listed
+ project.
+ :rtype: dict
+
+ """
+
+ repos = {}
+ for project in projects:
+ # FIXME(jeblair): the upstream root does not yet have a
+ # hostname component; that needs to be added, and this
+ # line removed:
+ tmp_project_name = '/'.join(project.split('/')[1:])
+ path = os.path.join(self.upstream_root, tmp_project_name)
+ repo = git.Repo(path)
+ repos[project] = repo
+ return repos
+
class AnsibleZuulTestCase(ZuulTestCase):
"""ZuulTestCase but with an actual ansible executor running"""
diff --git a/tests/fixtures/layouts/repo-checkout-four-project.yaml b/tests/fixtures/layouts/repo-checkout-four-project.yaml
new file mode 100644
index 0000000..392931a
--- /dev/null
+++ b/tests/fixtures/layouts/repo-checkout-four-project.yaml
@@ -0,0 +1,81 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: integration
+ required-projects:
+ - org/project1
+ - org/project2
+ - org/project3
+ - org/project4
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project3
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project4
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
diff --git a/tests/fixtures/layouts/repo-checkout-no-timer.yaml b/tests/fixtures/layouts/repo-checkout-no-timer.yaml
new file mode 100644
index 0000000..2b65850
--- /dev/null
+++ b/tests/fixtures/layouts/repo-checkout-no-timer.yaml
@@ -0,0 +1,20 @@
+- pipeline:
+ name: periodic
+ manager: independent
+ # Trigger is required, set it to one that is a noop
+ # during tests that check the timer trigger.
+ trigger:
+ gerrit:
+ - event: ref-updated
+
+- job:
+ name: integration
+ override-branch: stable/havana
+ required-projects:
+ - org/project1
+
+- project:
+ name: org/project1
+ periodic:
+ jobs:
+ - integration
diff --git a/tests/fixtures/layouts/repo-checkout-post.yaml b/tests/fixtures/layouts/repo-checkout-post.yaml
new file mode 100644
index 0000000..9698289
--- /dev/null
+++ b/tests/fixtures/layouts/repo-checkout-post.yaml
@@ -0,0 +1,25 @@
+- pipeline:
+ name: post
+ manager: independent
+ trigger:
+ gerrit:
+ - event: ref-updated
+ ref: ^(?!refs/).*$
+
+- job:
+ name: integration
+ required-projects:
+ - org/project1
+ - org/project2
+
+- project:
+ name: org/project1
+ post:
+ jobs:
+ - integration
+
+- project:
+ name: org/project2
+ post:
+ jobs:
+ - integration
diff --git a/tests/fixtures/layouts/repo-checkout-six-project.yaml b/tests/fixtures/layouts/repo-checkout-six-project.yaml
new file mode 100644
index 0000000..93a64ea
--- /dev/null
+++ b/tests/fixtures/layouts/repo-checkout-six-project.yaml
@@ -0,0 +1,104 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: integration
+ required-projects:
+ - org/project1
+ - org/project2
+ - org/project3
+ - name: org/project4
+ override-branch: master
+ - org/project5
+ - org/project6
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project3
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project4
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project5
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project6
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
diff --git a/tests/fixtures/layouts/repo-checkout-timer.yaml b/tests/fixtures/layouts/repo-checkout-timer.yaml
new file mode 100644
index 0000000..d5917d1
--- /dev/null
+++ b/tests/fixtures/layouts/repo-checkout-timer.yaml
@@ -0,0 +1,18 @@
+- pipeline:
+ name: periodic
+ manager: independent
+ trigger:
+ timer:
+ - time: '* * * * * */1'
+
+- job:
+ name: integration
+ override-branch: stable/havana
+ required-projects:
+ - org/project1
+
+- project:
+ name: org/project1
+ periodic:
+ jobs:
+ - integration
diff --git a/tests/fixtures/layouts/repo-checkout-two-project.yaml b/tests/fixtures/layouts/repo-checkout-two-project.yaml
new file mode 100644
index 0000000..239d80c
--- /dev/null
+++ b/tests/fixtures/layouts/repo-checkout-two-project.yaml
@@ -0,0 +1,59 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+- pipeline:
+ name: gate
+ manager: dependent
+ success-message: Build succeeded (gate).
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+- job:
+ name: integration
+ required-projects:
+ - org/project1
+ - org/project2
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - integration
+ gate:
+ queue: integrated
+ jobs:
+ - integration
diff --git a/tests/fixtures/layouts/reporting-github.yaml b/tests/fixtures/layouts/reporting-github.yaml
index c939f39..8dd35b0 100644
--- a/tests/fixtures/layouts/reporting-github.yaml
+++ b/tests/fixtures/layouts/reporting-github.yaml
@@ -29,6 +29,7 @@
github:
comment: false
status: 'success'
+ status-url: http://logs.example.com/{pipeline.name}/{change.project}/{change.number}/{change.patchset}/
failure:
github:
comment: false
diff --git a/tests/fixtures/zuul-github-driver.conf b/tests/fixtures/zuul-github-driver.conf
index ab34619..dfa813d 100644
--- a/tests/fixtures/zuul-github-driver.conf
+++ b/tests/fixtures/zuul-github-driver.conf
@@ -3,7 +3,7 @@
[zuul]
job_name_in_report=true
-status_url=http://zuul.example.com/status
+status_url=http://zuul.example.com/status/#{change.number},{change.patchset}
[merger]
git_dir=/tmp/zuul-test/git
diff --git a/tests/unit/test_cloner.py b/tests/unit/test_cloner.py
deleted file mode 100644
index e65904b..0000000
--- a/tests/unit/test_cloner.py
+++ /dev/null
@@ -1,752 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2012 Hewlett-Packard Development Company, L.P.
-# Copyright 2014 Wikimedia Foundation Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import logging
-import os
-import shutil
-import time
-
-import git
-
-import zuul.lib.cloner
-
-from tests.base import ZuulTestCase
-
-
-class TestCloner(ZuulTestCase):
-
- log = logging.getLogger("zuul.test.cloner")
- workspace_root = None
-
- def setUp(self):
- self.skip("Disabled for early v3 development")
-
- super(TestCloner, self).setUp()
- self.workspace_root = os.path.join(self.test_root, 'workspace')
-
- self.updateConfigLayout(
- 'tests/fixtures/layout-cloner.yaml')
- self.sched.reconfigure(self.config)
- self.registerJobs()
-
- def getWorkspaceRepos(self, projects):
- repos = {}
- for project in projects:
- repos[project] = git.Repo(
- os.path.join(self.workspace_root, project))
- return repos
-
- def getUpstreamRepos(self, projects):
- repos = {}
- for project in projects:
- repos[project] = git.Repo(
- os.path.join(self.upstream_root, project))
- return repos
-
- def test_cache_dir(self):
- projects = ['org/project1', 'org/project2']
- cache_root = os.path.join(self.test_root, "cache")
- for project in projects:
- upstream_repo_path = os.path.join(self.upstream_root, project)
- cache_repo_path = os.path.join(cache_root, project)
- git.Repo.clone_from(upstream_repo_path, cache_repo_path)
-
- self.worker.hold_jobs_in_build = True
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- A.addApproval('CRVW', 2)
- self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
-
- self.waitUntilSettled()
-
- self.assertEquals(1, len(self.builds), "One build is running")
-
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- B.setMerged()
-
- upstream = self.getUpstreamRepos(projects)
- states = [{
- 'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit('master')),
- }]
-
- for number, build in enumerate(self.builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters['ZUUL_BRANCH'],
- zuul_ref=build.parameters['ZUUL_REF'],
- zuul_url=self.src_root,
- cache_dir=cache_root,
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, number))
-
- work = self.getWorkspaceRepos(projects)
- # project1 is the zuul_project so the origin should be set to the
- # zuul_url since that is the most up to date.
- cache_repo_path = os.path.join(cache_root, 'org/project1')
- self.assertNotEqual(
- work['org/project1'].remotes.origin.url,
- cache_repo_path,
- 'workspace repo origin should not be the cache'
- )
- zuul_url_repo_path = os.path.join(self.git_root, 'org/project1')
- self.assertEqual(
- work['org/project1'].remotes.origin.url,
- zuul_url_repo_path,
- 'workspace repo origin should be the zuul url'
- )
-
- # project2 is not the zuul_project so the origin should be set
- # to upstream since that is the best we can do
- cache_repo_path = os.path.join(cache_root, 'org/project2')
- self.assertNotEqual(
- work['org/project2'].remotes.origin.url,
- cache_repo_path,
- 'workspace repo origin should not be the cache'
- )
- upstream_repo_path = os.path.join(self.upstream_root, 'org/project2')
- self.assertEqual(
- work['org/project2'].remotes.origin.url,
- upstream_repo_path,
- 'workspace repo origin should be the upstream url'
- )
-
- self.worker.hold_jobs_in_build = False
- self.worker.release()
- self.waitUntilSettled()
-
- def test_one_branch(self):
- self.worker.hold_jobs_in_build = True
-
- projects = ['org/project1', 'org/project2']
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- A.addApproval('CRVW', 2)
- B.addApproval('CRVW', 2)
- self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
-
- self.waitUntilSettled()
-
- self.assertEquals(2, len(self.builds), "Two builds are running")
-
- upstream = self.getUpstreamRepos(projects)
- states = [
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
- },
- ]
-
- for number, build in enumerate(self.builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters['ZUUL_BRANCH'],
- zuul_ref=build.parameters['ZUUL_REF'],
- zuul_url=self.src_root,
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, number))
-
- shutil.rmtree(self.workspace_root)
-
- self.worker.hold_jobs_in_build = False
- self.worker.release()
- self.waitUntilSettled()
-
- def test_multi_branch(self):
- self.worker.hold_jobs_in_build = True
- projects = ['org/project1', 'org/project2',
- 'org/project3', 'org/project4']
-
- self.create_branch('org/project2', 'stable/havana')
- self.create_branch('org/project4', 'stable/havana')
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana',
- 'B')
- C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
- A.addApproval('CRVW', 2)
- B.addApproval('CRVW', 2)
- C.addApproval('CRVW', 2)
- self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
-
- self.waitUntilSettled()
-
- self.assertEquals(3, len(self.builds), "Three builds are running")
-
- upstream = self.getUpstreamRepos(projects)
- states = [
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit('master')),
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].
- commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].
- commit('stable/havana')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit('master')),
- 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
- 'org/project4': str(upstream['org/project4'].
- commit('master')),
- },
- ]
-
- for number, build in enumerate(self.builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters['ZUUL_BRANCH'],
- zuul_ref=build.parameters['ZUUL_REF'],
- zuul_url=self.src_root,
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, number))
- shutil.rmtree(self.workspace_root)
-
- self.worker.hold_jobs_in_build = False
- self.worker.release()
- self.waitUntilSettled()
-
- def test_upgrade(self):
- # Simulates an upgrade test
- self.worker.hold_jobs_in_build = True
- projects = ['org/project1', 'org/project2', 'org/project3',
- 'org/project4', 'org/project5', 'org/project6']
-
- self.create_branch('org/project2', 'stable/havana')
- self.create_branch('org/project3', 'stable/havana')
- self.create_branch('org/project4', 'stable/havana')
- self.create_branch('org/project5', 'stable/havana')
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
- C = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana',
- 'C')
- D = self.fake_gerrit.addFakeChange('org/project3', 'master', 'D')
- E = self.fake_gerrit.addFakeChange('org/project4', 'stable/havana',
- 'E')
- A.addApproval('CRVW', 2)
- B.addApproval('CRVW', 2)
- C.addApproval('CRVW', 2)
- D.addApproval('CRVW', 2)
- E.addApproval('CRVW', 2)
- self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(E.addApproval('APRV', 1))
-
- self.waitUntilSettled()
-
- self.assertEquals(5, len(self.builds), "Five builds are running")
-
- # Check the old side of the upgrade first
- upstream = self.getUpstreamRepos(projects)
- states = [
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit(
- 'stable/havana')),
- 'org/project3': str(upstream['org/project3'].commit(
- 'stable/havana')),
- 'org/project4': str(upstream['org/project4'].commit(
- 'stable/havana')),
- 'org/project5': str(upstream['org/project5'].commit(
- 'stable/havana')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit(
- 'stable/havana')),
- 'org/project3': str(upstream['org/project3'].commit(
- 'stable/havana')),
- 'org/project4': str(upstream['org/project4'].commit(
- 'stable/havana')),
- 'org/project5': str(upstream['org/project5'].commit(
- 'stable/havana')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit(
- 'stable/havana')),
- 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
- 'org/project4': str(upstream['org/project4'].commit(
- 'stable/havana')),
-
- 'org/project5': str(upstream['org/project5'].commit(
- 'stable/havana')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit(
- 'stable/havana')),
- 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
- 'org/project4': str(upstream['org/project4'].commit(
- 'stable/havana')),
- 'org/project5': str(upstream['org/project5'].commit(
- 'stable/havana')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit(
- 'stable/havana')),
- 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
- 'org/project4': self.builds[4].parameters['ZUUL_COMMIT'],
- 'org/project5': str(upstream['org/project5'].commit(
- 'stable/havana')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- ]
-
- for number, build in enumerate(self.builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters['ZUUL_BRANCH'],
- zuul_ref=build.parameters['ZUUL_REF'],
- zuul_url=self.src_root,
- branch='stable/havana', # Old branch for upgrade
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct on old side of upgrade' %
- (project, number))
- shutil.rmtree(self.workspace_root)
-
- # Check the new side of the upgrade
- states = [
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit('master')),
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- ]
-
- for number, build in enumerate(self.builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters['ZUUL_BRANCH'],
- zuul_ref=build.parameters['ZUUL_REF'],
- zuul_url=self.src_root,
- branch='master', # New branch for upgrade
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct on old side of upgrade' %
- (project, number))
- shutil.rmtree(self.workspace_root)
-
- self.worker.hold_jobs_in_build = False
- self.worker.release()
- self.waitUntilSettled()
-
- def test_project_override(self):
- self.worker.hold_jobs_in_build = True
- projects = ['org/project1', 'org/project2', 'org/project3',
- 'org/project4', 'org/project5', 'org/project6']
-
- self.create_branch('org/project3', 'stable/havana')
- self.create_branch('org/project4', 'stable/havana')
- self.create_branch('org/project6', 'stable/havana')
- A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
- B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
- C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
- D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana',
- 'D')
- A.addApproval('CRVW', 2)
- B.addApproval('CRVW', 2)
- C.addApproval('CRVW', 2)
- D.addApproval('CRVW', 2)
- self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
- self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
-
- self.waitUntilSettled()
-
- self.assertEquals(4, len(self.builds), "Four builds are running")
-
- upstream = self.getUpstreamRepos(projects)
- states = [
- {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit('master')),
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project2': str(upstream['org/project2'].commit('master')),
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
- 'org/project3': str(upstream['org/project3'].commit('master')),
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit('master')),
- },
- {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
- 'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
- 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
- 'org/project4': str(upstream['org/project4'].commit('master')),
- 'org/project5': str(upstream['org/project5'].commit('master')),
- 'org/project6': str(upstream['org/project6'].commit(
- 'stable/havana')),
- },
- ]
-
- for number, build in enumerate(self.builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters['ZUUL_BRANCH'],
- zuul_ref=build.parameters['ZUUL_REF'],
- zuul_url=self.src_root,
- project_branches={'org/project4': 'master'},
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, number))
- shutil.rmtree(self.workspace_root)
-
- self.worker.hold_jobs_in_build = False
- self.worker.release()
- self.waitUntilSettled()
-
- def test_periodic(self):
- self.worker.hold_jobs_in_build = True
- self.create_branch('org/project', 'stable/havana')
- self.updateConfigLayout(
- 'tests/fixtures/layout-timer.yaml')
- self.sched.reconfigure(self.config)
- self.registerJobs()
-
- # The pipeline triggers every second, so we should have seen
- # several by now.
- time.sleep(5)
- self.waitUntilSettled()
-
- builds = self.builds[:]
-
- self.worker.hold_jobs_in_build = False
- # Stop queuing timer triggered jobs so that the assertions
- # below don't race against more jobs being queued.
- self.updateConfigLayout(
- 'tests/fixtures/layout-no-timer.yaml')
- self.sched.reconfigure(self.config)
- self.registerJobs()
- self.worker.release()
- self.waitUntilSettled()
-
- projects = ['org/project']
-
- self.assertEquals(2, len(builds), "Two builds are running")
-
- upstream = self.getUpstreamRepos(projects)
- states = [
- {'org/project':
- str(upstream['org/project'].commit('stable/havana')),
- },
- {'org/project':
- str(upstream['org/project'].commit('stable/havana')),
- },
- ]
-
- for number, build in enumerate(builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters.get('ZUUL_BRANCH', None),
- zuul_ref=build.parameters.get('ZUUL_REF', None),
- zuul_url=self.src_root,
- branch='stable/havana',
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, number))
-
- shutil.rmtree(self.workspace_root)
-
- self.worker.hold_jobs_in_build = False
- self.worker.release()
- self.waitUntilSettled()
-
- def test_periodic_update(self):
- # Test that the merger correctly updates its local repository
- # before running a periodic job.
-
- # Prime the merger with the current state
- A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
- self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
- self.waitUntilSettled()
-
- # Merge a different change
- B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
- B.setMerged()
-
- # Start a periodic job
- self.worker.hold_jobs_in_build = True
- self.executor.negative_function_cache_ttl = 0
- self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-timer.yaml')
- self.sched.reconfigure(self.config)
- self.registerJobs()
-
- # The pipeline triggers every second, so we should have seen
- # several by now.
- time.sleep(5)
- self.waitUntilSettled()
-
- builds = self.builds[:]
-
- self.worker.hold_jobs_in_build = False
- # Stop queuing timer triggered jobs so that the assertions
- # below don't race against more jobs being queued.
- self.config.set('zuul', 'layout_config',
- 'tests/fixtures/layout-no-timer.yaml')
- self.sched.reconfigure(self.config)
- self.registerJobs()
- self.worker.release()
- self.waitUntilSettled()
-
- projects = ['org/project']
-
- self.assertEquals(2, len(builds), "Two builds are running")
-
- upstream = self.getUpstreamRepos(projects)
- self.assertEqual(upstream['org/project'].commit('master').hexsha,
- B.patchsets[0]['revision'])
- states = [
- {'org/project':
- str(upstream['org/project'].commit('master')),
- },
- {'org/project':
- str(upstream['org/project'].commit('master')),
- },
- ]
-
- for number, build in enumerate(builds):
- self.log.debug("Build parameters: %s", build.parameters)
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters.get('ZUUL_BRANCH', None),
- zuul_ref=build.parameters.get('ZUUL_REF', None),
- zuul_url=self.git_root,
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
- state = states[number]
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, number))
-
- shutil.rmtree(self.workspace_root)
-
- self.worker.hold_jobs_in_build = False
- self.worker.release()
- self.waitUntilSettled()
-
- def test_post_checkout(self):
- self.worker.hold_jobs_in_build = True
- project = "org/project1"
-
- A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
- event = A.getRefUpdatedEvent()
- A.setMerged()
- self.fake_gerrit.addEvent(event)
- self.waitUntilSettled()
-
- build = self.builds[0]
- state = {'org/project1': build.parameters['ZUUL_COMMIT']}
-
- build.release()
- self.waitUntilSettled()
-
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=[project],
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters.get('ZUUL_BRANCH', None),
- zuul_ref=build.parameters.get('ZUUL_REF', None),
- zuul_newrev=build.parameters.get('ZUUL_NEWREV', None),
- zuul_url=self.git_root,
- )
- cloner.execute()
- work = self.getWorkspaceRepos([project])
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, 0))
- shutil.rmtree(self.workspace_root)
-
- def test_post_and_master_checkout(self):
- self.worker.hold_jobs_in_build = True
- projects = ["org/project1", "org/project2"]
-
- A = self.fake_gerrit.addFakeChange(projects[0], 'master', 'A')
- event = A.getRefUpdatedEvent()
- A.setMerged()
- self.fake_gerrit.addEvent(event)
- self.waitUntilSettled()
-
- build = self.builds[0]
- upstream = self.getUpstreamRepos(projects)
- state = {'org/project1':
- build.parameters['ZUUL_COMMIT'],
- 'org/project2':
- str(upstream['org/project2'].commit('master')),
- }
-
- build.release()
- self.waitUntilSettled()
-
- cloner = zuul.lib.cloner.Cloner(
- git_base_url=self.upstream_root,
- projects=projects,
- workspace=self.workspace_root,
- zuul_project=build.parameters.get('ZUUL_PROJECT', None),
- zuul_branch=build.parameters.get('ZUUL_BRANCH', None),
- zuul_ref=build.parameters.get('ZUUL_REF', None),
- zuul_newrev=build.parameters.get('ZUUL_NEWREV', None),
- zuul_url=self.git_root,
- )
- cloner.execute()
- work = self.getWorkspaceRepos(projects)
-
- for project in projects:
- self.assertEquals(state[project],
- str(work[project].commit('HEAD')),
- 'Project %s commit for build %s should '
- 'be correct' % (project, 0))
- shutil.rmtree(self.workspace_root)
diff --git a/tests/unit/test_cloner_cmd.py b/tests/unit/test_cloner_cmd.py
index 2d8747f..84bd243 100644
--- a/tests/unit/test_cloner_cmd.py
+++ b/tests/unit/test_cloner_cmd.py
@@ -26,7 +26,7 @@
def test_default_cache_dir_empty(self):
self.app.parse_arguments(['base', 'repo'])
- self.assertEqual(None, self.app.args.cache_dir)
+ self.assertIsNone(self.app.args.cache_dir)
def test_default_cache_dir_environ(self):
try:
diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py
new file mode 100644
index 0000000..100e4ec
--- /dev/null
+++ b/tests/unit/test_executor.py
@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# Copyright 2014 Wikimedia Foundation Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import time
+
+from tests.base import ZuulTestCase, simple_layout
+
+
+class TestExecutorRepos(ZuulTestCase):
+ tenant_config_file = 'config/single-tenant/main.yaml'
+
+ log = logging.getLogger("zuul.test.executor")
+
+ def assertRepoState(self, repo, state, project, build, number):
+ if 'branch' in state:
+ self.assertFalse(repo.head.is_detached,
+ 'Project %s commit for build %s #%s should '
+ 'not have a detached HEAD' % (
+ project, build, number))
+ self.assertEquals(repo.active_branch.name,
+ state['branch'],
+ 'Project %s commit for build %s #%s should '
+ 'be on the correct branch' % (
+ project, build, number))
+ if 'commit' in state:
+ self.assertEquals(state['commit'],
+ str(repo.commit('HEAD')),
+ 'Project %s commit for build %s #%s should '
+ 'be correct' % (
+ project, build, number))
+ ref = repo.commit('HEAD')
+ repo_messages = set(
+ [c.message.strip() for c in repo.iter_commits(ref)])
+ if 'present' in state:
+ for change in state['present']:
+ msg = '%s-1' % change.subject
+ self.assertTrue(msg in repo_messages,
+ 'Project %s for build %s #%s should '
+ 'have change %s' % (
+ project, build, number, change.subject))
+ if 'absent' in state:
+ for change in state['absent']:
+ msg = '%s-1' % change.subject
+ self.assertTrue(msg not in repo_messages,
+ 'Project %s for build %s #%s should '
+ 'not have change %s' % (
+ project, build, number, change.subject))
+
+ @simple_layout('layouts/repo-checkout-two-project.yaml')
+ def test_one_branch(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ p1 = 'review.example.com/org/project1'
+ p2 = 'review.example.com/org/project2'
+ projects = [p1, p2]
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ A.addApproval('code-review', 2)
+ B.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(B.addApproval('approved', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(2, len(self.builds), "Two builds are running")
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {p1: dict(present=[A], absent=[B], branch='master'),
+ p2: dict(commit=str(upstream[p2].commit('master')),
+ branch='master'),
+ },
+ {p1: dict(present=[A], absent=[B], branch='master'),
+ p2: dict(present=[B], absent=[A], branch='master'),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ work = build.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertRepoState(work[project], state[project],
+ project, build, number)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ @simple_layout('layouts/repo-checkout-four-project.yaml')
+ def test_multi_branch(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ p1 = 'review.example.com/org/project1'
+ p2 = 'review.example.com/org/project2'
+ p3 = 'review.example.com/org/project3'
+ p4 = 'review.example.com/org/project4'
+ projects = [p1, p2, p3, p4]
+
+ self.create_branch('org/project2', 'stable/havana')
+ self.create_branch('org/project4', 'stable/havana')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana',
+ 'B')
+ C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+ A.addApproval('code-review', 2)
+ B.addApproval('code-review', 2)
+ C.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(B.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(C.addApproval('approved', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(3, len(self.builds), "Three builds are running")
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {p1: dict(present=[A], absent=[B, C], branch='master'),
+ p2: dict(commit=str(upstream[p2].commit('master')),
+ branch='master'),
+ p3: dict(commit=str(upstream[p3].commit('master')),
+ branch='master'),
+ p4: dict(commit=str(upstream[p4].commit('master')),
+ branch='master'),
+ },
+ {p1: dict(present=[A], absent=[B, C], branch='master'),
+ p2: dict(present=[B], absent=[A, C], branch='stable/havana'),
+ p3: dict(commit=str(upstream[p3].commit('master')),
+ branch='master'),
+ p4: dict(commit=str(upstream[p4].commit('stable/havana')),
+ branch='stable/havana'),
+ },
+ {p1: dict(present=[A], absent=[B, C], branch='master'),
+ p2: dict(commit=str(upstream[p2].commit('master')),
+ branch='master'),
+ p3: dict(present=[C], absent=[A, B], branch='master'),
+ p4: dict(commit=str(upstream[p4].commit('master')),
+ branch='master'),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ work = build.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertRepoState(work[project], state[project],
+ project, build, number)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ @simple_layout('layouts/repo-checkout-six-project.yaml')
+ def test_project_override(self):
+ self.executor_server.hold_jobs_in_build = True
+
+ p1 = 'review.example.com/org/project1'
+ p2 = 'review.example.com/org/project2'
+ p3 = 'review.example.com/org/project3'
+ p4 = 'review.example.com/org/project4'
+ p5 = 'review.example.com/org/project5'
+ p6 = 'review.example.com/org/project6'
+ projects = [p1, p2, p3, p4, p5, p6]
+
+ self.create_branch('org/project3', 'stable/havana')
+ self.create_branch('org/project4', 'stable/havana')
+ self.create_branch('org/project6', 'stable/havana')
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
+ C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
+ D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana',
+ 'D')
+ A.addApproval('code-review', 2)
+ B.addApproval('code-review', 2)
+ C.addApproval('code-review', 2)
+ D.addApproval('code-review', 2)
+ self.fake_gerrit.addEvent(A.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(B.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(C.addApproval('approved', 1))
+ self.fake_gerrit.addEvent(D.addApproval('approved', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(4, len(self.builds), "Four builds are running")
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {p1: dict(present=[A], absent=[B, C, D], branch='master'),
+ p2: dict(commit=str(upstream[p2].commit('master')),
+ branch='master'),
+ p3: dict(commit=str(upstream[p3].commit('master')),
+ branch='master'),
+ p4: dict(commit=str(upstream[p4].commit('master')),
+ branch='master'),
+ p5: dict(commit=str(upstream[p5].commit('master')),
+ branch='master'),
+ p6: dict(commit=str(upstream[p6].commit('master')),
+ branch='master'),
+ },
+ {p1: dict(present=[A, B], absent=[C, D], branch='master'),
+ p2: dict(commit=str(upstream[p2].commit('master')),
+ branch='master'),
+ p3: dict(commit=str(upstream[p3].commit('master')),
+ branch='master'),
+ p4: dict(commit=str(upstream[p4].commit('master')),
+ branch='master'),
+ p5: dict(commit=str(upstream[p5].commit('master')),
+ branch='master'),
+ p6: dict(commit=str(upstream[p6].commit('master')),
+ branch='master'),
+ },
+ {p1: dict(present=[A, B], absent=[C, D], branch='master'),
+ p2: dict(present=[C], absent=[A, B, D], branch='master'),
+ p3: dict(commit=str(upstream[p3].commit('master')),
+ branch='master'),
+ p4: dict(commit=str(upstream[p4].commit('master')),
+ branch='master'),
+ p5: dict(commit=str(upstream[p5].commit('master')),
+ branch='master'),
+ p6: dict(commit=str(upstream[p6].commit('master')),
+ branch='master'),
+ },
+ {p1: dict(present=[A, B], absent=[C, D], branch='master'),
+ p2: dict(present=[C], absent=[A, B, D], branch='master'),
+ p3: dict(present=[D], absent=[A, B, C],
+ branch='stable/havana'),
+ p4: dict(commit=str(upstream[p4].commit('master')),
+ branch='master'),
+ p5: dict(commit=str(upstream[p5].commit('master')),
+ branch='master'),
+ p6: dict(commit=str(upstream[p6].commit('stable/havana')),
+ branch='stable/havana'),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ work = build.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertRepoState(work[project], state[project],
+ project, build, number)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ def test_periodic(self):
+ # This test can not use simple_layout because it must start
+ # with a configuration which does not include a
+ # timer-triggered job so that we have an opportunity to set
+ # the hold flag before the first job.
+ self.executor_server.hold_jobs_in_build = True
+ # Start timer trigger - also org/project
+ self.commitConfigUpdate('common-config',
+ 'layouts/repo-checkout-timer.yaml')
+ self.sched.reconfigure(self.config)
+
+ p1 = 'review.example.com/org/project1'
+ projects = [p1]
+ self.create_branch('org/project1', 'stable/havana')
+
+ # The pipeline triggers every second, so we should have seen
+ # several by now.
+ time.sleep(5)
+ self.waitUntilSettled()
+
+ # Stop queuing timer triggered jobs so that the assertions
+ # below don't race against more jobs being queued.
+ self.commitConfigUpdate('common-config',
+ 'layouts/repo-checkout-no-timer.yaml')
+ self.sched.reconfigure(self.config)
+
+ self.assertEquals(1, len(self.builds), "One build is running")
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {p1: dict(commit=str(upstream[p1].commit('stable/havana')),
+ branch='stable/havana'),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ work = build.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertRepoState(work[project], state[project],
+ project, build, number)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
+
+ @simple_layout('layouts/repo-checkout-post.yaml')
+ def test_post_and_master_checkout(self):
+ self.executor_server.hold_jobs_in_build = True
+ p1 = "review.example.com/org/project1"
+ p2 = "review.example.com/org/project2"
+ projects = [p1, p2]
+
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ event = A.getRefUpdatedEvent()
+ A.setMerged()
+ self.fake_gerrit.addEvent(event)
+ self.waitUntilSettled()
+
+ upstream = self.getUpstreamRepos(projects)
+ states = [
+ {p1: dict(commit=str(upstream[p1].commit('master')),
+ present=[A], branch='master'),
+ p2: dict(commit=str(upstream[p2].commit('master')),
+ absent=[A], branch='master'),
+ },
+ ]
+
+ for number, build in enumerate(self.builds):
+ self.log.debug("Build parameters: %s", build.parameters)
+ work = build.getWorkspaceRepos(projects)
+ state = states[number]
+
+ for project in projects:
+ self.assertRepoState(work[project], state[project],
+ project, build, number)
+
+ self.executor_server.hold_jobs_in_build = False
+ self.executor_server.release()
+ self.waitUntilSettled()
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 227d659..6cc010e 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -300,6 +300,9 @@
self.assertEqual('tenant-one/reporting', report_status['context'])
self.assertEqual('success', report_status['state'])
self.assertEqual(2, len(A.comments))
+ report_url = ('http://logs.example.com/reporting/%s/%s/%s/' %
+ (A.project, A.number, A.head_sha))
+ self.assertEqual(report_url, report_status['url'])
@simple_layout('layouts/merging-github.yaml', driver='github')
def test_report_pull_merge(self):
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 5f968b4..e7e53c4 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -106,7 +106,7 @@
base.auth = model.AuthContext()
py27 = model.Job('py27')
- self.assertEqual(None, py27.timeout)
+ self.assertIsNone(py27.timeout)
py27.inheritFrom(base)
self.assertEqual(30, py27.timeout)
self.assertEqual(['base-pre'],
@@ -115,7 +115,7 @@
[x.path for x in py27.run])
self.assertEqual(['base-post'],
[x.path for x in py27.post_run])
- self.assertEqual(None, py27.auth)
+ self.assertIsNone(py27.auth)
def test_job_variants(self):
# This simulates freezing a job.
@@ -433,11 +433,11 @@
})
layout.addJob(in_repo_job_with_inherit_false)
- self.assertEqual(None, in_repo_job_without_inherit.auth)
+ self.assertIsNone(in_repo_job_without_inherit.auth)
self.assertEqual(1, len(in_repo_job_with_inherit.auth.secrets))
self.assertEqual(in_repo_job_with_inherit.auth.secrets[0].name,
'pypi-credentials')
- self.assertEqual(None, in_repo_job_with_inherit_false.auth)
+ self.assertIsNone(in_repo_job_with_inherit_false.auth)
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 9cc5e60..0ac42c1 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -63,7 +63,11 @@
self.assertIsNone(self.getJobFromHistory('project-test2').node)
# TODOv3(jeblair): we may want to report stats by tenant (also?).
- self.assertReportedStat('gerrit.event.comment-added', value='1|c')
+ # Per-driver
+ self.assertReportedStat('zuul.event.gerrit.comment-added', value='1|c')
+ # Per-driver per-connection
+ self.assertReportedStat('zuul.event.gerrit.gerrit.comment-added',
+ value='1|c')
self.assertReportedStat('zuul.pipeline.gate.current_changes',
value='1|g')
self.assertReportedStat('zuul.pipeline.gate.job.project-merge.SUCCESS',
@@ -3545,9 +3549,9 @@
self.assertEqual([], running_item['failing_reasons'])
self.assertEqual([], running_item['items_behind'])
self.assertEqual('https://hostname/1', running_item['url'])
- self.assertEqual(None, running_item['item_ahead'])
+ self.assertIsNone(running_item['item_ahead'])
self.assertEqual('org/project', running_item['project'])
- self.assertEqual(None, running_item['remaining_time'])
+ self.assertIsNone(running_item['remaining_time'])
self.assertEqual(True, running_item['active'])
self.assertEqual('1,1', running_item['id'])
@@ -3562,7 +3566,7 @@
self.assertEqual(7, len(job['worker']))
self.assertEqual(False, job['canceled'])
self.assertEqual(True, job['voting'])
- self.assertEqual(None, job['result'])
+ self.assertIsNone(job['result'])
self.assertEqual('gate', job['pipeline'])
break
diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py
index 49624d7..90ab39c 100644
--- a/zuul/connection/__init__.py
+++ b/zuul/connection/__init__.py
@@ -14,6 +14,7 @@
import abc
+import extras
import six
@@ -43,6 +44,26 @@
self.driver = driver
self.connection_name = connection_name
self.connection_config = connection_config
+ self.statsd = extras.try_import('statsd.statsd')
+
+ def logEvent(self, event):
+ self.log.debug(
+ 'Scheduling {driver} event from {connection}: {event}'.format(
+ driver=self.driver.name,
+ connection=self.connection_name,
+ event=event.type))
+ try:
+ if self.statsd:
+ self.statsd.incr(
+ 'zuul.event.{driver}.{event}'.format(
+ driver=self.driver.name, event=event.type))
+ self.statsd.incr(
+ 'zuul.event.{driver}.{connection}.{event}'.format(
+ driver=self.driver.name,
+ connection=self.connection_name,
+ event=event.type))
+ except:
+ self.log.exception("Exception reporting event stats")
def onLoad(self):
pass
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 06962e5..a1d97e7 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -143,6 +143,7 @@
self.connection._getChange(event.change_number,
event.patch_number,
refresh=True)
+ self.connection.logEvent(event)
self.connection.sched.addEvent(event)
def run(self):
@@ -625,7 +626,7 @@
if val is True:
cmd += ' --%s' % key
else:
- cmd += ' --%s %s' % (key, val)
+ cmd += ' --label %s=%s' % (key, val)
cmd += ' %s' % change
out, err = self._ssh(cmd)
return err
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index 02c795e..6a3c09e 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -119,7 +119,7 @@
if event:
event.project_hostname = self.connection.canonical_hostname
- self.log.debug('Scheduling github event: {0}'.format(event.type))
+ self.connection.logEvent(event)
self.connection.sched.addEvent(event)
def _event_push(self, body):
diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py
index 68c6af0..fc3b64d 100644
--- a/zuul/driver/github/githubreporter.py
+++ b/zuul/driver/github/githubreporter.py
@@ -70,12 +70,14 @@
sha = item.change.patchset
context = '%s/%s' % (pipeline.layout.tenant.name, pipeline.name)
state = self._commit_status
- url = ''
- if self.connection.sched.config.has_option('zuul', 'status_url'):
- base = self.connection.sched.config.get('zuul', 'status_url')
- url = '%s/#%s,%s' % (base,
- item.change.number,
- item.change.patchset)
+
+ url_pattern = self.config.get('status-url')
+ if not url_pattern:
+ sched_config = self.connection.sched.config
+ if sched_config.has_option('zuul', 'status_url'):
+ url_pattern = sched_config.get('zuul', 'status_url')
+ url = item.formatUrlPattern(url_pattern) if url_pattern else ''
+
description = ''
if pipeline.description:
description = pipeline.description
@@ -157,6 +159,7 @@
def getSchema():
github_reporter = v.Schema({
'status': v.Any('pending', 'success', 'failure'),
+ 'status-url': str,
'comment': bool,
'merge': bool,
'label': scalar_or_list(str),
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index 46d538a..349abe8 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -39,13 +39,16 @@
return
with self.connection.engine.begin() as conn:
+ change = getattr(item.change, 'number', '')
+ patchset = getattr(item.change, 'patchset', '')
+ refspec = getattr(item.change, 'refspec', item.change.newrev)
buildset_ins = self.connection.zuul_buildset_table.insert().values(
zuul_ref=item.current_build_set.ref,
pipeline=item.pipeline.name,
project=item.change.project.name,
- change=item.change.number,
- patchset=item.change.patchset,
- ref=item.change.refspec,
+ change=change,
+ patchset=patchset,
+ ref=refspec,
score=self.result_score,
message=self._formatItemReport(
pipeline, item, with_jobs=False),
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 907cce5..cf8d973 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -261,6 +261,10 @@
params['timeout'] = job.timeout
params['items'] = merger_items
params['projects'] = []
+ if hasattr(item.change, 'branch'):
+ params['branch'] = item.change.branch
+ else:
+ params['branch'] = None
params['override_branch'] = job.override_branch
params['repo_state'] = item.current_build_set.repo_state
@@ -300,6 +304,7 @@
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
+ canonical_name=project.canonical_name,
override_branch=override_branch,
default_branch=project_default_branch)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index f88af53..421281e 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -27,7 +27,6 @@
from zuul.lib.yamlutil import yaml
import gear
-import git
from six.moves import shlex_quote
import zuul.merger.merger
@@ -435,9 +434,14 @@
self.job_workers = {}
- def _getMerger(self, root):
+ def _getMerger(self, root, logger=None):
+ if root != self.merge_root:
+ cache_root = self.merge_root
+ else:
+ cache_root = None
return zuul.merger.merger.Merger(root, self.connections,
- self.merge_email, self.merge_name)
+ self.merge_email, self.merge_name,
+ cache_root, logger)
def start(self):
self._running = True
@@ -653,19 +657,27 @@
if ret is None:
result['commit'] = result['files'] = result['repo_state'] = None
else:
- result['commit'], result['files'], result['repo_state'] = ret
+ (result['commit'], result['files'], result['repo_state'],
+ recent) = ret
job.sendWorkComplete(json.dumps(result))
-class AnsibleJob(object):
- log = logging.getLogger("zuul.AnsibleJob")
+class AnsibleJobLogAdapter(logging.LoggerAdapter):
+ def process(self, msg, kwargs):
+ msg, kwargs = super(AnsibleJobLogAdapter, self).process(msg, kwargs)
+ msg = '[build: %s] %s' % (kwargs['extra']['job'], msg)
+ return msg, kwargs
+
+class AnsibleJob(object):
RESULT_NORMAL = 1
RESULT_TIMED_OUT = 2
RESULT_UNREACHABLE = 3
RESULT_ABORTED = 4
def __init__(self, executor_server, job):
+ logger = logging.getLogger("zuul.AnsibleJob")
+ self.log = AnsibleJobLogAdapter(logger, {'job': job.unique})
self.executor_server = executor_server
self.job = job
self.jobdir = None
@@ -739,36 +751,37 @@
task.wait()
self.log.debug("Job %s: git updates complete" % (self.job.unique,))
- repos = []
+ merger = self.executor_server._getMerger(self.jobdir.src_root,
+ self.log)
+ repos = {}
for project in args['projects']:
self.log.debug("Cloning %s/%s" % (project['connection'],
project['name'],))
- source = self.executor_server.connections.getSource(
- project['connection'])
- project_object = source.getProject(project['name'])
- url = source.getGitUrl(project_object)
- repo = git.Repo.clone_from(
- os.path.join(self.executor_server.merge_root,
- source.canonical_hostname,
- project['name']),
- os.path.join(self.jobdir.src_root,
- source.canonical_hostname,
- project['name']))
-
- repo.remotes.origin.config_writer.set('url', url)
- repos.append(repo)
+ repo = merger.getRepo(project['connection'],
+ project['name'])
+ repos[project['canonical_name']] = repo
merge_items = [i for i in args['items'] if i.get('refspec')]
if merge_items:
- if not self.doMergeChanges(merge_items, args['repo_state']):
+ if not self.doMergeChanges(merger, merge_items,
+ args['repo_state']):
# There was a merge conflict and we have already sent
# a work complete result, don't run any jobs
return
+ for project in args['projects']:
+ repo = repos[project['canonical_name']]
+ self.checkoutBranch(repo,
+ project['name'],
+ args['branch'],
+ args['override_branch'],
+ project['override_branch'],
+ project['default_branch'])
+
# Delete the origin remote from each repo we set up since
# it will not be valid within the jobs.
- for repo in repos:
- repo.delete_remote(repo.remotes.origin)
+ for repo in repos.values():
+ repo.deleteRemote('origin')
# is the playbook in a repo that we have already prepared?
trusted, untrusted = self.preparePlaybookRepos(args)
@@ -806,16 +819,43 @@
result = dict(result=result)
self.job.sendWorkComplete(json.dumps(result))
- def doMergeChanges(self, items, repo_state):
- # Get a merger in order to update the repos involved in this job.
- merger = self.executor_server._getMerger(self.jobdir.src_root)
+ def doMergeChanges(self, merger, items, repo_state):
ret = merger.mergeChanges(items, repo_state=repo_state)
if not ret: # merge conflict
result = dict(result='MERGER_FAILURE')
self.job.sendWorkComplete(json.dumps(result))
return False
+ recent = ret[3]
+ for key, commit in recent.items():
+ (connection, project, branch) = key
+ repo = merger.getRepo(connection, project)
+ repo.setRef('refs/heads/' + branch, commit)
return True
+ def checkoutBranch(self, repo, project_name, zuul_branch,
+ job_branch, project_override_branch,
+ project_default_branch):
+ branches = repo.getBranches()
+ if project_override_branch in branches:
+ self.log.info("Checking out %s project override branch %s",
+ project_name, project_override_branch)
+ repo.checkoutLocalBranch(project_override_branch)
+ elif job_branch in branches:
+ self.log.info("Checking out %s job branch %s",
+ project_name, job_branch)
+ repo.checkoutLocalBranch(job_branch)
+ elif zuul_branch and zuul_branch in branches:
+ self.log.info("Checking out %s zuul branch %s",
+ project_name, zuul_branch)
+ repo.checkoutLocalBranch(zuul_branch)
+ elif project_default_branch in branches:
+ self.log.info("Checking out %s project default branch %s",
+ project_name, project_default_branch)
+ repo.checkoutLocalBranch(project_default_branch)
+ else:
+ raise Exception("Project %s does not have the default branch %s" %
+ (project_name, project_default_branch))
+
def runPlaybooks(self, args):
result = None
@@ -971,7 +1011,8 @@
# the stack of changes we are testing, so check out the branch
# tip into a dedicated space.
- merger = self.executor_server._getMerger(jobdir_playbook.root)
+ merger = self.executor_server._getMerger(jobdir_playbook.root,
+ self.log)
merger.checkoutBranch(playbook['connection'], project.name,
playbook['branch'])
@@ -1055,7 +1096,8 @@
# in the dependency chain for the change (in which case,
# there is no existing untrusted checkout of it). Check
# out the branch tip into a dedicated space.
- merger = self.executor_server._getMerger(trusted_root)
+ merger = self.executor_server._getMerger(trusted_root,
+ self.log)
merger.checkoutBranch(role['connection'], project.name,
'master')
orig_repo_path = os.path.join(trusted_root,
diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py
index 6aa51a6..de072b6 100644
--- a/zuul/lib/log_streamer.py
+++ b/zuul/lib/log_streamer.py
@@ -47,8 +47,39 @@
the (class/method/attribute) names were changed to protect the innocent.
'''
+ MAX_REQUEST_LEN = 1024
+ REQUEST_TIMEOUT = 10
+
+ def get_command(self):
+ poll = select.poll()
+ bitmask = (select.POLLIN | select.POLLERR |
+ select.POLLHUP | select.POLLNVAL)
+ poll.register(self.request, bitmask)
+ buffer = b''
+ ret = None
+ start = time.time()
+ while True:
+ elapsed = time.time() - start
+ timeout = max(self.REQUEST_TIMEOUT - elapsed, 0)
+ if not timeout:
+ raise Exception("Timeout while waiting for input")
+ for fd, event in poll.poll(timeout):
+ if event & select.POLLIN:
+ buffer += self.request.recv(self.MAX_REQUEST_LEN)
+ else:
+ raise Exception("Received error event")
+ if len(buffer) >= self.MAX_REQUEST_LEN:
+ raise Exception("Request too long")
+ try:
+ ret = buffer.decode('utf-8')
+ x = ret.find('\n')
+ if x > 0:
+ return ret[:x]
+ except UnicodeDecodeError:
+ pass
+
def handle(self):
- build_uuid = self.request.recv(1024).decode("utf-8")
+ build_uuid = self.get_command()
build_uuid = build_uuid.rstrip()
# validate build ID
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index ee83fa0..6cfd904 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -42,13 +42,17 @@
class Repo(object):
- log = logging.getLogger("zuul.Repo")
-
- def __init__(self, remote, local, email, username):
+ def __init__(self, remote, local, email, username,
+ cache_path=None, logger=None):
+ if logger is None:
+ self.log = logging.getLogger("zuul.Repo")
+ else:
+ self.log = logger
self.remote_url = remote
self.local_path = local
self.email = email
self.username = username
+ self.cache_path = cache_path
self._initialized = False
try:
self._ensure_cloned()
@@ -60,17 +64,32 @@
if self._initialized and repo_is_cloned:
return
# If the repo does not exist, clone the repo.
+ rewrite_url = False
if not repo_is_cloned:
self.log.debug("Cloning from %s to %s" % (self.remote_url,
self.local_path))
- git.Repo.clone_from(self.remote_url, self.local_path)
+ if self.cache_path:
+ git.Repo.clone_from(self.cache_path, self.local_path)
+ rewrite_url = True
+ else:
+ git.Repo.clone_from(self.remote_url, self.local_path)
repo = git.Repo(self.local_path)
+ # Create local branches corresponding to all the remote branches
+ if not repo_is_cloned:
+ origin = repo.remotes.origin
+ for ref in origin.refs:
+ if ref.remote_head == 'HEAD':
+ continue
+ repo.create_head(ref.remote_head, ref, force=True)
with repo.config_writer() as config_writer:
if self.email:
config_writer.set_value('user', 'email', self.email)
if self.username:
config_writer.set_value('user', 'name', self.username)
config_writer.write()
+ if rewrite_url:
+ with repo.remotes.origin.config_writer as config_writer:
+ config_writer.set('url', self.remote_url)
self._initialized = True
def isInitialized(self):
@@ -118,6 +137,10 @@
origin = repo.remotes.origin
return branch in origin.refs
+ def getBranches(self):
+ repo = self.createRepoObject()
+ return [x.name for x in repo.heads]
+
def getCommitFromRef(self, refname):
repo = self.createRepoObject()
if refname not in repo.refs:
@@ -129,6 +152,14 @@
repo = self.createRepoObject()
return repo.refs
+ def setRef(self, path, hexsha, repo=None):
+ if repo is None:
+ repo = self.createRepoObject()
+ binsha = gitdb.util.to_bin_sha(hexsha)
+ obj = git.objects.Object.new_from_sha(repo, binsha)
+ self.log.debug("Create reference %s", path)
+ git.refs.Reference.create(repo, path, obj, force=True)
+
def setRefs(self, refs):
repo = self.createRepoObject()
current_refs = {}
@@ -136,10 +167,7 @@
current_refs[ref.path] = ref
unseen = set(current_refs.keys())
for path, hexsha in refs.items():
- binsha = gitdb.util.to_bin_sha(hexsha)
- obj = git.objects.Object.new_from_sha(repo, binsha)
- self.log.debug("Create reference %s", path)
- git.refs.Reference.create(repo, path, obj, force=True)
+ self.setRef(path, hexsha, repo)
unseen.discard(path)
for path in unseen:
self.log.debug("Delete reference %s", path)
@@ -152,6 +180,13 @@
reset_repo_to_head(repo)
return repo.head.commit
+ def checkoutLocalBranch(self, branch):
+ repo = self.createRepoObject()
+ # Perform a hard reset before checking out so that we clean up
+ # anything that might be left over from a merge.
+ reset_repo_to_head(repo)
+ repo.heads[branch].checkout()
+
def cherryPick(self, ref):
repo = self.createRepoObject()
self.log.debug("Cherry-picking %s" % ref)
@@ -225,11 +260,19 @@
ret[fn] = None
return ret
+ def deleteRemote(self, remote):
+ repo = self.createRepoObject()
+ repo.delete_remote(repo.remotes[remote])
+
class Merger(object):
- log = logging.getLogger("zuul.Merger")
-
- def __init__(self, working_root, connections, email, username):
+ def __init__(self, working_root, connections, email, username,
+ cache_root=None, logger=None):
+ self.logger = logger
+ if logger is None:
+ self.log = logging.getLogger("zuul.Merger")
+ else:
+ self.log = logger
self.repos = {}
self.working_root = working_root
if not os.path.exists(working_root):
@@ -237,6 +280,7 @@
self.connections = connections
self.email = email
self.username = username
+ self.cache_root = cache_root
def _get_ssh_cmd(self, connection_name):
sshkey = self.connections.connections.get(connection_name).\
@@ -259,7 +303,13 @@
key = '/'.join([hostname, project_name])
try:
path = os.path.join(self.working_root, hostname, project_name)
- repo = Repo(url, path, self.email, self.username)
+ if self.cache_root:
+ cache_path = os.path.join(self.cache_root, hostname,
+ project_name)
+ else:
+ cache_path = None
+ repo = Repo(url, path, self.email, self.username, cache_path,
+ self.logger)
self.repos[key] = repo
except Exception:
@@ -296,28 +346,25 @@
connection_name, project_name)
def checkoutBranch(self, connection_name, project_name, branch):
+ self.log.info("Checking out %s/%s branch %s",
+ connection_name, project_name, branch)
repo = self.getRepo(connection_name, project_name)
- if repo.hasBranch(branch):
- self.log.info("Checking out branch %s of %s/%s" %
- (branch, connection_name, project_name))
- head = repo.getBranchHead(branch)
- repo.checkout(head)
- else:
- raise Exception("Project %s/%s does not have branch %s" %
- (connection_name, project_name, branch))
+ repo.checkoutLocalBranch(branch)
def _saveRepoState(self, connection_name, project_name, repo,
- repo_state):
+ repo_state, recent):
projects = repo_state.setdefault(connection_name, {})
project = projects.setdefault(project_name, {})
- if project:
- # We already have a state for this project.
- return
for ref in repo.getRefs():
- if ref.path.startswith('refs/zuul'):
+ if ref.path.startswith('refs/zuul/'):
continue
- if ref.path.startswith('refs/remotes'):
+ if ref.path.startswith('refs/remotes/'):
continue
+ if ref.path.startswith('refs/heads/'):
+ branch = ref.path[len('refs/heads/'):]
+ key = (connection_name, project_name, branch)
+ if key not in recent:
+ recent[key] = ref.object
project[ref.path] = ref.object.hexsha
def _restoreRepoState(self, connection_name, project_name, repo,
@@ -386,7 +433,7 @@
# Save the repo state so that later mergers can repeat
# this process.
self._saveRepoState(item['connection'], item['project'], repo,
- repo_state)
+ repo_state, recent)
else:
self.log.debug("Found base commit %s for %s" % (base, key,))
# Merge the change
@@ -439,7 +486,10 @@
project=item['project'],
branch=item['branch'],
files=repo_files))
- return commit.hexsha, read_files, repo_state
+ ret_recent = {}
+ for k, v in recent.items():
+ ret_recent[k] = v.hexsha
+ return commit.hexsha, read_files, repo_state, ret_recent
def getFiles(self, connection_name, project_name, branch, files):
repo = self.getRepo(connection_name, project_name)
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index 15f1a41..1a32f96 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -110,7 +110,8 @@
if ret is None:
result['commit'] = result['files'] = result['repo_state'] = None
else:
- result['commit'], result['files'], result['repo_state'] = ret
+ (result['commit'], result['files'], result['repo_state'],
+ recent) = ret
job.sendWorkComplete(json.dumps(result))
def cat(self, job):
diff --git a/zuul/model.py b/zuul/model.py
index b468bbf..6ad34ff 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1588,8 +1588,8 @@
# secrets, etc.
safe_change = self.change.getSafeAttributes()
safe_pipeline = self.pipeline.getSafeAttributes()
- safe_job = job.getSafeAttributes()
- safe_build = build.getSafeAttributes()
+ safe_job = job.getSafeAttributes() if job else {}
+ safe_build = build.getSafeAttributes() if build else {}
try:
url = url_pattern.format(change=safe_change,
pipeline=safe_pipeline,
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 40d5eb7..61f1e5f 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -258,11 +258,6 @@
def addEvent(self, event):
self.log.debug("Adding trigger event: %s" % event)
- try:
- if self.statsd:
- self.statsd.incr('gerrit.event.%s' % event.type)
- except:
- self.log.exception("Exception reporting event stats")
self.trigger_event_queue.put(event)
self.wake_event.set()
self.log.debug("Done adding trigger event: %s" % event)