Merge "Report correctly when dequeuing dependent changes"
diff --git a/doc/source/client.rst b/doc/source/client.rst
index 1a843e5..5fe2252 100644
--- a/doc/source/client.rst
+++ b/doc/source/client.rst
@@ -31,3 +31,21 @@
zuul enqueue --trigger gerrit --pipeline check --project example_project --change 12345,1
Note that the format of change id is <number>,<patchset>.
+
+Promote
+^^^^^^^
+.. program-output:: zuul promote --help
+
+Example::
+
+ zuul promote --pipeline check --changes 12345,1 13336,3
+
+Note that the format of changes id is <number>,<patchset>.
+
+Show
+^^^^
+.. program-output:: zuul show --help
+
+Example::
+
+ zuul show running-jobs
diff --git a/doc/source/cloner.rst b/doc/source/cloner.rst
new file mode 100644
index 0000000..bb33f82
--- /dev/null
+++ b/doc/source/cloner.rst
@@ -0,0 +1,77 @@
+:title: Zuul Cloner
+
+Zuul Cloner
+===========
+
+Zuul includes a simple command line client that may be used to clone
+repositories with Zuul references applied.
+
+Configuration
+-------------
+
+Clone map
+'''''''''
+
+By default, Zuul cloner will clone the project under ``basepath`` which
+would create sub directories whenever a project name contains slashes. Since
+you might want to finely tweak the final destination, a clone map lets you
+change the destination on a per project basis. The configuration is done using
+a YAML file passed with ``-m``.
+
+With a project hierarchy such as::
+
+ project
+ thirdparty/plugins/plugin1
+
+You might want to get ``project`` straight in the base path, the clone map would be::
+
+ clonemap:
+ - name: 'project'
+ dest: '.'
+
+Then to strip out ``thirdparty`` such that the plugins land under the
+``/plugins`` directory of the basepath, you can use regex and capturing
+groups::
+
+ clonemap:
+ - name: 'project'
+ dest: '.'
+ - name: 'thirdparty/(plugins/.*)'
+ dest: '\1'
+
+The resulting workspace will contains::
+
+ project -> ./
+ thirdparty/plugins/plugin1 -> ./plugins/plugin1
+
+
+Zuul parameters
+'''''''''''''''
+
+The Zuul cloner reuses Zuul parameters such as ZUUL_BRANCH, ZUUL_REF or
+ZUUL_PROJECT. It will attempt to load them from the environment variables or
+you can pass them as parameters (in which case it will override the
+environment variable if it is set). The matching command line parameters use
+the ``zuul`` prefix hence ZUUL_REF can be passed to the cloner using
+``--zuul-ref``.
+
+Usage
+-----
+The general options that apply are:
+
+.. program-output:: zuul-cloner --help
+
+Clone order
+-----------
+
+When cloning repositories, the destination folder should not exist or
+``git clone`` will complain. This happens whenever cloning a sub project
+before its parent project. For example::
+
+ zuul-cloner project/plugins/plugin1 project
+
+Will create the directory ``project`` when cloning the plugin. The
+cloner processes the clones in the order supplied, so you should swap the
+projects::
+
+ zuul-cloner project project/plugins/plugin1
diff --git a/doc/source/index.rst b/doc/source/index.rst
index fcc0d45..abe8089 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -20,6 +20,7 @@
reporters
zuul
client
+ cloner
statsd
Indices and tables
diff --git a/etc/clonemap.yaml-sample b/etc/clonemap.yaml-sample
new file mode 100644
index 0000000..9695d9d
--- /dev/null
+++ b/etc/clonemap.yaml-sample
@@ -0,0 +1,16 @@
+# vim: ft=yaml
+#
+# Example clone map for Zuul cloner
+#
+# By default it would clone projects under the directory specified by its
+# option --basepath, but you can override this behavior by definining per
+# project destinations.
+clonemap:
+
+ # Clone project 'mediawiki/core' directly in {basepath}
+ - name: 'mediawiki/core'
+ dest: '.'
+
+ # Clone projects below mediawiki/extensions to {basepath}/extensions/
+ - name: 'mediawiki/extensions/(.*)'
+ dest: 'extensions/\1'
diff --git a/requirements.txt b/requirements.txt
index 92d8e7c..eabcef3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,5 +17,5 @@
python-swiftclient>=1.6
python-keystoneclient>=0.4.2
PrettyTable>=0.6,<0.8
-babel
+babel>=1.0
six>=1.6.0
diff --git a/setup.cfg b/setup.cfg
index 21b1199..a4deb2f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -24,6 +24,7 @@
zuul-server = zuul.cmd.server:main
zuul-merger = zuul.cmd.merger:main
zuul = zuul.cmd.client:main
+ zuul-cloner = zuul.cmd.cloner:main
[build_sphinx]
source-dir = doc/source
diff --git a/tests/fixtures/clonemap.yaml b/tests/fixtures/clonemap.yaml
new file mode 100644
index 0000000..0f9e084
--- /dev/null
+++ b/tests/fixtures/clonemap.yaml
@@ -0,0 +1,5 @@
+clonemap:
+ - name: 'mediawiki/core'
+ dest: '.'
+ - name: 'mediawiki/extensions/(.*)'
+ dest: '\1'
diff --git a/tests/fixtures/layout-gating.yaml b/tests/fixtures/layout-gating.yaml
new file mode 100644
index 0000000..a544a80
--- /dev/null
+++ b/tests/fixtures/layout-gating.yaml
@@ -0,0 +1,29 @@
+pipelines:
+ - name: gate
+ manager: DependentPipelineManager
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ start:
+ gerrit:
+ verified: 0
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+
+projects:
+
+ - name: org/project1
+ gate:
+ - project1-project2-integration
+
+ - name: org/project2
+ gate:
+ - project1-project2-integration
diff --git a/tests/test_clonemapper.py b/tests/test_clonemapper.py
new file mode 100644
index 0000000..b7814f8
--- /dev/null
+++ b/tests/test_clonemapper.py
@@ -0,0 +1,84 @@
+# Copyright 2014 Antoine "hashar" Musso
+# Copyright 2014 Wikimedia Foundation Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import testtools
+from zuul.lib.clonemapper import CloneMapper
+
+logging.basicConfig(level=logging.DEBUG,
+ format='%(asctime)s %(name)-17s '
+ '%(levelname)-8s %(message)s')
+
+
+class TestCloneMapper(testtools.TestCase):
+
+ def test_empty_mapper(self):
+ """Given an empty map, the slashes in project names are directory
+ separators"""
+ cmap = CloneMapper(
+ {},
+ [
+ 'project1',
+ 'plugins/plugin1'
+ ])
+
+ self.assertEqual(
+ {'project1': '/basepath/project1',
+ 'plugins/plugin1': '/basepath/plugins/plugin1'},
+ cmap.expand('/basepath')
+ )
+
+ def test_map_to_a_dot_dir(self):
+ """Verify we normalize path, hence '.' refers to the basepath"""
+ cmap = CloneMapper(
+ [{'name': 'mediawiki/core', 'dest': '.'}],
+ ['mediawiki/core'])
+ self.assertEqual(
+ {'mediawiki/core': '/basepath'},
+ cmap.expand('/basepath'))
+
+ def test_map_using_regex(self):
+ """One can use regex in maps and use \\1 to forge the directory"""
+ cmap = CloneMapper(
+ [{'name': 'plugins/(.*)', 'dest': 'project/plugins/\\1'}],
+ ['plugins/PluginFirst'])
+ self.assertEqual(
+ {'plugins/PluginFirst': '/basepath/project/plugins/PluginFirst'},
+ cmap.expand('/basepath'))
+
+ def test_map_discarding_regex_group(self):
+ cmap = CloneMapper(
+ [{'name': 'plugins/(.*)', 'dest': 'project/'}],
+ ['plugins/Plugin_1'])
+ self.assertEqual(
+ {'plugins/Plugin_1': '/basepath/project'},
+ cmap.expand('/basepath'))
+
+ def test_cant_dupe_destinations(self):
+ """We cant clone multiple projects in the same directory"""
+ cmap = CloneMapper(
+ [{'name': 'plugins/(.*)', 'dest': 'catchall/'}],
+ ['plugins/plugin1', 'plugins/plugin2']
+ )
+ self.assertRaises(Exception, cmap.expand, '/basepath')
+
+ def test_map_with_dot_and_regex(self):
+ """Combining relative path and regex"""
+ cmap = CloneMapper(
+ [{'name': 'plugins/(.*)', 'dest': './\\1'}],
+ ['plugins/PluginInBasePath'])
+ self.assertEqual(
+ {'plugins/PluginInBasePath': '/basepath/PluginInBasePath'},
+ cmap.expand('/basepath'))
diff --git a/tests/test_cloner.py b/tests/test_cloner.py
new file mode 100644
index 0000000..bb9d91f
--- /dev/null
+++ b/tests/test_cloner.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# Copyright 2014 Wikimedia Foundation Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+import shutil
+
+import git
+
+import zuul.lib.cloner
+
+from tests.base import ZuulTestCase
+from tests.base import FIXTURE_DIR
+
+logging.basicConfig(level=logging.DEBUG,
+ format='%(asctime)s %(name)-32s '
+ '%(levelname)-8s %(message)s')
+
+
+class TestCloner(ZuulTestCase):
+
+ log = logging.getLogger("zuul.test.cloner")
+ workspace_root = None
+
+ def setUp(self):
+ super(TestCloner, self).setUp()
+ self.workspace_root = os.path.join(self.test_root, 'workspace')
+
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-gating.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
+
+ def test_cloner(self):
+ self.worker.hold_jobs_in_build = True
+
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+
+ A.addPatchset(['project_one.txt'])
+ B.addPatchset(['project_two.txt'])
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+
+ A.addApproval('CRVW', 2)
+ B.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
+
+ self.waitUntilSettled()
+
+ self.assertEquals(2, len(self.builds), "Two builds are running")
+
+ a_zuul_ref = b_zuul_ref = None
+ for build in self.builds:
+ self.log.debug("Build parameters: %s", build.parameters)
+ if build.parameters['ZUUL_CHANGE'] == '1':
+ a_zuul_ref = build.parameters['ZUUL_REF']
+ a_zuul_commit = build.parameters['ZUUL_COMMIT']
+ if build.parameters['ZUUL_CHANGE'] == '2':
+ b_zuul_ref = build.parameters['ZUUL_REF']
+ b_zuul_commit = build.parameters['ZUUL_COMMIT']
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()
+
+ # Repos setup, now test the cloner
+ for zuul_ref in [a_zuul_ref, b_zuul_ref]:
+ cloner = zuul.lib.cloner.Cloner(
+ git_base_url=self.upstream_root,
+ projects=['org/project1', 'org/project2'],
+ workspace=self.workspace_root,
+ zuul_branch='master',
+ zuul_ref=zuul_ref,
+ zuul_url=self.git_root,
+ branch='master',
+ clone_map_file=os.path.join(FIXTURE_DIR, 'clonemap.yaml')
+ )
+ cloner.execute()
+ work_repo1 = git.Repo(os.path.join(self.workspace_root,
+ 'org/project1'))
+ self.assertEquals(a_zuul_commit, str(work_repo1.commit('HEAD')))
+
+ work_repo2 = git.Repo(os.path.join(self.workspace_root,
+ 'org/project2'))
+ self.assertEquals(b_zuul_commit, str(work_repo2.commit('HEAD')))
+
+ shutil.rmtree(self.workspace_root)
diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py
new file mode 100755
index 0000000..1310c16
--- /dev/null
+++ b/zuul/cmd/cloner.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Antoine "hashar" Musso
+# Copyright 2014 Wikimedia Foundation Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+import os
+import sys
+
+import zuul.cmd
+import zuul.lib.cloner
+
+ZUUL_ENV_SUFFIXES = (
+ 'branch',
+ 'change',
+ 'patchset',
+ 'pipeline',
+ 'project',
+ 'ref',
+ 'url',
+)
+
+
+class Cloner(zuul.cmd.ZuulApp):
+ log = logging.getLogger("zuul.Cloner")
+
+ def parse_arguments(self):
+ """Parse command line arguments and returns argparse structure"""
+ parser = argparse.ArgumentParser(
+ description='Zuul Project Gating System Cloner.')
+ parser.add_argument('-m', '--map', dest='clone_map_file',
+ help='specifiy clone map file')
+ parser.add_argument('--workspace', dest='workspace',
+ default=os.getcwd(),
+ help='where to clone repositories too')
+ parser.add_argument('-v', '--verbose', dest='verbose',
+ action='store_true',
+ help='verbose output')
+ parser.add_argument('--color', dest='color', action='store_true',
+ help='use color output')
+ parser.add_argument('--version', dest='version', action='version',
+ version=self._get_version(),
+ help='show zuul version')
+ parser.add_argument('git_base_url',
+ help='reference repo to clone from')
+ parser.add_argument('projects', nargs='+',
+ help='list of Gerrit projects to clone')
+
+ project_env = parser.add_argument_group(
+ 'project tuning'
+ )
+ project_env.add_argument(
+ '--branch',
+ help=('branch to checkout instead of Zuul selected branch, '
+ 'for example to specify an alternate branch to test '
+ 'client library compatibility.')
+ )
+
+ zuul_env = parser.add_argument_group(
+ 'zuul environnement',
+ 'Let you override $ZUUL_* environnement variables.'
+ )
+ for zuul_suffix in ZUUL_ENV_SUFFIXES:
+ env_name = 'ZUUL_%s' % zuul_suffix.upper()
+ zuul_env.add_argument(
+ '--zuul-%s' % zuul_suffix, metavar='$' + env_name,
+ default=os.environ.get(env_name)
+ )
+
+ args = parser.parse_args()
+
+ # Validate ZUUL_* arguments
+ zuul_missing = [zuul_opt for zuul_opt, val in vars(args).items()
+ if zuul_opt.startswith('zuul') and val is None]
+ if zuul_missing:
+ parser.error(("Some Zuul parameters are not properly set:\n\t%s\n"
+ "Define them either via environment variables or "
+ "using options above." %
+ "\n\t".join(sorted(zuul_missing))))
+ self.args = args
+
+ def setup_logging(self, color=False, verbose=False):
+ """Cloner logging does not rely on conf file"""
+ if verbose:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+
+ if color:
+ # Color codes http://www.tldp.org/HOWTO/Bash-Prompt-HOWTO/x329.html
+ logging.addLevelName( # cyan
+ logging.DEBUG, "\033[36m%s\033[0m" %
+ logging.getLevelName(logging.DEBUG))
+ logging.addLevelName( # green
+ logging.INFO, "\033[32m%s\033[0m" %
+ logging.getLevelName(logging.INFO))
+ logging.addLevelName( # yellow
+ logging.WARNING, "\033[33m%s\033[0m" %
+ logging.getLevelName(logging.WARNING))
+ logging.addLevelName( # red
+ logging.ERROR, "\033[31m%s\033[0m" %
+ logging.getLevelName(logging.ERROR))
+ logging.addLevelName( # red background
+ logging.CRITICAL, "\033[41m%s\033[0m" %
+ logging.getLevelName(logging.CRITICAL))
+
+ def main(self):
+ self.parse_arguments()
+ self.setup_logging(color=self.args.color, verbose=self.args.verbose)
+ cloner = zuul.lib.cloner.Cloner(
+ git_base_url=self.args.git_base_url,
+ projects=self.args.projects,
+ workspace=self.args.workspace,
+ zuul_branch=self.args.zuul_branch,
+ zuul_ref=self.args.zuul_ref,
+ zuul_url=self.args.zuul_url,
+ branch=self.args.branch,
+ clone_map_file=self.args.clone_map_file
+ )
+ cloner.execute()
+
+
+def main():
+ cloner = Cloner()
+ cloner.main()
+
+
+if __name__ == "__main__":
+ sys.path.insert(0, '.')
+ main()
diff --git a/zuul/lib/clonemapper.py b/zuul/lib/clonemapper.py
new file mode 100644
index 0000000..ae558cd
--- /dev/null
+++ b/zuul/lib/clonemapper.py
@@ -0,0 +1,78 @@
+# Copyright 2014 Antoine "hashar" Musso
+# Copyright 2014 Wikimedia Foundation Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from collections import defaultdict
+import extras
+import logging
+import os
+import re
+
+OrderedDict = extras.try_imports(['collections.OrderedDict',
+ 'ordereddict.OrderedDict'])
+
+
+class CloneMapper(object):
+ log = logging.getLogger("zuul.CloneMapper")
+
+ def __init__(self, clonemap, projects):
+ self.clonemap = clonemap
+ self.projects = projects
+
+ def expand(self, workspace):
+ self.log.info("Workspace path set to: %s", workspace)
+
+ is_valid = True
+ ret = OrderedDict()
+ for project in self.projects:
+ dests = []
+ for mapping in self.clonemap:
+ if re.match(r'^%s$' % mapping['name'],
+ project):
+ # Might be matched more than one time
+ dests.append(
+ re.sub(mapping['name'], mapping['dest'], project))
+
+ if len(dests) > 1:
+ self.log.error("Duplicate destinations for %s: %s.",
+ project, dests)
+ is_valid = False
+ elif len(dests) == 0:
+ self.log.debug("Using %s as destination (unmatched)",
+ project)
+ ret[project] = [project]
+ else:
+ ret[project] = dests
+
+ if not is_valid:
+ raise Exception("Expansion error. Check error messages above")
+
+ self.log.info("Mapping projects to workspace...")
+ for project, dest in ret.iteritems():
+ dest = os.path.normpath(os.path.join(workspace, dest[0]))
+ ret[project] = dest
+ self.log.info(" %s -> %s", project, dest)
+
+ self.log.debug("Checking overlap in destination directories...")
+ check = defaultdict(list)
+ for project, dest in ret.iteritems():
+ check[dest].append(project)
+
+ dupes = dict((d, p) for (d, p) in check.iteritems() if len(p) > 1)
+ if dupes:
+ raise Exception("Some projects share the same destination: %s",
+ dupes)
+
+ self.log.info("Expansion completed.")
+ return ret
diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py
new file mode 100644
index 0000000..0961eb4
--- /dev/null
+++ b/zuul/lib/cloner.py
@@ -0,0 +1,145 @@
+# Copyright 2014 Antoine "hashar" Musso
+# Copyright 2014 Wikimedia Foundation Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import git
+import logging
+import os
+import re
+import yaml
+
+from git import GitCommandError
+from zuul.lib.clonemapper import CloneMapper
+from zuul.merger.merger import Repo
+
+
+class Cloner(object):
+ log = logging.getLogger("zuul.Cloner")
+
+ def __init__(self, git_base_url, projects, workspace, zuul_branch,
+ zuul_ref, zuul_url, branch=None, clone_map_file=None):
+
+ self.clone_map = []
+ self.dests = None
+
+ self.branch = branch
+ self.git_url = git_base_url
+ self.projects = projects
+ self.workspace = workspace
+ self.zuul_branch = zuul_branch
+ self.zuul_ref = zuul_ref
+ self.zuul_url = zuul_url
+
+ if clone_map_file:
+ self.readCloneMap(clone_map_file)
+
+ def readCloneMap(self, clone_map_file):
+ clone_map_file = os.path.expanduser(clone_map_file)
+ if not os.path.exists(clone_map_file):
+ raise Exception("Unable to read clone map file at %s." %
+ clone_map_file)
+ clone_map_file = open(clone_map_file)
+ self.clone_map = yaml.load(clone_map_file).get('clonemap')
+ self.log.info("Loaded map containing %s rules", len(self.clone_map))
+ return self.clone_map
+
+ def execute(self):
+ mapper = CloneMapper(self.clone_map, self.projects)
+ dests = mapper.expand(workspace=self.workspace)
+
+ self.log.info("Preparing %s repositories", len(dests))
+ for project, dest in dests.iteritems():
+ self.prepareRepo(project, dest)
+ self.log.info("Prepared all repositories")
+
+ def cloneUpstream(self, project, dest):
+ git_upstream = '%s/%s' % (self.git_url, project)
+ self.log.info("Creating repo %s from upstream %s",
+ project, git_upstream)
+ repo = Repo(
+ remote=git_upstream,
+ local=dest,
+ email=None,
+ username=None)
+
+ if not repo.isInitialized():
+ raise Exception("Error cloning %s to %s" % (git_upstream, dest))
+
+ return repo
+
+ def fetchFromZuul(self, repo, project, ref):
+ zuul_remote = '%s/%s' % (self.zuul_url, project)
+
+ try:
+ repo.fetchFrom(zuul_remote, ref)
+ self.log.debug("Fetched ref %s from %s", ref, project)
+ return True
+ except (ValueError, GitCommandError):
+ self.log.debug("Project %s in Zuul does not have ref %s",
+ project, ref)
+ return False
+
+ def prepareRepo(self, project, dest):
+ """Clone a repository for project at dest and apply a reference
+ suitable for testing. The reference lookup is attempted in this order:
+
+ 1) Zuul reference for the indicated branch
+ 2) Zuul reference for the master branch
+ 3) The tip of the indicated branch
+ 4) The tip of the master branch
+ """
+
+ repo = self.cloneUpstream(project, dest)
+
+ repo.update()
+ # Ensure that we don't have stale remotes around
+ repo.prune()
+
+ override_zuul_ref = self.zuul_ref
+ # FIXME should be origin HEAD branch which might not be 'master'
+ fallback_branch = 'master'
+ fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
+ self.zuul_ref)
+
+ if self.branch:
+ override_zuul_ref = re.sub(self.zuul_branch, self.branch,
+ self.zuul_ref)
+ if repo.hasBranch(self.branch):
+ self.log.debug("upstream repo has branch %s", self.branch)
+ fallback_branch = self.branch
+ fallback_zuul_ref = self.zuul_ref
+ else:
+ self.log.exception("upstream repo is missing branch %s",
+ self.branch)
+
+ if (self.fetchFromZuul(repo, project, override_zuul_ref)
+ or (fallback_zuul_ref != override_zuul_ref and
+ self.fetchFromZuul(repo, project, fallback_zuul_ref))
+ ):
+ # Work around a bug in GitPython which can not parse FETCH_HEAD
+ gitcmd = git.Git(dest)
+ fetch_head = gitcmd.rev_parse('FETCH_HEAD')
+ repo.checkout(fetch_head)
+ self.log.info("Prepared %s repo with commit %s",
+ project, fetch_head)
+ else:
+ # Checkout branch
+ self.log.debug("Falling back to branch %s", fallback_branch)
+ try:
+ repo.checkout('remotes/origin/%s' % fallback_branch)
+ except (ValueError, GitCommandError):
+ self.log.exception("Fallback branch not found: %s",
+ fallback_branch)
+ self.log.info("Prepared %s repo with branch %s",
+ project, fallback_branch)
diff --git a/zuul/lib/swift.py b/zuul/lib/swift.py
index 1780378..f953058 100644
--- a/zuul/lib/swift.py
+++ b/zuul/lib/swift.py
@@ -14,6 +14,7 @@
import hmac
from hashlib import sha1
+import logging
from time import time
import os
import random
@@ -24,6 +25,8 @@
class Swift(object):
+ log = logging.getLogger("zuul.lib.swift")
+
def __init__(self, config):
self.config = config
self.connection = False
@@ -36,7 +39,13 @@
for x in range(20)
)
- self.connect()
+ self.storage_url = ''
+
+ try:
+ self.connect()
+ except Exception as e:
+ self.log.warning("Unable to set up swift. Signed storage URL is "
+ "likely to be wrong. %s" % e)
def connect(self):
if self.config.has_section('swift'):
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 9e1f4c1..922c67e 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -58,6 +58,9 @@
repo.config_writer().write()
self._initialized = True
+ def isInitialized(self):
+ return self._initialized
+
def createRepoObject(self):
try:
self._ensure_cloned()
@@ -82,11 +85,24 @@
repo.head.reset(index=True, working_tree=True)
repo.git.clean('-x', '-f', '-d')
+ def prune(self):
+ repo = self.createRepoObject()
+ origin = repo.remotes.origin
+ stale_refs = origin.stale_refs
+ if stale_refs:
+ self.log.debug("Pruning stale refs: %s", stale_refs)
+ git.refs.RemoteReference.delete(repo, *stale_refs)
+
def getBranchHead(self, branch):
repo = self.createRepoObject()
branch_head = repo.heads[branch]
return branch_head.commit
+ def hasBranch(self, branch):
+ repo = self.createRepoObject()
+ origin = repo.remotes.origin
+ return branch in origin.refs
+
def getCommitFromRef(self, refname):
repo = self.createRepoObject()
if not refname in repo.refs:
@@ -130,6 +146,10 @@
except AssertionError:
origin.fetch(ref)
+ def fetchFrom(self, repository, refspec):
+ repo = self.createRepoObject()
+ repo.git.fetch(repository, refspec)
+
def createZuulRef(self, ref, commit='HEAD'):
repo = self.createRepoObject()
self.log.debug("CreateZuulRef %s at %s" % (ref, commit))
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 5e20e58..b3163b3 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -919,9 +919,7 @@
pipelines = []
data['pipelines'] = pipelines
- keys = self.layout.pipelines.keys()
- for key in keys:
- pipeline = self.layout.pipelines[key]
+ for pipeline in self.layout.pipelines.values():
pipelines.append(pipeline.formatStatusJSON())
return json.dumps(data)