Merge "Use the executor cached repos more often" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index c977b73..7360625 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1307,10 +1307,10 @@
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
- def doMergeChanges(self, items, repo_state):
+ def doMergeChanges(self, merger, items, repo_state):
# Get a merger in order to update the repos involved in this job.
commit = super(RecordingAnsibleJob, self).doMergeChanges(
- items, repo_state)
+ merger, items, repo_state)
if not commit: # merge conflict
self.recordResult('MERGER_FAILURE')
return commit
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index f82b7c2..1d9cd72 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -298,6 +298,7 @@
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
+ canonical_name=project.canonical_name,
override_branch=override_branch,
default_branch=project_default_branch)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 7cdff9c..e906863 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -27,7 +27,6 @@
from zuul.lib.yamlutil import yaml
import gear
-import git
from six.moves import shlex_quote
import zuul.merger.merger
@@ -412,8 +411,13 @@
self.job_workers = {}
def _getMerger(self, root):
+ if root != self.merge_root:
+ cache_root = self.merge_root
+ else:
+ cache_root = None
return zuul.merger.merger.Merger(root, self.connections,
- self.merge_email, self.merge_name)
+ self.merge_email, self.merge_name,
+ cache_root)
def start(self):
self._running = True
@@ -716,36 +720,27 @@
task.wait()
self.log.debug("Job %s: git updates complete" % (self.job.unique,))
- repos = []
+ merger = self.executor_server._getMerger(self.jobdir.src_root)
+ repos = {}
for project in args['projects']:
self.log.debug("Cloning %s/%s" % (project['connection'],
project['name'],))
- source = self.executor_server.connections.getSource(
- project['connection'])
- project_object = source.getProject(project['name'])
- url = source.getGitUrl(project_object)
- repo = git.Repo.clone_from(
- os.path.join(self.executor_server.merge_root,
- source.canonical_hostname,
- project['name']),
- os.path.join(self.jobdir.src_root,
- source.canonical_hostname,
- project['name']))
-
- repo.remotes.origin.config_writer.set('url', url)
- repos.append(repo)
+ repo = merger.getRepo(project['connection'],
+ project['name'])
+ repos[project['canonical_name']] = repo
merge_items = [i for i in args['items'] if i.get('refspec')]
if merge_items:
- if not self.doMergeChanges(merge_items, args['repo_state']):
+ if not self.doMergeChanges(merger, merge_items,
+ args['repo_state']):
# There was a merge conflict and we have already sent
# a work complete result, don't run any jobs
return
# Delete the origin remote from each repo we set up since
# it will not be valid within the jobs.
- for repo in repos:
- repo.delete_remote(repo.remotes.origin)
+ for repo in repos.values():
+ repo.deleteRemote('origin')
# is the playbook in a repo that we have already prepared?
trusted, untrusted = self.preparePlaybookRepos(args)
@@ -783,9 +778,7 @@
result = dict(result=result)
self.job.sendWorkComplete(json.dumps(result))
- def doMergeChanges(self, items, repo_state):
- # Get a merger in order to update the repos involved in this job.
- merger = self.executor_server._getMerger(self.jobdir.src_root)
+ def doMergeChanges(self, merger, items, repo_state):
ret = merger.mergeChanges(items, repo_state=repo_state)
if not ret: # merge conflict
result = dict(result='MERGER_FAILURE')
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index bc57ca0..4df24aa 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -44,11 +44,12 @@
class Repo(object):
log = logging.getLogger("zuul.Repo")
- def __init__(self, remote, local, email, username):
+ def __init__(self, remote, local, email, username, cache_path=None):
self.remote_url = remote
self.local_path = local
self.email = email
self.username = username
+ self.cache_path = cache_path
self._initialized = False
try:
self._ensure_cloned()
@@ -60,17 +61,32 @@
if self._initialized and repo_is_cloned:
return
# If the repo does not exist, clone the repo.
+ rewrite_url = False
if not repo_is_cloned:
self.log.debug("Cloning from %s to %s" % (self.remote_url,
self.local_path))
- git.Repo.clone_from(self.remote_url, self.local_path)
+ if self.cache_path:
+ git.Repo.clone_from(self.cache_path, self.local_path)
+ rewrite_url = True
+ else:
+ git.Repo.clone_from(self.remote_url, self.local_path)
repo = git.Repo(self.local_path)
+ # Create local branches corresponding to all the remote branches
+ if not repo_is_cloned:
+ origin = repo.remotes.origin
+ for ref in origin.refs:
+ if ref.remote_head == 'HEAD':
+ continue
+ repo.create_head(ref.remote_head, ref, force=True)
with repo.config_writer() as config_writer:
if self.email:
config_writer.set_value('user', 'email', self.email)
if self.username:
config_writer.set_value('user', 'name', self.username)
config_writer.write()
+ if rewrite_url:
+ with repo.remotes.origin.config_writer as config_writer:
+ config_writer.set('url', self.remote_url)
self._initialized = True
def isInitialized(self):
@@ -157,6 +173,11 @@
reset_repo_to_head(repo)
return repo.head.commit
+ def checkoutLocalBranch(self, branch):
+ repo = self.createRepoObject()
+ ref = repo.heads[branch].commit
+ self.checkout(ref)
+
def cherryPick(self, ref):
repo = self.createRepoObject()
self.log.debug("Cherry-picking %s" % ref)
@@ -230,11 +251,16 @@
ret[fn] = None
return ret
+ def deleteRemote(self, remote):
+ repo = self.createRepoObject()
+ repo.delete_remote(repo.remotes[remote])
+
class Merger(object):
log = logging.getLogger("zuul.Merger")
- def __init__(self, working_root, connections, email, username):
+ def __init__(self, working_root, connections, email, username,
+ cache_root=None):
self.repos = {}
self.working_root = working_root
if not os.path.exists(working_root):
@@ -242,6 +268,7 @@
self.connections = connections
self.email = email
self.username = username
+ self.cache_root = cache_root
def _get_ssh_cmd(self, connection_name):
sshkey = self.connections.connections.get(connection_name).\
@@ -264,7 +291,12 @@
key = '/'.join([hostname, project_name])
try:
path = os.path.join(self.working_root, hostname, project_name)
- repo = Repo(url, path, self.email, self.username)
+ if self.cache_root:
+ cache_path = os.path.join(self.cache_root, hostname,
+ project_name)
+ else:
+ cache_path = None
+ repo = Repo(url, path, self.email, self.username, cache_path)
self.repos[key] = repo
except Exception:
@@ -301,15 +333,10 @@
connection_name, project_name)
def checkoutBranch(self, connection_name, project_name, branch):
+ self.log.info("Checking out %s/%s branch %s",
+ connection_name, project_name, branch)
repo = self.getRepo(connection_name, project_name)
- if repo.hasBranch(branch):
- self.log.info("Checking out branch %s of %s/%s" %
- (branch, connection_name, project_name))
- head = repo.getBranchHead(branch)
- repo.checkout(head)
- else:
- raise Exception("Project %s/%s does not have branch %s" %
- (connection_name, project_name, branch))
+ repo.checkoutLocalBranch(branch)
def _saveRepoState(self, connection_name, project_name, repo,
repo_state):