| # Copyright 2014 Antoine "hashar" Musso |
| # Copyright 2014 Wikimedia Foundation Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| import git |
| import logging |
| import os |
| import re |
| import yaml |
| |
| from git import GitCommandError |
| from zuul.lib.clonemapper import CloneMapper |
| from zuul.merger.merger import Repo |
| |
| |
| class Cloner(object): |
| log = logging.getLogger("zuul.Cloner") |
| |
| def __init__(self, git_base_url, projects, workspace, zuul_branch, |
| zuul_ref, zuul_url, branch=None, clone_map_file=None, |
| project_branches=None, cache_dir=None): |
| |
| self.clone_map = [] |
| self.dests = None |
| |
| self.branch = branch |
| self.git_url = git_base_url |
| self.cache_dir = cache_dir |
| self.projects = projects |
| self.workspace = workspace |
| self.zuul_branch = zuul_branch |
| self.zuul_ref = zuul_ref |
| self.zuul_url = zuul_url |
| self.project_branches = project_branches or {} |
| |
| if clone_map_file: |
| self.readCloneMap(clone_map_file) |
| |
| def readCloneMap(self, clone_map_file): |
| clone_map_file = os.path.expanduser(clone_map_file) |
| if not os.path.exists(clone_map_file): |
| raise Exception("Unable to read clone map file at %s." % |
| clone_map_file) |
| clone_map_file = open(clone_map_file) |
| self.clone_map = yaml.load(clone_map_file).get('clonemap') |
| self.log.info("Loaded map containing %s rules", len(self.clone_map)) |
| return self.clone_map |
| |
| def execute(self): |
| mapper = CloneMapper(self.clone_map, self.projects) |
| dests = mapper.expand(workspace=self.workspace) |
| |
| self.log.info("Preparing %s repositories", len(dests)) |
| for project, dest in dests.iteritems(): |
| self.prepareRepo(project, dest) |
| self.log.info("Prepared all repositories") |
| |
| def cloneUpstream(self, project, dest): |
| # Check for a cached git repo first |
| git_cache = '%s/%s' % (self.cache_dir, project) |
| git_upstream = '%s/%s' % (self.git_url, project) |
| if (self.cache_dir and |
| os.path.exists(git_cache) and |
| not os.path.exists(dest)): |
| # file:// tells git not to hard-link across repos |
| git_cache = 'file://%s' % git_cache |
| self.log.info("Creating repo %s from cache %s", |
| project, git_cache) |
| new_repo = git.Repo.clone_from(git_cache, dest) |
| self.log.info("Updating origin remote in repo %s to %s", |
| project, git_upstream) |
| origin = new_repo.remotes.origin.config_writer.set( |
| 'url', git_upstream) |
| else: |
| self.log.info("Creating repo %s from upstream %s", |
| project, git_upstream) |
| repo = Repo( |
| remote=git_upstream, |
| local=dest, |
| email=None, |
| username=None) |
| |
| if not repo.isInitialized(): |
| raise Exception("Error cloning %s to %s" % (git_upstream, dest)) |
| |
| return repo |
| |
| def fetchFromZuul(self, repo, project, ref): |
| zuul_remote = '%s/%s' % (self.zuul_url, project) |
| |
| try: |
| repo.fetchFrom(zuul_remote, ref) |
| self.log.debug("Fetched ref %s from %s", ref, project) |
| return True |
| except (ValueError, GitCommandError): |
| self.log.debug("Project %s in Zuul does not have ref %s", |
| project, ref) |
| return False |
| |
| def prepareRepo(self, project, dest): |
| """Clone a repository for project at dest and apply a reference |
| suitable for testing. The reference lookup is attempted in this order: |
| |
| 1) Zuul reference for the indicated branch |
| 2) Zuul reference for the master branch |
| 3) The tip of the indicated branch |
| 4) The tip of the master branch |
| |
| The "indicated branch" is one of the following: |
| |
| A) The project-specific override branch (from project_branches arg) |
| B) The user specified branch (from the branch arg) |
| C) ZUUL_BRANCH (from the zuul_branch arg) |
| """ |
| |
| repo = self.cloneUpstream(project, dest) |
| |
| repo.update() |
| # Ensure that we don't have stale remotes around |
| repo.prune() |
| |
| indicated_branch = self.branch or self.zuul_branch |
| if project in self.project_branches: |
| indicated_branch = self.project_branches[project] |
| |
| override_zuul_ref = re.sub(self.zuul_branch, indicated_branch, |
| self.zuul_ref) |
| |
| if repo.hasBranch(indicated_branch): |
| self.log.debug("upstream repo has branch %s", indicated_branch) |
| fallback_branch = indicated_branch |
| else: |
| self.log.debug("upstream repo is missing branch %s", |
| self.branch) |
| # FIXME should be origin HEAD branch which might not be 'master' |
| fallback_branch = 'master' |
| |
| fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch, |
| self.zuul_ref) |
| |
| if (self.fetchFromZuul(repo, project, override_zuul_ref) |
| or (fallback_zuul_ref != override_zuul_ref and |
| self.fetchFromZuul(repo, project, fallback_zuul_ref)) |
| ): |
| # Work around a bug in GitPython which can not parse FETCH_HEAD |
| gitcmd = git.Git(dest) |
| fetch_head = gitcmd.rev_parse('FETCH_HEAD') |
| repo.checkout(fetch_head) |
| self.log.info("Prepared %s repo with commit %s", |
| project, fetch_head) |
| else: |
| # Checkout branch |
| self.log.debug("Falling back to branch %s", fallback_branch) |
| try: |
| repo.checkout('remotes/origin/%s' % fallback_branch) |
| except (ValueError, GitCommandError): |
| self.log.exception("Fallback branch not found: %s", |
| fallback_branch) |
| self.log.info("Prepared %s repo with branch %s", |
| project, fallback_branch) |