Merge "Expand templates for project-specific matchers" into feature/zuulv3
diff --git a/.zuul.yaml b/.zuul.yaml
index e96bc5a..ab03a59 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -57,6 +57,7 @@
- tox-docs
- tox-pep8
- tox-py35
+ - zuul-stream-functional
post:
jobs:
- publish-openstack-python-docs-infra:
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 6dd8333..2248aa9 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -800,14 +800,22 @@
self.db = model.TimeDataBase(self.tmp_root)
def test_timedatabase(self):
- self.assertEqual(self.db.getEstimatedTime('job-name'), 0)
- self.db.update('job-name', 50, 'SUCCESS')
- self.assertEqual(self.db.getEstimatedTime('job-name'), 50)
- self.db.update('job-name', 100, 'SUCCESS')
- self.assertEqual(self.db.getEstimatedTime('job-name'), 75)
+ pipeline = Dummy(layout=Dummy(tenant=Dummy(name='test-tenant')))
+ change = Dummy(project=Dummy(canonical_name='git.example.com/foo/bar'))
+ job = Dummy(name='job-name')
+ item = Dummy(pipeline=pipeline,
+ change=change)
+ build = Dummy(build_set=Dummy(item=item),
+ job=job)
+
+ self.assertEqual(self.db.getEstimatedTime(build), 0)
+ self.db.update(build, 50, 'SUCCESS')
+ self.assertEqual(self.db.getEstimatedTime(build), 50)
+ self.db.update(build, 100, 'SUCCESS')
+ self.assertEqual(self.db.getEstimatedTime(build), 75)
for x in range(10):
- self.db.update('job-name', 100, 'SUCCESS')
- self.assertEqual(self.db.getEstimatedTime('job-name'), 100)
+ self.db.update(build, 100, 'SUCCESS')
+ self.assertEqual(self.db.getEstimatedTime(build), 100)
class TestGraph(BaseTestCase):
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index f33d964..f22d98c 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -617,7 +617,6 @@
self.assertEqual(B.reported, 2)
self.assertEqual(C.reported, 2)
- @skip("Disabled for early v3 development")
def _test_time_database(self, iteration):
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
@@ -626,7 +625,7 @@
self.waitUntilSettled()
time.sleep(2)
- data = json.loads(self.sched.formatStatusJSON())
+ data = json.loads(self.sched.formatStatusJSON('tenant-one'))
found_job = None
for pipeline in data['pipelines']:
if pipeline['name'] != 'gate':
@@ -652,7 +651,6 @@
self.executor_server.release()
self.waitUntilSettled()
- @skip("Disabled for early v3 development")
def test_time_database(self):
"Test the time database"
diff --git a/tools/zuul-cloner-shim.py b/tools/zuul-cloner-shim.py
deleted file mode 100755
index 3d1b2ae..0000000
--- a/tools/zuul-cloner-shim.py
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017 Red Hat
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import argparse
-import os
-import re
-import sys
-import yaml
-
-from collections import defaultdict
-from collections import OrderedDict
-
-REPO_SRC_DIR = "~zuul/src/git.openstack.org/"
-
-
-# Class copied from zuul/lib/conemapper.py with minor logging changes
-class CloneMapper(object):
-
- def __init__(self, clonemap, projects):
- self.clonemap = clonemap
- self.projects = projects
-
- def expand(self, workspace):
- print("Workspace path set to: %s" % workspace)
-
- is_valid = True
- ret = OrderedDict()
- errors = []
- for project in self.projects:
- dests = []
- for mapping in self.clonemap:
- if re.match(r'^%s$' % mapping['name'], project):
- # Might be matched more than one time
- dests.append(
- re.sub(mapping['name'], mapping['dest'], project))
-
- if len(dests) > 1:
- errors.append(
- "Duplicate destinations for %s: %s." % (project, dests))
- is_valid = False
- elif len(dests) == 0:
- print("Using %s as destination (unmatched)" % project)
- ret[project] = [project]
- else:
- ret[project] = dests
-
- if not is_valid:
- raise Exception("Expansion errors: %s" % errors)
-
- print("Mapping projects to workspace...")
- for project, dest in ret.items():
- dest = os.path.normpath(os.path.join(workspace, dest[0]))
- ret[project] = dest
- print(" %s -> %s" % (project, dest))
-
- print("Checking overlap in destination directories...")
- check = defaultdict(list)
- for project, dest in ret.items():
- check[dest].append(project)
-
- dupes = dict((d, p) for (d, p) in check.items() if len(p) > 1)
- if dupes:
- raise Exception("Some projects share the same destination: %s",
- dupes)
-
- print("Expansion completed.")
- return ret
-
-
-def parseArgs():
- ZUUL_ENV_SUFFIXES = ('branch', 'ref', 'url', 'project', 'newrev')
-
- parser = argparse.ArgumentParser()
-
- # Ignored arguments
- parser.add_argument('-v', '--verbose', dest='verbose',
- action='store_true', help='IGNORED')
- parser.add_argument('--color', dest='color', action='store_true',
- help='IGNORED')
- parser.add_argument('--cache-dir', dest='cache_dir', help='IGNORED')
- parser.add_argument('git_base_url', help='IGNORED')
- parser.add_argument('--branch', help='IGNORED')
- parser.add_argument('--project-branch', nargs=1, action='append',
- metavar='PROJECT=BRANCH', help='IGNORED')
- for zuul_suffix in ZUUL_ENV_SUFFIXES:
- env_name = 'ZUUL_%s' % zuul_suffix.upper()
- parser.add_argument(
- '--zuul-%s' % zuul_suffix, metavar='$' + env_name,
- help='IGNORED'
- )
-
- # Active arguments
- parser.add_argument('-m', '--map', dest='clone_map_file',
- help='specify clone map file')
- parser.add_argument('--workspace', dest='workspace',
- default=os.getcwd(),
- help='where to clone repositories too')
- parser.add_argument('projects', nargs='+',
- help='list of Gerrit projects to clone')
-
- return parser.parse_args()
-
-
-def readCloneMap(clone_map):
- clone_map_file = os.path.expanduser(clone_map)
- if not os.path.exists(clone_map_file):
- raise Exception("Unable to read clone map file at %s." %
- clone_map_file)
- clone_map_file = open(clone_map_file)
- clone_map = yaml.safe_load(clone_map_file).get('clonemap')
- return clone_map
-
-
-def main():
- args = parseArgs()
-
- clone_map = []
- if args.clone_map_file:
- clone_map = readCloneMap(args.clone_map_file)
-
- mapper = CloneMapper(clone_map, args.projects)
- dests = mapper.expand(workspace=args.workspace)
-
- for project in args.projects:
- src = os.path.join(os.path.expanduser(REPO_SRC_DIR), project)
- dst = dests[project]
-
- # Remove the tail end of the path (since the copy operation will
- # automatically create that)
- d = dst.rstrip('/')
- d, base = os.path.split(d)
- if not os.path.exists(d):
- print("Creating %s" % d)
- os.makedirs(d)
-
- # Create hard link copy of the source directory
- cmd = "cp -al %s %s" % (src, dst)
- print("%s" % cmd)
- if os.system(cmd):
- print("Error executing: %s" % cmd)
- sys.exit(1)
-
-
-if __name__ == "__main__":
- main()
diff --git a/zuul/model.py b/zuul/model.py
index 429a0c3..0e42368 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -2668,20 +2668,30 @@
class TimeDataBase(object):
def __init__(self, root):
self.root = root
- self.jobs = {}
- def _getTD(self, name):
- td = self.jobs.get(name)
- if not td:
- td = JobTimeData(os.path.join(self.root, name))
- self.jobs[name] = td
- td.load()
+ def _getTD(self, build):
+ if hasattr(build.build_set.item.change, 'branch'):
+ branch = build.build_set.item.change.branch
+ else:
+ branch = ''
+
+ dir_path = os.path.join(
+ self.root,
+ build.build_set.item.pipeline.layout.tenant.name,
+ build.build_set.item.change.project.canonical_name,
+ branch)
+ if not os.path.exists(dir_path):
+ os.makedirs(dir_path)
+ path = os.path.join(dir_path, build.job.name)
+
+ td = JobTimeData(path)
+ td.load()
return td
def getEstimatedTime(self, name):
return self._getTD(name).getEstimatedTime()
- def update(self, name, elapsed, result):
- td = self._getTD(name)
+ def update(self, build, elapsed, result):
+ td = self._getTD(build)
td.add(elapsed, result)
td.save()
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 52b34ec..806ba86 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -826,7 +826,7 @@
return
try:
build.estimated_time = float(self.time_database.getEstimatedTime(
- build.job.name))
+ build))
except Exception:
self.log.exception("Exception estimating build time:")
pipeline.manager.onBuildStarted(event.build)
@@ -865,8 +865,7 @@
if build.end_time and build.start_time and build.result:
duration = build.end_time - build.start_time
try:
- self.time_database.update(
- build.job.name, duration, build.result)
+ self.time_database.update(build, duration, build.result)
except Exception:
self.log.exception("Exception recording build time:")
pipeline.manager.onBuildCompleted(event.build)