James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 1 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 2 | # not use this file except in compliance with the License. You may obtain |
| 3 | # a copy of the License at |
| 4 | # |
| 5 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 6 | # |
| 7 | # Unless required by applicable law or agreed to in writing, software |
| 8 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 9 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 10 | # License for the specific language governing permissions and limitations |
| 11 | # under the License. |
| 12 | |
| 13 | import os |
| 14 | import logging |
| 15 | import yaml |
| 16 | |
| 17 | import voluptuous as vs |
| 18 | |
| 19 | import model |
| 20 | import zuul.manager |
| 21 | import zuul.manager.dependent |
| 22 | import zuul.manager.independent |
| 23 | from zuul import change_matcher |
| 24 | |
| 25 | |
| 26 | # Several forms accept either a single item or a list, this makes |
| 27 | # specifying that in the schema easy (and explicit). |
| 28 | def to_list(x): |
| 29 | return vs.Any([x], x) |
| 30 | |
| 31 | |
| 32 | def as_list(item): |
| 33 | if not item: |
| 34 | return [] |
| 35 | if isinstance(item, list): |
| 36 | return item |
| 37 | return [item] |
| 38 | |
| 39 | |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 40 | class JobParser(object): |
| 41 | @staticmethod |
| 42 | def getSchema(): |
| 43 | # TODOv3(jeblair, jhesketh): move to auth |
| 44 | swift = {vs.Required('name'): str, |
| 45 | 'container': str, |
| 46 | 'expiry': int, |
| 47 | 'max_file_size': int, |
| 48 | 'max-file-size': int, |
| 49 | 'max_file_count': int, |
| 50 | 'max-file-count': int, |
| 51 | 'logserver_prefix': str, |
| 52 | 'logserver-prefix': str, |
| 53 | } |
| 54 | |
| 55 | job = {vs.Required('name'): str, |
| 56 | 'parent': str, |
| 57 | 'queue-name': str, |
| 58 | 'failure-message': str, |
| 59 | 'success-message': str, |
| 60 | 'failure-url': str, |
| 61 | 'success-url': str, |
| 62 | 'voting': bool, |
Joshua Hesketh | 89b67f6 | 2016-02-11 21:22:14 +1100 | [diff] [blame] | 63 | 'mutex': str, |
Joshua Hesketh | dc7820c | 2016-03-11 13:14:28 +1100 | [diff] [blame] | 64 | 'tags': to_list(str), |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 65 | 'branches': to_list(str), |
| 66 | 'files': to_list(str), |
| 67 | 'swift': to_list(swift), |
| 68 | 'irrelevant-files': to_list(str), |
| 69 | 'timeout': int, |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 70 | '_project_source': str, # used internally |
| 71 | '_project_name': str, # used internally |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 72 | } |
| 73 | |
| 74 | return vs.Schema(job) |
| 75 | |
| 76 | @staticmethod |
| 77 | def fromYaml(layout, conf): |
| 78 | JobParser.getSchema()(conf) |
| 79 | job = model.Job(conf['name']) |
| 80 | if 'parent' in conf: |
| 81 | parent = layout.getJob(conf['parent']) |
| 82 | job.inheritFrom(parent) |
| 83 | job.timeout = conf.get('timeout', job.timeout) |
| 84 | job.workspace = conf.get('workspace', job.workspace) |
| 85 | job.pre_run = as_list(conf.get('pre-run', job.pre_run)) |
| 86 | job.post_run = as_list(conf.get('post-run', job.post_run)) |
| 87 | job.voting = conf.get('voting', True) |
Joshua Hesketh | 89b67f6 | 2016-02-11 21:22:14 +1100 | [diff] [blame] | 88 | job.mutex = conf.get('mutex', None) |
Joshua Hesketh | dc7820c | 2016-03-11 13:14:28 +1100 | [diff] [blame] | 89 | tags = conf.get('tags') |
| 90 | if tags: |
| 91 | # Tags are merged via a union rather than a |
| 92 | # destructive copy because they are intended to |
| 93 | # accumulate onto any previously applied tags from |
| 94 | # metajobs. |
| 95 | job.tags = job.tags.union(set(tags)) |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 96 | if not job.project_source: |
| 97 | # Thes attributes may not be overidden -- the first |
| 98 | # reference definition of a job is in the repo where it is |
| 99 | # first defined. |
| 100 | job.project_source = conf.get('_project_source') |
| 101 | job.project_name = conf.get('_project_name') |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 102 | job.failure_message = conf.get('failure-message', job.failure_message) |
| 103 | job.success_message = conf.get('success-message', job.success_message) |
| 104 | job.failure_url = conf.get('failure-url', job.failure_url) |
| 105 | job.success_url = conf.get('success-url', job.success_url) |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 106 | |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 107 | if 'branches' in conf: |
| 108 | matchers = [] |
| 109 | for branch in as_list(conf['branches']): |
| 110 | matchers.append(change_matcher.BranchMatcher(branch)) |
| 111 | job.branch_matcher = change_matcher.MatchAny(matchers) |
| 112 | if 'files' in conf: |
| 113 | matchers = [] |
| 114 | for fn in as_list(conf['files']): |
| 115 | matchers.append(change_matcher.FileMatcher(fn)) |
| 116 | job.file_matcher = change_matcher.MatchAny(matchers) |
| 117 | if 'irrelevant-files' in conf: |
| 118 | matchers = [] |
| 119 | for fn in as_list(conf['irrelevant-files']): |
| 120 | matchers.append(change_matcher.FileMatcher(fn)) |
| 121 | job.irrelevant_file_matcher = change_matcher.MatchAllFiles( |
| 122 | matchers) |
| 123 | return job |
| 124 | |
| 125 | |
James E. Blair | b97ed80 | 2015-12-21 15:55:35 -0800 | [diff] [blame] | 126 | class ProjectTemplateParser(object): |
| 127 | log = logging.getLogger("zuul.ProjectTemplateParser") |
| 128 | |
| 129 | @staticmethod |
| 130 | def getSchema(layout): |
| 131 | project_template = {vs.Required('name'): str} |
| 132 | for p in layout.pipelines.values(): |
| 133 | project_template[p.name] = {'queue': str, |
| 134 | 'jobs': [vs.Any(str, dict)]} |
| 135 | return vs.Schema(project_template) |
| 136 | |
| 137 | @staticmethod |
| 138 | def fromYaml(layout, conf): |
| 139 | ProjectTemplateParser.getSchema(layout)(conf) |
| 140 | project_template = model.ProjectConfig(conf['name']) |
| 141 | for pipeline in layout.pipelines.values(): |
| 142 | conf_pipeline = conf.get(pipeline.name) |
| 143 | if not conf_pipeline: |
| 144 | continue |
| 145 | project_pipeline = model.ProjectPipelineConfig() |
| 146 | project_template.pipelines[pipeline.name] = project_pipeline |
| 147 | project_pipeline.queue_name = conf.get('queue') |
| 148 | project_pipeline.job_tree = ProjectTemplateParser._parseJobTree( |
| 149 | layout, conf_pipeline.get('jobs')) |
| 150 | return project_template |
| 151 | |
| 152 | @staticmethod |
| 153 | def _parseJobTree(layout, conf, tree=None): |
| 154 | if not tree: |
| 155 | tree = model.JobTree(None) |
| 156 | for conf_job in conf: |
| 157 | if isinstance(conf_job, basestring): |
| 158 | tree.addJob(layout.getJob(conf_job)) |
| 159 | elif isinstance(conf_job, dict): |
| 160 | # A dictionary in a job tree may override params, or |
| 161 | # be the root of a sub job tree, or both. |
| 162 | jobname, attrs = dict.items()[0] |
| 163 | jobs = attrs.pop('jobs') |
| 164 | if attrs: |
| 165 | # We are overriding params, so make a new job def |
| 166 | attrs['name'] = jobname |
| 167 | subtree = tree.addJob(JobParser.fromYaml(layout, attrs)) |
| 168 | else: |
| 169 | # Not overriding, so get existing job |
| 170 | subtree = tree.addJob(layout.getJob(jobname)) |
| 171 | |
| 172 | if jobs: |
| 173 | # This is the root of a sub tree |
| 174 | ProjectTemplateParser._parseJobTree(layout, jobs, subtree) |
| 175 | else: |
| 176 | raise Exception("Job must be a string or dictionary") |
| 177 | return tree |
| 178 | |
| 179 | |
| 180 | class ProjectParser(object): |
| 181 | log = logging.getLogger("zuul.ProjectParser") |
| 182 | |
| 183 | @staticmethod |
| 184 | def getSchema(layout): |
| 185 | project = {vs.Required('name'): str, |
| 186 | 'templates': [str]} |
| 187 | for p in layout.pipelines.values(): |
| 188 | project[p.name] = {'queue': str, |
| 189 | 'jobs': [vs.Any(str, dict)]} |
| 190 | return vs.Schema(project) |
| 191 | |
| 192 | @staticmethod |
| 193 | def fromYaml(layout, conf): |
| 194 | ProjectParser.getSchema(layout)(conf) |
| 195 | conf_templates = conf.pop('templates', []) |
| 196 | # The way we construct a project definition is by parsing the |
| 197 | # definition as a template, then applying all of the |
| 198 | # templates, including the newly parsed one, in order. |
| 199 | project_template = ProjectTemplateParser.fromYaml(layout, conf) |
| 200 | configs = [layout.project_templates[name] for name in conf_templates] |
| 201 | configs.append(project_template) |
| 202 | project = model.ProjectConfig(conf['name']) |
| 203 | for pipeline in layout.pipelines.values(): |
| 204 | project_pipeline = model.ProjectPipelineConfig() |
| 205 | project_pipeline.job_tree = model.JobTree(None) |
| 206 | queue_name = None |
| 207 | # For every template, iterate over the job tree and replace or |
| 208 | # create the jobs in the final definition as needed. |
| 209 | pipeline_defined = False |
| 210 | for template in configs: |
| 211 | ProjectParser.log.debug("Applying template %s to pipeline %s" % |
| 212 | (template.name, pipeline.name)) |
| 213 | if pipeline.name in template.pipelines: |
| 214 | pipeline_defined = True |
| 215 | template_pipeline = template.pipelines[pipeline.name] |
| 216 | project_pipeline.job_tree.inheritFrom( |
| 217 | template_pipeline.job_tree) |
| 218 | if template_pipeline.queue_name: |
| 219 | queue_name = template_pipeline.queue_name |
| 220 | if queue_name: |
| 221 | project_pipeline.queue_name = queue_name |
| 222 | if pipeline_defined: |
| 223 | project.pipelines[pipeline.name] = project_pipeline |
| 224 | return project |
| 225 | |
| 226 | |
James E. Blair | fb610ce | 2015-12-22 10:24:40 -0800 | [diff] [blame] | 227 | class PipelineParser(object): |
| 228 | log = logging.getLogger("zuul.PipelineParser") |
| 229 | |
| 230 | # A set of reporter configuration keys to action mapping |
| 231 | reporter_actions = { |
| 232 | 'start': 'start_actions', |
| 233 | 'success': 'success_actions', |
| 234 | 'failure': 'failure_actions', |
| 235 | 'merge-failure': 'merge_failure_actions', |
| 236 | 'disabled': 'disabled_actions', |
| 237 | } |
| 238 | |
| 239 | @staticmethod |
| 240 | def getDriverSchema(dtype, connections): |
| 241 | # TODO(jhesketh): Make the driver discovery dynamic |
| 242 | connection_drivers = { |
| 243 | 'trigger': { |
| 244 | 'gerrit': 'zuul.trigger.gerrit', |
| 245 | }, |
| 246 | 'reporter': { |
| 247 | 'gerrit': 'zuul.reporter.gerrit', |
| 248 | 'smtp': 'zuul.reporter.smtp', |
| 249 | }, |
| 250 | } |
| 251 | standard_drivers = { |
| 252 | 'trigger': { |
| 253 | 'timer': 'zuul.trigger.timer', |
| 254 | 'zuul': 'zuul.trigger.zuultrigger', |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | schema = {} |
| 259 | # Add the configured connections as available layout options |
| 260 | for connection_name, connection in connections.connections.items(): |
| 261 | for dname, dmod in connection_drivers.get(dtype, {}).items(): |
| 262 | if connection.driver_name == dname: |
| 263 | schema[connection_name] = to_list(__import__( |
| 264 | connection_drivers[dtype][dname], |
| 265 | fromlist=['']).getSchema()) |
| 266 | |
| 267 | # Standard drivers are always available and don't require a unique |
| 268 | # (connection) name |
| 269 | for dname, dmod in standard_drivers.get(dtype, {}).items(): |
| 270 | schema[dname] = to_list(__import__( |
| 271 | standard_drivers[dtype][dname], fromlist=['']).getSchema()) |
| 272 | |
| 273 | return schema |
| 274 | |
| 275 | @staticmethod |
| 276 | def getSchema(layout, connections): |
| 277 | manager = vs.Any('independent', |
| 278 | 'dependent') |
| 279 | |
| 280 | precedence = vs.Any('normal', 'low', 'high') |
| 281 | |
| 282 | approval = vs.Schema({'username': str, |
| 283 | 'email-filter': str, |
| 284 | 'email': str, |
| 285 | 'older-than': str, |
| 286 | 'newer-than': str, |
| 287 | }, extra=True) |
| 288 | |
| 289 | require = {'approval': to_list(approval), |
| 290 | 'open': bool, |
| 291 | 'current-patchset': bool, |
| 292 | 'status': to_list(str)} |
| 293 | |
| 294 | reject = {'approval': to_list(approval)} |
| 295 | |
| 296 | window = vs.All(int, vs.Range(min=0)) |
| 297 | window_floor = vs.All(int, vs.Range(min=1)) |
| 298 | window_type = vs.Any('linear', 'exponential') |
| 299 | window_factor = vs.All(int, vs.Range(min=1)) |
| 300 | |
| 301 | pipeline = {vs.Required('name'): str, |
| 302 | vs.Required('manager'): manager, |
| 303 | 'source': str, |
| 304 | 'precedence': precedence, |
| 305 | 'description': str, |
| 306 | 'require': require, |
| 307 | 'reject': reject, |
| 308 | 'success-message': str, |
| 309 | 'failure-message': str, |
| 310 | 'merge-failure-message': str, |
| 311 | 'footer-message': str, |
| 312 | 'dequeue-on-new-patchset': bool, |
| 313 | 'ignore-dependencies': bool, |
| 314 | 'disable-after-consecutive-failures': |
| 315 | vs.All(int, vs.Range(min=1)), |
| 316 | 'window': window, |
| 317 | 'window-floor': window_floor, |
| 318 | 'window-increase-type': window_type, |
| 319 | 'window-increase-factor': window_factor, |
| 320 | 'window-decrease-type': window_type, |
| 321 | 'window-decrease-factor': window_factor, |
| 322 | } |
| 323 | pipeline['trigger'] = vs.Required( |
| 324 | PipelineParser.getDriverSchema('trigger', connections)) |
| 325 | for action in ['start', 'success', 'failure', 'merge-failure', |
| 326 | 'disabled']: |
| 327 | pipeline[action] = PipelineParser.getDriverSchema('reporter', |
| 328 | connections) |
| 329 | return vs.Schema(pipeline) |
| 330 | |
| 331 | @staticmethod |
| 332 | def fromYaml(layout, connections, scheduler, conf): |
| 333 | PipelineParser.getSchema(layout, connections)(conf) |
| 334 | pipeline = model.Pipeline(conf['name'], layout) |
| 335 | pipeline.description = conf.get('description') |
| 336 | |
| 337 | pipeline.source = connections.getSource(conf['source']) |
| 338 | |
| 339 | precedence = model.PRECEDENCE_MAP[conf.get('precedence')] |
| 340 | pipeline.precedence = precedence |
| 341 | pipeline.failure_message = conf.get('failure-message', |
| 342 | "Build failed.") |
| 343 | pipeline.merge_failure_message = conf.get( |
| 344 | 'merge-failure-message', "Merge Failed.\n\nThis change or one " |
| 345 | "of its cross-repo dependencies was unable to be " |
| 346 | "automatically merged with the current state of its " |
| 347 | "repository. Please rebase the change and upload a new " |
| 348 | "patchset.") |
| 349 | pipeline.success_message = conf.get('success-message', |
| 350 | "Build succeeded.") |
| 351 | pipeline.footer_message = conf.get('footer-message', "") |
| 352 | pipeline.dequeue_on_new_patchset = conf.get( |
| 353 | 'dequeue-on-new-patchset', True) |
| 354 | pipeline.ignore_dependencies = conf.get( |
| 355 | 'ignore-dependencies', False) |
| 356 | |
| 357 | for conf_key, action in PipelineParser.reporter_actions.items(): |
| 358 | reporter_set = [] |
| 359 | if conf.get(conf_key): |
| 360 | for reporter_name, params \ |
| 361 | in conf.get(conf_key).items(): |
| 362 | reporter = connections.getReporter(reporter_name, |
| 363 | params) |
| 364 | reporter.setAction(conf_key) |
| 365 | reporter_set.append(reporter) |
| 366 | setattr(pipeline, action, reporter_set) |
| 367 | |
| 368 | # If merge-failure actions aren't explicit, use the failure actions |
| 369 | if not pipeline.merge_failure_actions: |
| 370 | pipeline.merge_failure_actions = pipeline.failure_actions |
| 371 | |
| 372 | pipeline.disable_at = conf.get( |
| 373 | 'disable-after-consecutive-failures', None) |
| 374 | |
| 375 | pipeline.window = conf.get('window', 20) |
| 376 | pipeline.window_floor = conf.get('window-floor', 3) |
| 377 | pipeline.window_increase_type = conf.get( |
| 378 | 'window-increase-type', 'linear') |
| 379 | pipeline.window_increase_factor = conf.get( |
| 380 | 'window-increase-factor', 1) |
| 381 | pipeline.window_decrease_type = conf.get( |
| 382 | 'window-decrease-type', 'exponential') |
| 383 | pipeline.window_decrease_factor = conf.get( |
| 384 | 'window-decrease-factor', 2) |
| 385 | |
| 386 | manager_name = conf['manager'] |
| 387 | if manager_name == 'dependent': |
| 388 | manager = zuul.manager.dependent.DependentPipelineManager( |
| 389 | scheduler, pipeline) |
| 390 | elif manager_name == 'independent': |
| 391 | manager = zuul.manager.independent.IndependentPipelineManager( |
| 392 | scheduler, pipeline) |
| 393 | |
| 394 | pipeline.setManager(manager) |
| 395 | layout.pipelines[conf['name']] = pipeline |
| 396 | |
| 397 | if 'require' in conf or 'reject' in conf: |
| 398 | require = conf.get('require', {}) |
| 399 | reject = conf.get('reject', {}) |
| 400 | f = model.ChangeishFilter( |
| 401 | open=require.get('open'), |
| 402 | current_patchset=require.get('current-patchset'), |
| 403 | statuses=to_list(require.get('status')), |
| 404 | required_approvals=to_list(require.get('approval')), |
| 405 | reject_approvals=to_list(reject.get('approval')) |
| 406 | ) |
| 407 | manager.changeish_filters.append(f) |
| 408 | |
| 409 | for trigger_name, trigger_config\ |
| 410 | in conf.get('trigger').items(): |
| 411 | trigger = connections.getTrigger(trigger_name, trigger_config) |
| 412 | pipeline.triggers.append(trigger) |
| 413 | |
| 414 | # TODO: move |
| 415 | manager.event_filters += trigger.getEventFilters( |
| 416 | conf['trigger'][trigger_name]) |
| 417 | |
| 418 | return pipeline |
| 419 | |
| 420 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 421 | class TenantParser(object): |
| 422 | log = logging.getLogger("zuul.TenantParser") |
| 423 | |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 424 | tenant_source = vs.Schema({'config-repos': [str], |
| 425 | 'project-repos': [str]}) |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 426 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 427 | @staticmethod |
| 428 | def validateTenantSources(connections): |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 429 | def v(value, path=[]): |
| 430 | if isinstance(value, dict): |
| 431 | for k, val in value.items(): |
| 432 | connections.getSource(k) |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 433 | TenantParser.validateTenantSource(val, path + [k]) |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 434 | else: |
| 435 | raise vs.Invalid("Invalid tenant source", path) |
| 436 | return v |
| 437 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 438 | @staticmethod |
| 439 | def validateTenantSource(value, path=[]): |
| 440 | TenantParser.tenant_source(value) |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 441 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 442 | @staticmethod |
| 443 | def getSchema(connections=None): |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 444 | tenant = {vs.Required('name'): str, |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 445 | 'source': TenantParser.validateTenantSources(connections)} |
| 446 | return vs.Schema(tenant) |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 447 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 448 | @staticmethod |
| 449 | def fromYaml(base, connections, scheduler, merger, conf): |
| 450 | TenantParser.getSchema(connections)(conf) |
| 451 | tenant = model.Tenant(conf['name']) |
| 452 | tenant_config = model.UnparsedTenantConfig() |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 453 | incdata = TenantParser._loadTenantInRepoLayouts(merger, connections, |
| 454 | conf) |
| 455 | tenant_config.extend(incdata) |
| 456 | tenant.layout = TenantParser._parseLayout(base, tenant_config, |
| 457 | scheduler, connections) |
| 458 | return tenant |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 459 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 460 | @staticmethod |
| 461 | def _loadTenantInRepoLayouts(merger, connections, conf_tenant): |
| 462 | config = model.UnparsedTenantConfig() |
| 463 | jobs = [] |
| 464 | for source_name, conf_source in conf_tenant.get('source', {}).items(): |
| 465 | source = connections.getSource(source_name) |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 466 | |
| 467 | # Get main config files. These files are permitted the |
| 468 | # full range of configuration. |
| 469 | for conf_repo in conf_source.get('config-repos', []): |
| 470 | project = source.getProject(conf_repo) |
| 471 | url = source.getGitUrl(project) |
| 472 | job = merger.getFiles(project.name, url, 'master', |
| 473 | files=['zuul.yaml', '.zuul.yaml']) |
| 474 | job.project = project |
| 475 | job.config_repo = True |
| 476 | jobs.append(job) |
| 477 | |
| 478 | # Get in-project-repo config files which have a restricted |
| 479 | # set of options. |
| 480 | for conf_repo in conf_source.get('project-repos', []): |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 481 | project = source.getProject(conf_repo) |
| 482 | url = source.getGitUrl(project) |
| 483 | # TODOv3(jeblair): config should be branch specific |
| 484 | job = merger.getFiles(project.name, url, 'master', |
| 485 | files=['.zuul.yaml']) |
| 486 | job.project = project |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 487 | job.config_repo = False |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 488 | jobs.append(job) |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 489 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 490 | for job in jobs: |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 491 | # Note: this is an ordered list -- we wait for cat jobs to |
| 492 | # complete in the order they were launched which is the |
| 493 | # same order they were defined in the main config file. |
| 494 | # This is important for correct inheritence. |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 495 | TenantParser.log.debug("Waiting for cat job %s" % (job,)) |
| 496 | job.wait() |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 497 | for fn in ['zuul.yaml', '.zuul.yaml']: |
| 498 | if job.files.get(fn): |
| 499 | TenantParser.log.info( |
| 500 | "Loading configuration from %s/%s" % |
| 501 | (job.project, fn)) |
| 502 | if job.config_repo: |
| 503 | incdata = TenantParser._parseConfigRepoLayout( |
| 504 | job.files[fn], source_name, job.project.name) |
| 505 | else: |
| 506 | incdata = TenantParser._parseProjectRepoLayout( |
| 507 | job.files[fn], source_name, job.project.name) |
| 508 | config.extend(incdata) |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 509 | return config |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 510 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 511 | @staticmethod |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 512 | def _parseConfigRepoLayout(data, source_name, project_name): |
| 513 | # This is the top-level configuration for a tenant. |
| 514 | config = model.UnparsedTenantConfig() |
| 515 | config.extend(yaml.load(data)) |
| 516 | |
| 517 | # Remember where this job was defined |
| 518 | for conf_job in config.jobs: |
| 519 | conf_job['_project_source'] = source_name |
| 520 | conf_job['_project_name'] = project_name |
| 521 | |
| 522 | return config |
| 523 | |
| 524 | @staticmethod |
| 525 | def _parseProjectRepoLayout(data, source_name, project_name): |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 526 | # TODOv3(jeblair): this should implement some rules to protect |
| 527 | # aspects of the config that should not be changed in-repo |
James E. Blair | 96c6bf8 | 2016-01-15 16:20:40 -0800 | [diff] [blame^] | 528 | config = model.UnparsedTenantConfig() |
| 529 | config.extend(yaml.load(data)) |
| 530 | |
| 531 | # Remember where this job was defined |
| 532 | for conf_job in config.jobs: |
| 533 | conf_job['_project_source'] = source_name |
| 534 | conf_job['_project_name'] = project_name |
| 535 | |
| 536 | return config |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 537 | |
| 538 | @staticmethod |
| 539 | def _parseLayout(base, data, scheduler, connections): |
| 540 | layout = model.Layout() |
| 541 | |
| 542 | for config_pipeline in data.pipelines: |
| 543 | layout.addPipeline(PipelineParser.fromYaml(layout, connections, |
| 544 | scheduler, |
| 545 | config_pipeline)) |
| 546 | |
| 547 | for config_job in data.jobs: |
| 548 | layout.addJob(JobParser.fromYaml(layout, config_job)) |
| 549 | |
| 550 | for config_template in data.project_templates: |
| 551 | layout.addProjectTemplate(ProjectTemplateParser.fromYaml( |
| 552 | layout, config_template)) |
| 553 | |
| 554 | for config_project in data.projects: |
| 555 | layout.addProjectConfig(ProjectParser.fromYaml( |
| 556 | layout, config_project)) |
| 557 | |
| 558 | for pipeline in layout.pipelines.values(): |
| 559 | pipeline.manager._postConfig(layout) |
| 560 | |
| 561 | return layout |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 562 | |
| 563 | |
| 564 | class ConfigLoader(object): |
| 565 | log = logging.getLogger("zuul.ConfigLoader") |
| 566 | |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 567 | def loadConfig(self, config_path, scheduler, merger, connections): |
| 568 | abide = model.Abide() |
| 569 | |
| 570 | if config_path: |
| 571 | config_path = os.path.expanduser(config_path) |
| 572 | if not os.path.exists(config_path): |
| 573 | raise Exception("Unable to read tenant config file at %s" % |
| 574 | config_path) |
| 575 | with open(config_path) as config_file: |
| 576 | self.log.info("Loading configuration from %s" % (config_path,)) |
| 577 | data = yaml.load(config_file) |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 578 | config = model.UnparsedAbideConfig() |
| 579 | config.extend(data) |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 580 | base = os.path.dirname(os.path.realpath(config_path)) |
| 581 | |
James E. Blair | d8e778f | 2015-12-22 14:09:20 -0800 | [diff] [blame] | 582 | for conf_tenant in config.tenants: |
| 583 | tenant = TenantParser.fromYaml(base, connections, scheduler, |
| 584 | merger, conf_tenant) |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 585 | abide.tenants[tenant.name] = tenant |
James E. Blair | 8300578 | 2015-12-11 14:46:03 -0800 | [diff] [blame] | 586 | return abide |