blob: a9bcee15e5fb053824d8434e0c455a168436affa [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070019import gc
20import hashlib
21import json
22import logging
23import os
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070025from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070026import random
27import re
28import select
29import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030030from six.moves import reload_module
Clark Boylan21a2c812017-04-24 15:44:55 -070031try:
32 from cStringIO import StringIO
33except Exception:
34 from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070035import socket
36import string
37import subprocess
James E. Blair1c236df2017-02-01 14:07:24 -080038import sys
James E. Blairf84026c2015-12-08 16:11:46 -080039import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070040import threading
Clark Boylan8208c192017-04-24 18:08:08 -070041import traceback
Clark Boylanb640e052014-04-03 16:41:46 -070042import time
Joshua Heskethd78b4482015-09-14 16:56:34 -060043import uuid
44
Clark Boylanb640e052014-04-03 16:41:46 -070045
46import git
47import gear
48import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080049import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080050import kazoo.exceptions
Joshua Heskethd78b4482015-09-14 16:56:34 -060051import pymysql
Clark Boylanb640e052014-04-03 16:41:46 -070052import statsd
53import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080054import testtools.content
55import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080056from git.exc import NoSuchPathError
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +000057import yaml
Clark Boylanb640e052014-04-03 16:41:46 -070058
James E. Blaire511d2f2016-12-08 15:22:26 -080059import zuul.driver.gerrit.gerritsource as gerritsource
60import zuul.driver.gerrit.gerritconnection as gerritconnection
Gregory Haynes4fc12542015-04-22 20:38:06 -070061import zuul.driver.github.githubconnection as githubconnection
Clark Boylanb640e052014-04-03 16:41:46 -070062import zuul.scheduler
63import zuul.webapp
64import zuul.rpclistener
Paul Belanger174a8272017-03-14 13:20:10 -040065import zuul.executor.server
66import zuul.executor.client
James E. Blair83005782015-12-11 14:46:03 -080067import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070068import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070069import zuul.merger.merger
70import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070071import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080072import zuul.zk
Jan Hruban49bff072015-11-03 11:45:46 +010073from zuul.exceptions import MergeFailure
Clark Boylanb640e052014-04-03 16:41:46 -070074
75FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
76 'fixtures')
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -080077
78KEEP_TEMPDIRS = bool(os.environ.get('KEEP_TEMPDIRS', False))
Clark Boylanb640e052014-04-03 16:41:46 -070079
Clark Boylanb640e052014-04-03 16:41:46 -070080
81def repack_repo(path):
82 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
83 output = subprocess.Popen(cmd, close_fds=True,
84 stdout=subprocess.PIPE,
85 stderr=subprocess.PIPE)
86 out = output.communicate()
87 if output.returncode:
88 raise Exception("git repack returned %d" % output.returncode)
89 return out
90
91
92def random_sha1():
Clint Byrumc0923d52017-05-10 15:47:41 -040093 return hashlib.sha1(str(random.random()).encode('ascii')).hexdigest()
Clark Boylanb640e052014-04-03 16:41:46 -070094
95
James E. Blaira190f3b2015-01-05 14:56:54 -080096def iterate_timeout(max_seconds, purpose):
97 start = time.time()
98 count = 0
99 while (time.time() < start + max_seconds):
100 count += 1
101 yield count
102 time.sleep(0)
103 raise Exception("Timeout waiting for %s" % purpose)
104
105
Jesse Keating436a5452017-04-20 11:48:41 -0700106def simple_layout(path, driver='gerrit'):
James E. Blair06cc3922017-04-19 10:08:10 -0700107 """Specify a layout file for use by a test method.
108
109 :arg str path: The path to the layout file.
Jesse Keating436a5452017-04-20 11:48:41 -0700110 :arg str driver: The source driver to use, defaults to gerrit.
James E. Blair06cc3922017-04-19 10:08:10 -0700111
112 Some tests require only a very simple configuration. For those,
113 establishing a complete config directory hierachy is too much
114 work. In those cases, you can add a simple zuul.yaml file to the
115 test fixtures directory (in fixtures/layouts/foo.yaml) and use
116 this decorator to indicate the test method should use that rather
117 than the tenant config file specified by the test class.
118
119 The decorator will cause that layout file to be added to a
120 config-project called "common-config" and each "project" instance
121 referenced in the layout file will have a git repo automatically
122 initialized.
123 """
124
125 def decorator(test):
Jesse Keating436a5452017-04-20 11:48:41 -0700126 test.__simple_layout__ = (path, driver)
James E. Blair06cc3922017-04-19 10:08:10 -0700127 return test
128 return decorator
129
130
Gregory Haynes4fc12542015-04-22 20:38:06 -0700131class GerritChangeReference(git.Reference):
Clark Boylanb640e052014-04-03 16:41:46 -0700132 _common_path_default = "refs/changes"
133 _points_to_commits_only = True
134
135
Gregory Haynes4fc12542015-04-22 20:38:06 -0700136class FakeGerritChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700137 categories = {'approved': ('Approved', -1, 1),
138 'code-review': ('Code-Review', -2, 2),
139 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700140
141 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700142 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700143 self.gerrit = gerrit
Gregory Haynes4fc12542015-04-22 20:38:06 -0700144 self.source = gerrit
Clark Boylanb640e052014-04-03 16:41:46 -0700145 self.reported = 0
146 self.queried = 0
147 self.patchsets = []
148 self.number = number
149 self.project = project
150 self.branch = branch
151 self.subject = subject
152 self.latest_patchset = 0
153 self.depends_on_change = None
154 self.needed_by_changes = []
155 self.fail_merge = False
156 self.messages = []
157 self.data = {
158 'branch': branch,
159 'comments': [],
160 'commitMessage': subject,
161 'createdOn': time.time(),
162 'id': 'I' + random_sha1(),
163 'lastUpdated': time.time(),
164 'number': str(number),
165 'open': status == 'NEW',
166 'owner': {'email': 'user@example.com',
167 'name': 'User Name',
168 'username': 'username'},
169 'patchSets': self.patchsets,
170 'project': project,
171 'status': status,
172 'subject': subject,
173 'submitRecords': [],
174 'url': 'https://hostname/%s' % number}
175
176 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700177 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700178 self.data['submitRecords'] = self.getSubmitRecords()
179 self.open = status == 'NEW'
180
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700181 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700182 path = os.path.join(self.upstream_root, self.project)
183 repo = git.Repo(path)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700184 ref = GerritChangeReference.create(
185 repo, '1/%s/%s' % (self.number, self.latest_patchset),
186 'refs/tags/init')
Clark Boylanb640e052014-04-03 16:41:46 -0700187 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700188 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700189 repo.git.clean('-x', '-f', '-d')
190
191 path = os.path.join(self.upstream_root, self.project)
192 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700193 for fn, content in files.items():
194 fn = os.path.join(path, fn)
195 with open(fn, 'w') as f:
196 f.write(content)
197 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700198 else:
199 for fni in range(100):
200 fn = os.path.join(path, str(fni))
201 f = open(fn, 'w')
202 for ci in range(4096):
203 f.write(random.choice(string.printable))
204 f.close()
205 repo.index.add([fn])
206
207 r = repo.index.commit(msg)
208 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700209 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700210 repo.git.clean('-x', '-f', '-d')
211 repo.heads['master'].checkout()
212 return r
213
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700214 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700215 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700216 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700217 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700218 data = ("test %s %s %s\n" %
219 (self.branch, self.number, self.latest_patchset))
220 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700221 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700222 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700223 ps_files = [{'file': '/COMMIT_MSG',
224 'type': 'ADDED'},
225 {'file': 'README',
226 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700227 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700228 ps_files.append({'file': f, 'type': 'ADDED'})
229 d = {'approvals': [],
230 'createdOn': time.time(),
231 'files': ps_files,
232 'number': str(self.latest_patchset),
233 'ref': 'refs/changes/1/%s/%s' % (self.number,
234 self.latest_patchset),
235 'revision': c.hexsha,
236 'uploader': {'email': 'user@example.com',
237 'name': 'User name',
238 'username': 'user'}}
239 self.data['currentPatchSet'] = d
240 self.patchsets.append(d)
241 self.data['submitRecords'] = self.getSubmitRecords()
242
243 def getPatchsetCreatedEvent(self, patchset):
244 event = {"type": "patchset-created",
245 "change": {"project": self.project,
246 "branch": self.branch,
247 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
248 "number": str(self.number),
249 "subject": self.subject,
250 "owner": {"name": "User Name"},
251 "url": "https://hostname/3"},
252 "patchSet": self.patchsets[patchset - 1],
253 "uploader": {"name": "User Name"}}
254 return event
255
256 def getChangeRestoredEvent(self):
257 event = {"type": "change-restored",
258 "change": {"project": self.project,
259 "branch": self.branch,
260 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
261 "number": str(self.number),
262 "subject": self.subject,
263 "owner": {"name": "User Name"},
264 "url": "https://hostname/3"},
265 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100266 "patchSet": self.patchsets[-1],
267 "reason": ""}
268 return event
269
270 def getChangeAbandonedEvent(self):
271 event = {"type": "change-abandoned",
272 "change": {"project": self.project,
273 "branch": self.branch,
274 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
275 "number": str(self.number),
276 "subject": self.subject,
277 "owner": {"name": "User Name"},
278 "url": "https://hostname/3"},
279 "abandoner": {"name": "User Name"},
280 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700281 "reason": ""}
282 return event
283
284 def getChangeCommentEvent(self, patchset):
285 event = {"type": "comment-added",
286 "change": {"project": self.project,
287 "branch": self.branch,
288 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
289 "number": str(self.number),
290 "subject": self.subject,
291 "owner": {"name": "User Name"},
292 "url": "https://hostname/3"},
293 "patchSet": self.patchsets[patchset - 1],
294 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700295 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700296 "description": "Code-Review",
297 "value": "0"}],
298 "comment": "This is a comment"}
299 return event
300
James E. Blairc2a5ed72017-02-20 14:12:01 -0500301 def getChangeMergedEvent(self):
302 event = {"submitter": {"name": "Jenkins",
303 "username": "jenkins"},
304 "newRev": "29ed3b5f8f750a225c5be70235230e3a6ccb04d9",
305 "patchSet": self.patchsets[-1],
306 "change": self.data,
307 "type": "change-merged",
308 "eventCreatedOn": 1487613810}
309 return event
310
James E. Blair8cce42e2016-10-18 08:18:36 -0700311 def getRefUpdatedEvent(self):
312 path = os.path.join(self.upstream_root, self.project)
313 repo = git.Repo(path)
314 oldrev = repo.heads[self.branch].commit.hexsha
315
316 event = {
317 "type": "ref-updated",
318 "submitter": {
319 "name": "User Name",
320 },
321 "refUpdate": {
322 "oldRev": oldrev,
323 "newRev": self.patchsets[-1]['revision'],
324 "refName": self.branch,
325 "project": self.project,
326 }
327 }
328 return event
329
Joshua Hesketh642824b2014-07-01 17:54:59 +1000330 def addApproval(self, category, value, username='reviewer_john',
331 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700332 if not granted_on:
333 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000334 approval = {
335 'description': self.categories[category][0],
336 'type': category,
337 'value': str(value),
338 'by': {
339 'username': username,
340 'email': username + '@example.com',
341 },
342 'grantedOn': int(granted_on)
343 }
Clark Boylanb640e052014-04-03 16:41:46 -0700344 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
345 if x['by']['username'] == username and x['type'] == category:
346 del self.patchsets[-1]['approvals'][i]
347 self.patchsets[-1]['approvals'].append(approval)
348 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000349 'author': {'email': 'author@example.com',
350 'name': 'Patchset Author',
351 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700352 'change': {'branch': self.branch,
353 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
354 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000355 'owner': {'email': 'owner@example.com',
356 'name': 'Change Owner',
357 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700358 'project': self.project,
359 'subject': self.subject,
360 'topic': 'master',
361 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000362 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700363 'patchSet': self.patchsets[-1],
364 'type': 'comment-added'}
365 self.data['submitRecords'] = self.getSubmitRecords()
366 return json.loads(json.dumps(event))
367
368 def getSubmitRecords(self):
369 status = {}
370 for cat in self.categories.keys():
371 status[cat] = 0
372
373 for a in self.patchsets[-1]['approvals']:
374 cur = status[a['type']]
375 cat_min, cat_max = self.categories[a['type']][1:]
376 new = int(a['value'])
377 if new == cat_min:
378 cur = new
379 elif abs(new) > abs(cur):
380 cur = new
381 status[a['type']] = cur
382
383 labels = []
384 ok = True
385 for typ, cat in self.categories.items():
386 cur = status[typ]
387 cat_min, cat_max = cat[1:]
388 if cur == cat_min:
389 value = 'REJECT'
390 ok = False
391 elif cur == cat_max:
392 value = 'OK'
393 else:
394 value = 'NEED'
395 ok = False
396 labels.append({'label': cat[0], 'status': value})
397 if ok:
398 return [{'status': 'OK'}]
399 return [{'status': 'NOT_READY',
400 'labels': labels}]
401
402 def setDependsOn(self, other, patchset):
403 self.depends_on_change = other
404 d = {'id': other.data['id'],
405 'number': other.data['number'],
406 'ref': other.patchsets[patchset - 1]['ref']
407 }
408 self.data['dependsOn'] = [d]
409
410 other.needed_by_changes.append(self)
411 needed = other.data.get('neededBy', [])
412 d = {'id': self.data['id'],
413 'number': self.data['number'],
414 'ref': self.patchsets[patchset - 1]['ref'],
415 'revision': self.patchsets[patchset - 1]['revision']
416 }
417 needed.append(d)
418 other.data['neededBy'] = needed
419
420 def query(self):
421 self.queried += 1
422 d = self.data.get('dependsOn')
423 if d:
424 d = d[0]
425 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
426 d['isCurrentPatchSet'] = True
427 else:
428 d['isCurrentPatchSet'] = False
429 return json.loads(json.dumps(self.data))
430
431 def setMerged(self):
432 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000433 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700434 return
435 if self.fail_merge:
436 return
437 self.data['status'] = 'MERGED'
438 self.open = False
439
440 path = os.path.join(self.upstream_root, self.project)
441 repo = git.Repo(path)
442 repo.heads[self.branch].commit = \
443 repo.commit(self.patchsets[-1]['revision'])
444
445 def setReported(self):
446 self.reported += 1
447
448
James E. Blaire511d2f2016-12-08 15:22:26 -0800449class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700450 """A Fake Gerrit connection for use in tests.
451
452 This subclasses
453 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
454 ability for tests to add changes to the fake Gerrit it represents.
455 """
456
Joshua Hesketh352264b2015-08-11 23:42:08 +1000457 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700458
James E. Blaire511d2f2016-12-08 15:22:26 -0800459 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700460 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800461 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000462 connection_config)
463
James E. Blair7fc8daa2016-08-08 15:37:15 -0700464 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700465 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
466 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000467 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700468 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200469 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700470
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700471 def addFakeChange(self, project, branch, subject, status='NEW',
472 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700473 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700474 self.change_number += 1
Gregory Haynes4fc12542015-04-22 20:38:06 -0700475 c = FakeGerritChange(self, self.change_number, project, branch,
476 subject, upstream_root=self.upstream_root,
477 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700478 self.changes[self.change_number] = c
479 return c
480
Clark Boylanb640e052014-04-03 16:41:46 -0700481 def review(self, project, changeid, message, action):
482 number, ps = changeid.split(',')
483 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000484
485 # Add the approval back onto the change (ie simulate what gerrit would
486 # do).
487 # Usually when zuul leaves a review it'll create a feedback loop where
488 # zuul's review enters another gerrit event (which is then picked up by
489 # zuul). However, we can't mimic this behaviour (by adding this
490 # approval event into the queue) as it stops jobs from checking what
491 # happens before this event is triggered. If a job needs to see what
492 # happens they can add their own verified event into the queue.
493 # Nevertheless, we can update change with the new review in gerrit.
494
James E. Blair8b5408c2016-08-08 15:37:46 -0700495 for cat in action.keys():
496 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000497 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000498
Clark Boylanb640e052014-04-03 16:41:46 -0700499 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000500
Clark Boylanb640e052014-04-03 16:41:46 -0700501 if 'submit' in action:
502 change.setMerged()
503 if message:
504 change.setReported()
505
506 def query(self, number):
507 change = self.changes.get(int(number))
508 if change:
509 return change.query()
510 return {}
511
James E. Blairc494d542014-08-06 09:23:52 -0700512 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700513 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700514 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800515 if query.startswith('change:'):
516 # Query a specific changeid
517 changeid = query[len('change:'):]
518 l = [change.query() for change in self.changes.values()
519 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700520 elif query.startswith('message:'):
521 # Query the content of a commit message
522 msg = query[len('message:'):].strip()
523 l = [change.query() for change in self.changes.values()
524 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800525 else:
526 # Query all open changes
527 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700528 return l
James E. Blairc494d542014-08-06 09:23:52 -0700529
Joshua Hesketh352264b2015-08-11 23:42:08 +1000530 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700531 pass
532
Joshua Hesketh352264b2015-08-11 23:42:08 +1000533 def getGitUrl(self, project):
534 return os.path.join(self.upstream_root, project.name)
535
Clark Boylanb640e052014-04-03 16:41:46 -0700536
Gregory Haynes4fc12542015-04-22 20:38:06 -0700537class GithubChangeReference(git.Reference):
538 _common_path_default = "refs/pull"
539 _points_to_commits_only = True
540
541
542class FakeGithubPullRequest(object):
543
544 def __init__(self, github, number, project, branch,
Jan Hruban570d01c2016-03-10 21:51:32 +0100545 subject, upstream_root, files=[], number_of_commits=1):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700546 """Creates a new PR with several commits.
547 Sends an event about opened PR."""
548 self.github = github
549 self.source = github
550 self.number = number
551 self.project = project
552 self.branch = branch
Jan Hruban37615e52015-11-19 14:30:49 +0100553 self.subject = subject
554 self.number_of_commits = 0
Gregory Haynes4fc12542015-04-22 20:38:06 -0700555 self.upstream_root = upstream_root
Jan Hruban570d01c2016-03-10 21:51:32 +0100556 self.files = []
Gregory Haynes4fc12542015-04-22 20:38:06 -0700557 self.comments = []
Jan Hruban16ad31f2015-11-07 14:39:07 +0100558 self.labels = []
Jan Hrubane252a732017-01-03 15:03:09 +0100559 self.statuses = {}
Gregory Haynes4fc12542015-04-22 20:38:06 -0700560 self.updated_at = None
561 self.head_sha = None
Jan Hruban49bff072015-11-03 11:45:46 +0100562 self.is_merged = False
Jan Hruban3b415922016-02-03 13:10:22 +0100563 self.merge_message = None
Gregory Haynes4fc12542015-04-22 20:38:06 -0700564 self._createPRRef()
Jan Hruban570d01c2016-03-10 21:51:32 +0100565 self._addCommitToRepo(files=files)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700566 self._updateTimeStamp()
567
Jan Hruban570d01c2016-03-10 21:51:32 +0100568 def addCommit(self, files=[]):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700569 """Adds a commit on top of the actual PR head."""
Jan Hruban570d01c2016-03-10 21:51:32 +0100570 self._addCommitToRepo(files=files)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700571 self._updateTimeStamp()
Jan Hrubane252a732017-01-03 15:03:09 +0100572 self._clearStatuses()
Gregory Haynes4fc12542015-04-22 20:38:06 -0700573
Jan Hruban570d01c2016-03-10 21:51:32 +0100574 def forcePush(self, files=[]):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700575 """Clears actual commits and add a commit on top of the base."""
Jan Hruban570d01c2016-03-10 21:51:32 +0100576 self._addCommitToRepo(files=files, reset=True)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700577 self._updateTimeStamp()
Jan Hrubane252a732017-01-03 15:03:09 +0100578 self._clearStatuses()
Gregory Haynes4fc12542015-04-22 20:38:06 -0700579
580 def getPullRequestOpenedEvent(self):
581 return self._getPullRequestEvent('opened')
582
583 def getPullRequestSynchronizeEvent(self):
584 return self._getPullRequestEvent('synchronize')
585
586 def getPullRequestReopenedEvent(self):
587 return self._getPullRequestEvent('reopened')
588
589 def getPullRequestClosedEvent(self):
590 return self._getPullRequestEvent('closed')
591
592 def addComment(self, message):
593 self.comments.append(message)
594 self._updateTimeStamp()
595
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200596 def getCommentAddedEvent(self, text):
597 name = 'issue_comment'
598 data = {
599 'action': 'created',
600 'issue': {
601 'number': self.number
602 },
603 'comment': {
604 'body': text
605 },
606 'repository': {
607 'full_name': self.project
Jan Hruban3b415922016-02-03 13:10:22 +0100608 },
609 'sender': {
610 'login': 'ghuser'
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200611 }
612 }
613 return (name, data)
614
Jesse Keating5c05a9f2017-01-12 14:44:58 -0800615 def getReviewAddedEvent(self, review):
616 name = 'pull_request_review'
617 data = {
618 'action': 'submitted',
619 'pull_request': {
620 'number': self.number,
621 'title': self.subject,
622 'updated_at': self.updated_at,
623 'base': {
624 'ref': self.branch,
625 'repo': {
626 'full_name': self.project
627 }
628 },
629 'head': {
630 'sha': self.head_sha
631 }
632 },
633 'review': {
634 'state': review
635 },
636 'repository': {
637 'full_name': self.project
638 },
639 'sender': {
640 'login': 'ghuser'
641 }
642 }
643 return (name, data)
644
Jan Hruban16ad31f2015-11-07 14:39:07 +0100645 def addLabel(self, name):
646 if name not in self.labels:
647 self.labels.append(name)
648 self._updateTimeStamp()
649 return self._getLabelEvent(name)
650
651 def removeLabel(self, name):
652 if name in self.labels:
653 self.labels.remove(name)
654 self._updateTimeStamp()
655 return self._getUnlabelEvent(name)
656
657 def _getLabelEvent(self, label):
658 name = 'pull_request'
659 data = {
660 'action': 'labeled',
661 'pull_request': {
662 'number': self.number,
663 'updated_at': self.updated_at,
664 'base': {
665 'ref': self.branch,
666 'repo': {
667 'full_name': self.project
668 }
669 },
670 'head': {
671 'sha': self.head_sha
672 }
673 },
674 'label': {
675 'name': label
Jan Hruban3b415922016-02-03 13:10:22 +0100676 },
677 'sender': {
678 'login': 'ghuser'
Jan Hruban16ad31f2015-11-07 14:39:07 +0100679 }
680 }
681 return (name, data)
682
683 def _getUnlabelEvent(self, label):
684 name = 'pull_request'
685 data = {
686 'action': 'unlabeled',
687 'pull_request': {
688 'number': self.number,
Jan Hruban3b415922016-02-03 13:10:22 +0100689 'title': self.subject,
Jan Hruban16ad31f2015-11-07 14:39:07 +0100690 'updated_at': self.updated_at,
691 'base': {
692 'ref': self.branch,
693 'repo': {
694 'full_name': self.project
695 }
696 },
697 'head': {
698 'sha': self.head_sha
699 }
700 },
701 'label': {
702 'name': label
Jan Hruban3b415922016-02-03 13:10:22 +0100703 },
704 'sender': {
705 'login': 'ghuser'
Jan Hruban16ad31f2015-11-07 14:39:07 +0100706 }
707 }
708 return (name, data)
709
Gregory Haynes4fc12542015-04-22 20:38:06 -0700710 def _getRepo(self):
711 repo_path = os.path.join(self.upstream_root, self.project)
712 return git.Repo(repo_path)
713
714 def _createPRRef(self):
715 repo = self._getRepo()
716 GithubChangeReference.create(
717 repo, self._getPRReference(), 'refs/tags/init')
718
Jan Hruban570d01c2016-03-10 21:51:32 +0100719 def _addCommitToRepo(self, files=[], reset=False):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700720 repo = self._getRepo()
721 ref = repo.references[self._getPRReference()]
722 if reset:
Jan Hruban37615e52015-11-19 14:30:49 +0100723 self.number_of_commits = 0
Gregory Haynes4fc12542015-04-22 20:38:06 -0700724 ref.set_object('refs/tags/init')
Jan Hruban37615e52015-11-19 14:30:49 +0100725 self.number_of_commits += 1
Gregory Haynes4fc12542015-04-22 20:38:06 -0700726 repo.head.reference = ref
727 zuul.merger.merger.reset_repo_to_head(repo)
728 repo.git.clean('-x', '-f', '-d')
729
Jan Hruban570d01c2016-03-10 21:51:32 +0100730 if files:
731 fn = files[0]
732 self.files = files
733 else:
734 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
735 self.files = [fn]
Jan Hruban37615e52015-11-19 14:30:49 +0100736 msg = self.subject + '-' + str(self.number_of_commits)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700737 fn = os.path.join(repo.working_dir, fn)
738 f = open(fn, 'w')
739 with open(fn, 'w') as f:
740 f.write("test %s %s\n" %
741 (self.branch, self.number))
742 repo.index.add([fn])
743
744 self.head_sha = repo.index.commit(msg).hexsha
745 repo.head.reference = 'master'
746 zuul.merger.merger.reset_repo_to_head(repo)
747 repo.git.clean('-x', '-f', '-d')
748 repo.heads['master'].checkout()
749
750 def _updateTimeStamp(self):
751 self.updated_at = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
752
753 def getPRHeadSha(self):
754 repo = self._getRepo()
755 return repo.references[self._getPRReference()].commit.hexsha
756
Jan Hrubane252a732017-01-03 15:03:09 +0100757 def setStatus(self, state, url, description, context):
758 self.statuses[context] = {
759 'state': state,
760 'url': url,
761 'description': description
762 }
763
764 def _clearStatuses(self):
765 self.statuses = {}
766
Gregory Haynes4fc12542015-04-22 20:38:06 -0700767 def _getPRReference(self):
768 return '%s/head' % self.number
769
770 def _getPullRequestEvent(self, action):
771 name = 'pull_request'
772 data = {
773 'action': action,
774 'number': self.number,
775 'pull_request': {
776 'number': self.number,
Jan Hruban3b415922016-02-03 13:10:22 +0100777 'title': self.subject,
Gregory Haynes4fc12542015-04-22 20:38:06 -0700778 'updated_at': self.updated_at,
779 'base': {
780 'ref': self.branch,
781 'repo': {
782 'full_name': self.project
783 }
784 },
785 'head': {
786 'sha': self.head_sha
787 }
Jan Hruban3b415922016-02-03 13:10:22 +0100788 },
789 'sender': {
790 'login': 'ghuser'
Gregory Haynes4fc12542015-04-22 20:38:06 -0700791 }
792 }
793 return (name, data)
794
795
796class FakeGithubConnection(githubconnection.GithubConnection):
797 log = logging.getLogger("zuul.test.FakeGithubConnection")
798
799 def __init__(self, driver, connection_name, connection_config,
800 upstream_root=None):
801 super(FakeGithubConnection, self).__init__(driver, connection_name,
802 connection_config)
803 self.connection_name = connection_name
804 self.pr_number = 0
805 self.pull_requests = []
806 self.upstream_root = upstream_root
Jan Hruban49bff072015-11-03 11:45:46 +0100807 self.merge_failure = False
808 self.merge_not_allowed_count = 0
Gregory Haynes4fc12542015-04-22 20:38:06 -0700809
Jan Hruban570d01c2016-03-10 21:51:32 +0100810 def openFakePullRequest(self, project, branch, subject, files=[]):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700811 self.pr_number += 1
812 pull_request = FakeGithubPullRequest(
Jan Hruban570d01c2016-03-10 21:51:32 +0100813 self, self.pr_number, project, branch, subject, self.upstream_root,
814 files=files)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700815 self.pull_requests.append(pull_request)
816 return pull_request
817
Wayne1a78c612015-06-11 17:14:13 -0700818 def getPushEvent(self, project, ref, old_rev=None, new_rev=None):
819 if not old_rev:
820 old_rev = '00000000000000000000000000000000'
821 if not new_rev:
822 new_rev = random_sha1()
823 name = 'push'
824 data = {
825 'ref': ref,
826 'before': old_rev,
827 'after': new_rev,
828 'repository': {
829 'full_name': project
830 }
831 }
832 return (name, data)
833
Gregory Haynes4fc12542015-04-22 20:38:06 -0700834 def emitEvent(self, event):
835 """Emulates sending the GitHub webhook event to the connection."""
836 port = self.webapp.server.socket.getsockname()[1]
837 name, data = event
Clint Byrum607d10e2017-05-18 12:05:13 -0700838 payload = json.dumps(data).encode('utf8')
Gregory Haynes4fc12542015-04-22 20:38:06 -0700839 headers = {'X-Github-Event': name}
840 req = urllib.request.Request(
841 'http://localhost:%s/connection/%s/payload'
842 % (port, self.connection_name),
843 data=payload, headers=headers)
844 urllib.request.urlopen(req)
845
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200846 def getPull(self, project, number):
847 pr = self.pull_requests[number - 1]
848 data = {
849 'number': number,
Jan Hruban3b415922016-02-03 13:10:22 +0100850 'title': pr.subject,
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200851 'updated_at': pr.updated_at,
852 'base': {
853 'repo': {
854 'full_name': pr.project
855 },
856 'ref': pr.branch,
857 },
Jan Hruban37615e52015-11-19 14:30:49 +0100858 'mergeable': True,
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200859 'head': {
860 'sha': pr.head_sha
861 }
862 }
863 return data
864
Jan Hruban570d01c2016-03-10 21:51:32 +0100865 def getPullFileNames(self, project, number):
866 pr = self.pull_requests[number - 1]
867 return pr.files
868
Jan Hruban3b415922016-02-03 13:10:22 +0100869 def getUser(self, login):
870 data = {
871 'username': login,
872 'name': 'Github User',
873 'email': 'github.user@example.com'
874 }
875 return data
876
Gregory Haynes4fc12542015-04-22 20:38:06 -0700877 def getGitUrl(self, project):
878 return os.path.join(self.upstream_root, str(project))
879
Jan Hruban6d53c5e2015-10-24 03:03:34 +0200880 def real_getGitUrl(self, project):
881 return super(FakeGithubConnection, self).getGitUrl(project)
882
Gregory Haynes4fc12542015-04-22 20:38:06 -0700883 def getProjectBranches(self, project):
884 """Masks getProjectBranches since we don't have a real github"""
885
886 # just returns master for now
887 return ['master']
888
Jan Hrubane252a732017-01-03 15:03:09 +0100889 def commentPull(self, project, pr_number, message):
Wayne40f40042015-06-12 16:56:30 -0700890 pull_request = self.pull_requests[pr_number - 1]
891 pull_request.addComment(message)
892
Jan Hruban3b415922016-02-03 13:10:22 +0100893 def mergePull(self, project, pr_number, commit_message='', sha=None):
Jan Hruban49bff072015-11-03 11:45:46 +0100894 pull_request = self.pull_requests[pr_number - 1]
895 if self.merge_failure:
896 raise Exception('Pull request was not merged')
897 if self.merge_not_allowed_count > 0:
898 self.merge_not_allowed_count -= 1
899 raise MergeFailure('Merge was not successful due to mergeability'
900 ' conflict')
901 pull_request.is_merged = True
Jan Hruban3b415922016-02-03 13:10:22 +0100902 pull_request.merge_message = commit_message
Jan Hruban49bff072015-11-03 11:45:46 +0100903
Jan Hrubane252a732017-01-03 15:03:09 +0100904 def setCommitStatus(self, project, sha, state,
905 url='', description='', context=''):
906 owner, proj = project.split('/')
907 for pr in self.pull_requests:
908 pr_owner, pr_project = pr.project.split('/')
909 if (pr_owner == owner and pr_project == proj and
910 pr.head_sha == sha):
911 pr.setStatus(state, url, description, context)
912
Jan Hruban16ad31f2015-11-07 14:39:07 +0100913 def labelPull(self, project, pr_number, label):
914 pull_request = self.pull_requests[pr_number - 1]
915 pull_request.addLabel(label)
916
917 def unlabelPull(self, project, pr_number, label):
918 pull_request = self.pull_requests[pr_number - 1]
919 pull_request.removeLabel(label)
920
Gregory Haynes4fc12542015-04-22 20:38:06 -0700921
Clark Boylanb640e052014-04-03 16:41:46 -0700922class BuildHistory(object):
923 def __init__(self, **kw):
924 self.__dict__.update(kw)
925
926 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -0700927 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
928 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -0700929
930
931class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200932 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700933 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700934 self.url = url
935
936 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -0700937 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -0700938 path = res.path
939 project = '/'.join(path.split('/')[2:-2])
940 ret = '001e# service=git-upload-pack\n'
941 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
942 'multi_ack thin-pack side-band side-band-64k ofs-delta '
943 'shallow no-progress include-tag multi_ack_detailed no-done\n')
944 path = os.path.join(self.upstream_root, project)
945 repo = git.Repo(path)
946 for ref in repo.refs:
947 r = ref.object.hexsha + ' ' + ref.path + '\n'
948 ret += '%04x%s' % (len(r) + 4, r)
949 ret += '0000'
950 return ret
951
952
Clark Boylanb640e052014-04-03 16:41:46 -0700953class FakeStatsd(threading.Thread):
954 def __init__(self):
955 threading.Thread.__init__(self)
956 self.daemon = True
957 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
958 self.sock.bind(('', 0))
959 self.port = self.sock.getsockname()[1]
960 self.wake_read, self.wake_write = os.pipe()
961 self.stats = []
962
963 def run(self):
964 while True:
965 poll = select.poll()
966 poll.register(self.sock, select.POLLIN)
967 poll.register(self.wake_read, select.POLLIN)
968 ret = poll.poll()
969 for (fd, event) in ret:
970 if fd == self.sock.fileno():
971 data = self.sock.recvfrom(1024)
972 if not data:
973 return
974 self.stats.append(data[0])
975 if fd == self.wake_read:
976 return
977
978 def stop(self):
Clint Byrumf322fe22017-05-10 20:53:12 -0700979 os.write(self.wake_write, b'1\n')
Clark Boylanb640e052014-04-03 16:41:46 -0700980
981
James E. Blaire1767bc2016-08-02 10:00:27 -0700982class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -0700983 log = logging.getLogger("zuul.test")
984
Paul Belanger174a8272017-03-14 13:20:10 -0400985 def __init__(self, executor_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -0700986 self.daemon = True
Paul Belanger174a8272017-03-14 13:20:10 -0400987 self.executor_server = executor_server
Clark Boylanb640e052014-04-03 16:41:46 -0700988 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -0700989 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -0700990 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -0700991 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -0700992 # TODOv3(jeblair): self.node is really "the image of the node
993 # assigned". We should rename it (self.node_image?) if we
994 # keep using it like this, or we may end up exposing more of
995 # the complexity around multi-node jobs here
996 # (self.nodes[0].image?)
997 self.node = None
998 if len(self.parameters.get('nodes')) == 1:
999 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -07001000 self.unique = self.parameters['ZUUL_UUID']
Jamie Lennoxd2e37332016-12-05 15:26:19 +11001001 self.pipeline = self.parameters['ZUUL_PIPELINE']
1002 self.project = self.parameters['ZUUL_PROJECT']
James E. Blair3f876d52016-07-22 13:07:14 -07001003 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -07001004 self.wait_condition = threading.Condition()
1005 self.waiting = False
1006 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -05001007 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -07001008 self.created = time.time()
James E. Blaire1767bc2016-08-02 10:00:27 -07001009 self.changes = None
1010 if 'ZUUL_CHANGE_IDS' in self.parameters:
1011 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -07001012
James E. Blair3158e282016-08-19 09:34:11 -07001013 def __repr__(self):
1014 waiting = ''
1015 if self.waiting:
1016 waiting = ' [waiting]'
Jamie Lennoxd2e37332016-12-05 15:26:19 +11001017 return '<FakeBuild %s:%s %s%s>' % (self.pipeline, self.name,
1018 self.changes, waiting)
James E. Blair3158e282016-08-19 09:34:11 -07001019
Clark Boylanb640e052014-04-03 16:41:46 -07001020 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001021 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -07001022 self.wait_condition.acquire()
1023 self.wait_condition.notify()
1024 self.waiting = False
1025 self.log.debug("Build %s released" % self.unique)
1026 self.wait_condition.release()
1027
1028 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001029 """Return whether this build is being held.
1030
1031 :returns: Whether the build is being held.
1032 :rtype: bool
1033 """
1034
Clark Boylanb640e052014-04-03 16:41:46 -07001035 self.wait_condition.acquire()
1036 if self.waiting:
1037 ret = True
1038 else:
1039 ret = False
1040 self.wait_condition.release()
1041 return ret
1042
1043 def _wait(self):
1044 self.wait_condition.acquire()
1045 self.waiting = True
1046 self.log.debug("Build %s waiting" % self.unique)
1047 self.wait_condition.wait()
1048 self.wait_condition.release()
1049
1050 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001051 self.log.debug('Running build %s' % self.unique)
1052
Paul Belanger174a8272017-03-14 13:20:10 -04001053 if self.executor_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -07001054 self.log.debug('Holding build %s' % self.unique)
1055 self._wait()
1056 self.log.debug("Build %s continuing" % self.unique)
1057
James E. Blair412fba82017-01-26 15:00:50 -08001058 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
James E. Blaira5dba232016-08-08 15:53:24 -07001059 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
James E. Blair412fba82017-01-26 15:00:50 -08001060 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
Clark Boylanb640e052014-04-03 16:41:46 -07001061 if self.aborted:
James E. Blair412fba82017-01-26 15:00:50 -08001062 result = (RecordingAnsibleJob.RESULT_ABORTED, None)
Paul Belanger71d98172016-11-08 10:56:31 -05001063 if self.requeue:
James E. Blair412fba82017-01-26 15:00:50 -08001064 result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
Clark Boylanb640e052014-04-03 16:41:46 -07001065
James E. Blaire1767bc2016-08-02 10:00:27 -07001066 return result
Clark Boylanb640e052014-04-03 16:41:46 -07001067
James E. Blaira5dba232016-08-08 15:53:24 -07001068 def shouldFail(self):
Paul Belanger174a8272017-03-14 13:20:10 -04001069 changes = self.executor_server.fail_tests.get(self.name, [])
James E. Blaira5dba232016-08-08 15:53:24 -07001070 for change in changes:
1071 if self.hasChanges(change):
1072 return True
1073 return False
1074
James E. Blaire7b99a02016-08-05 14:27:34 -07001075 def hasChanges(self, *changes):
1076 """Return whether this build has certain changes in its git repos.
1077
1078 :arg FakeChange changes: One or more changes (varargs) that
Clark Boylan500992b2017-04-03 14:28:24 -07001079 are expected to be present (in order) in the git repository of
1080 the active project.
James E. Blaire7b99a02016-08-05 14:27:34 -07001081
1082 :returns: Whether the build has the indicated changes.
1083 :rtype: bool
1084
1085 """
Clint Byrum3343e3e2016-11-15 16:05:03 -08001086 for change in changes:
Gregory Haynes4fc12542015-04-22 20:38:06 -07001087 hostname = change.source.canonical_hostname
James E. Blair2a535672017-04-27 12:03:15 -07001088 path = os.path.join(self.jobdir.src_root, hostname, change.project)
Clint Byrum3343e3e2016-11-15 16:05:03 -08001089 try:
1090 repo = git.Repo(path)
1091 except NoSuchPathError as e:
1092 self.log.debug('%s' % e)
1093 return False
1094 ref = self.parameters['ZUUL_REF']
1095 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
1096 commit_message = '%s-1' % change.subject
1097 self.log.debug("Checking if build %s has changes; commit_message "
1098 "%s; repo_messages %s" % (self, commit_message,
1099 repo_messages))
1100 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -07001101 self.log.debug(" messages do not match")
1102 return False
1103 self.log.debug(" OK")
1104 return True
1105
Clark Boylanb640e052014-04-03 16:41:46 -07001106
Paul Belanger174a8272017-03-14 13:20:10 -04001107class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1108 """An Ansible executor to be used in tests.
James E. Blaire7b99a02016-08-05 14:27:34 -07001109
Paul Belanger174a8272017-03-14 13:20:10 -04001110 :ivar bool hold_jobs_in_build: If true, when jobs are executed
James E. Blaire7b99a02016-08-05 14:27:34 -07001111 they will report that they have started but then pause until
1112 released before reporting completion. This attribute may be
1113 changed at any time and will take effect for subsequently
Paul Belanger174a8272017-03-14 13:20:10 -04001114 executed builds, but previously held builds will still need to
James E. Blaire7b99a02016-08-05 14:27:34 -07001115 be explicitly released.
1116
1117 """
James E. Blairf5dbd002015-12-23 15:26:17 -08001118 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -07001119 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -08001120 self._test_root = kw.pop('_test_root', False)
Paul Belanger174a8272017-03-14 13:20:10 -04001121 super(RecordingExecutorServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -07001122 self.hold_jobs_in_build = False
1123 self.lock = threading.Lock()
1124 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -07001125 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -07001126 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -07001127 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -08001128
James E. Blaira5dba232016-08-08 15:53:24 -07001129 def failJob(self, name, change):
Paul Belanger174a8272017-03-14 13:20:10 -04001130 """Instruct the executor to report matching builds as failures.
James E. Blaire7b99a02016-08-05 14:27:34 -07001131
1132 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -07001133 :arg Change change: The :py:class:`~tests.base.FakeChange`
1134 instance which should cause the job to fail. This job
1135 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -07001136
1137 """
James E. Blaire1767bc2016-08-02 10:00:27 -07001138 l = self.fail_tests.get(name, [])
1139 l.append(change)
1140 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -08001141
James E. Blair962220f2016-08-03 11:22:38 -07001142 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -07001143 """Release a held build.
1144
1145 :arg str regex: A regular expression which, if supplied, will
1146 cause only builds with matching names to be released. If
1147 not supplied, all builds will be released.
1148
1149 """
James E. Blair962220f2016-08-03 11:22:38 -07001150 builds = self.running_builds[:]
1151 self.log.debug("Releasing build %s (%s)" % (regex,
1152 len(self.running_builds)))
1153 for build in builds:
1154 if not regex or re.match(regex, build.name):
1155 self.log.debug("Releasing build %s" %
1156 (build.parameters['ZUUL_UUID']))
1157 build.release()
1158 else:
1159 self.log.debug("Not releasing build %s" %
1160 (build.parameters['ZUUL_UUID']))
1161 self.log.debug("Done releasing builds %s (%s)" %
1162 (regex, len(self.running_builds)))
1163
Paul Belanger174a8272017-03-14 13:20:10 -04001164 def executeJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -07001165 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -07001166 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -07001167 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -07001168 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -08001169 args = json.loads(job.arguments)
James E. Blair490cf042017-02-24 23:07:21 -05001170 args['vars']['zuul']['_test'] = dict(test_root=self._test_root)
James E. Blaira92cbc82017-01-23 14:56:49 -08001171 job.arguments = json.dumps(args)
Joshua Hesketh50c21782016-10-13 21:34:14 +11001172 self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
1173 self.job_workers[job.unique].run()
James E. Blair17302972016-08-10 16:11:42 -07001174
1175 def stopJob(self, job):
1176 self.log.debug("handle stop")
1177 parameters = json.loads(job.arguments)
1178 uuid = parameters['uuid']
1179 for build in self.running_builds:
1180 if build.unique == uuid:
1181 build.aborted = True
1182 build.release()
Paul Belanger174a8272017-03-14 13:20:10 -04001183 super(RecordingExecutorServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -07001184
James E. Blaira002b032017-04-18 10:35:48 -07001185 def stop(self):
1186 for build in self.running_builds:
1187 build.release()
1188 super(RecordingExecutorServer, self).stop()
1189
Joshua Hesketh50c21782016-10-13 21:34:14 +11001190
Paul Belanger174a8272017-03-14 13:20:10 -04001191class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
James E. Blair1960d682017-04-28 15:44:14 -07001192 def doMergeChanges(self, items, repo_state):
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001193 # Get a merger in order to update the repos involved in this job.
James E. Blair1960d682017-04-28 15:44:14 -07001194 commit = super(RecordingAnsibleJob, self).doMergeChanges(
1195 items, repo_state)
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001196 if not commit: # merge conflict
1197 self.recordResult('MERGER_FAILURE')
1198 return commit
1199
1200 def recordResult(self, result):
Paul Belanger174a8272017-03-14 13:20:10 -04001201 build = self.executor_server.job_builds[self.job.unique]
Paul Belanger174a8272017-03-14 13:20:10 -04001202 self.executor_server.lock.acquire()
1203 self.executor_server.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -07001204 BuildHistory(name=build.name, result=result, changes=build.changes,
1205 node=build.node, uuid=build.unique,
K Jonathan Harker2c1a6232017-02-21 14:34:08 -08001206 parameters=build.parameters, jobdir=build.jobdir,
James E. Blaire1767bc2016-08-02 10:00:27 -07001207 pipeline=build.parameters['ZUUL_PIPELINE'])
1208 )
Paul Belanger174a8272017-03-14 13:20:10 -04001209 self.executor_server.running_builds.remove(build)
1210 del self.executor_server.job_builds[self.job.unique]
1211 self.executor_server.lock.release()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001212
1213 def runPlaybooks(self, args):
1214 build = self.executor_server.job_builds[self.job.unique]
1215 build.jobdir = self.jobdir
1216
1217 result = super(RecordingAnsibleJob, self).runPlaybooks(args)
1218 self.recordResult(result)
James E. Blair412fba82017-01-26 15:00:50 -08001219 return result
1220
Monty Taylore6562aa2017-02-20 07:37:39 -05001221 def runAnsible(self, cmd, timeout, trusted=False):
Paul Belanger174a8272017-03-14 13:20:10 -04001222 build = self.executor_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -08001223
Paul Belanger174a8272017-03-14 13:20:10 -04001224 if self.executor_server._run_ansible:
Monty Taylorc231d932017-02-03 09:57:15 -06001225 result = super(RecordingAnsibleJob, self).runAnsible(
Monty Taylore6562aa2017-02-20 07:37:39 -05001226 cmd, timeout, trusted=trusted)
James E. Blair412fba82017-01-26 15:00:50 -08001227 else:
1228 result = build.run()
James E. Blaire1767bc2016-08-02 10:00:27 -07001229 return result
James E. Blairf5dbd002015-12-23 15:26:17 -08001230
James E. Blairad8dca02017-02-21 11:48:32 -05001231 def getHostList(self, args):
1232 self.log.debug("hostlist")
1233 hosts = super(RecordingAnsibleJob, self).getHostList(args)
Paul Belangerc5bf3752017-03-16 19:38:43 -04001234 for host in hosts:
1235 host['host_vars']['ansible_connection'] = 'local'
1236
1237 hosts.append(dict(
1238 name='localhost',
1239 host_vars=dict(ansible_connection='local'),
1240 host_keys=[]))
James E. Blairad8dca02017-02-21 11:48:32 -05001241 return hosts
1242
James E. Blairf5dbd002015-12-23 15:26:17 -08001243
Clark Boylanb640e052014-04-03 16:41:46 -07001244class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -07001245 """A Gearman server for use in tests.
1246
1247 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
1248 added to the queue but will not be distributed to workers
1249 until released. This attribute may be changed at any time and
1250 will take effect for subsequently enqueued jobs, but
1251 previously held jobs will still need to be explicitly
1252 released.
1253
1254 """
1255
Clark Boylanb640e052014-04-03 16:41:46 -07001256 def __init__(self):
1257 self.hold_jobs_in_queue = False
1258 super(FakeGearmanServer, self).__init__(0)
1259
1260 def getJobForConnection(self, connection, peek=False):
1261 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
1262 for job in queue:
1263 if not hasattr(job, 'waiting'):
Clint Byrumf322fe22017-05-10 20:53:12 -07001264 if job.name.startswith(b'executor:execute'):
Clark Boylanb640e052014-04-03 16:41:46 -07001265 job.waiting = self.hold_jobs_in_queue
1266 else:
1267 job.waiting = False
1268 if job.waiting:
1269 continue
1270 if job.name in connection.functions:
1271 if not peek:
1272 queue.remove(job)
1273 connection.related_jobs[job.handle] = job
1274 job.worker_connection = connection
1275 job.running = True
1276 return job
1277 return None
1278
1279 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -07001280 """Release a held job.
1281
1282 :arg str regex: A regular expression which, if supplied, will
1283 cause only jobs with matching names to be released. If
1284 not supplied, all jobs will be released.
1285 """
Clark Boylanb640e052014-04-03 16:41:46 -07001286 released = False
1287 qlen = (len(self.high_queue) + len(self.normal_queue) +
1288 len(self.low_queue))
1289 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
1290 for job in self.getQueue():
Paul Belanger174a8272017-03-14 13:20:10 -04001291 if job.name != 'executor:execute':
Clark Boylanb640e052014-04-03 16:41:46 -07001292 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -05001293 parameters = json.loads(job.arguments)
1294 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -07001295 self.log.debug("releasing queued job %s" %
1296 job.unique)
1297 job.waiting = False
1298 released = True
1299 else:
1300 self.log.debug("not releasing queued job %s" %
1301 job.unique)
1302 if released:
1303 self.wakeConnections()
1304 qlen = (len(self.high_queue) + len(self.normal_queue) +
1305 len(self.low_queue))
1306 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
1307
1308
1309class FakeSMTP(object):
1310 log = logging.getLogger('zuul.FakeSMTP')
1311
1312 def __init__(self, messages, server, port):
1313 self.server = server
1314 self.port = port
1315 self.messages = messages
1316
1317 def sendmail(self, from_email, to_email, msg):
1318 self.log.info("Sending email from %s, to %s, with msg %s" % (
1319 from_email, to_email, msg))
1320
1321 headers = msg.split('\n\n', 1)[0]
1322 body = msg.split('\n\n', 1)[1]
1323
1324 self.messages.append(dict(
1325 from_email=from_email,
1326 to_email=to_email,
1327 msg=msg,
1328 headers=headers,
1329 body=body,
1330 ))
1331
1332 return True
1333
1334 def quit(self):
1335 return True
1336
1337
James E. Blairdce6cea2016-12-20 16:45:32 -08001338class FakeNodepool(object):
1339 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -08001340 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -08001341
1342 log = logging.getLogger("zuul.test.FakeNodepool")
1343
1344 def __init__(self, host, port, chroot):
1345 self.client = kazoo.client.KazooClient(
1346 hosts='%s:%s%s' % (host, port, chroot))
1347 self.client.start()
1348 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -08001349 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001350 self.thread = threading.Thread(target=self.run)
1351 self.thread.daemon = True
1352 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -08001353 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -08001354
1355 def stop(self):
1356 self._running = False
1357 self.thread.join()
1358 self.client.stop()
1359 self.client.close()
1360
1361 def run(self):
1362 while self._running:
James E. Blaircbbce0d2017-05-19 07:28:29 -07001363 try:
1364 self._run()
1365 except Exception:
1366 self.log.exception("Error in fake nodepool:")
James E. Blairdce6cea2016-12-20 16:45:32 -08001367 time.sleep(0.1)
1368
1369 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001370 if self.paused:
1371 return
James E. Blairdce6cea2016-12-20 16:45:32 -08001372 for req in self.getNodeRequests():
1373 self.fulfillRequest(req)
1374
1375 def getNodeRequests(self):
1376 try:
1377 reqids = self.client.get_children(self.REQUEST_ROOT)
1378 except kazoo.exceptions.NoNodeError:
1379 return []
1380 reqs = []
1381 for oid in sorted(reqids):
1382 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -08001383 try:
1384 data, stat = self.client.get(path)
Clint Byrumf322fe22017-05-10 20:53:12 -07001385 data = json.loads(data.decode('utf8'))
James E. Blair0ef64f82017-02-02 11:25:16 -08001386 data['_oid'] = oid
1387 reqs.append(data)
1388 except kazoo.exceptions.NoNodeError:
1389 pass
James E. Blairdce6cea2016-12-20 16:45:32 -08001390 return reqs
1391
James E. Blaire18d4602017-01-05 11:17:28 -08001392 def getNodes(self):
1393 try:
1394 nodeids = self.client.get_children(self.NODE_ROOT)
1395 except kazoo.exceptions.NoNodeError:
1396 return []
1397 nodes = []
1398 for oid in sorted(nodeids):
1399 path = self.NODE_ROOT + '/' + oid
1400 data, stat = self.client.get(path)
Clint Byrumf322fe22017-05-10 20:53:12 -07001401 data = json.loads(data.decode('utf8'))
James E. Blaire18d4602017-01-05 11:17:28 -08001402 data['_oid'] = oid
1403 try:
1404 lockfiles = self.client.get_children(path + '/lock')
1405 except kazoo.exceptions.NoNodeError:
1406 lockfiles = []
1407 if lockfiles:
1408 data['_lock'] = True
1409 else:
1410 data['_lock'] = False
1411 nodes.append(data)
1412 return nodes
1413
James E. Blaira38c28e2017-01-04 10:33:20 -08001414 def makeNode(self, request_id, node_type):
1415 now = time.time()
1416 path = '/nodepool/nodes/'
1417 data = dict(type=node_type,
1418 provider='test-provider',
1419 region='test-region',
Paul Belanger30ba93a2017-03-16 16:28:10 -04001420 az='test-az',
Monty Taylor56f61332017-04-11 05:38:12 -05001421 interface_ip='127.0.0.1',
James E. Blaira38c28e2017-01-04 10:33:20 -08001422 public_ipv4='127.0.0.1',
1423 private_ipv4=None,
1424 public_ipv6=None,
1425 allocated_to=request_id,
1426 state='ready',
1427 state_time=now,
1428 created_time=now,
1429 updated_time=now,
1430 image_id=None,
Paul Belangerc5bf3752017-03-16 19:38:43 -04001431 host_keys=["fake-key1", "fake-key2"],
Paul Belanger174a8272017-03-14 13:20:10 -04001432 executor='fake-nodepool')
Clint Byrumf322fe22017-05-10 20:53:12 -07001433 data = json.dumps(data).encode('utf8')
James E. Blaira38c28e2017-01-04 10:33:20 -08001434 path = self.client.create(path, data,
1435 makepath=True,
1436 sequence=True)
1437 nodeid = path.split("/")[-1]
1438 return nodeid
1439
James E. Blair6ab79e02017-01-06 10:10:17 -08001440 def addFailRequest(self, request):
1441 self.fail_requests.add(request['_oid'])
1442
James E. Blairdce6cea2016-12-20 16:45:32 -08001443 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -08001444 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -08001445 return
1446 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -08001447 oid = request['_oid']
1448 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -08001449
James E. Blair6ab79e02017-01-06 10:10:17 -08001450 if oid in self.fail_requests:
1451 request['state'] = 'failed'
1452 else:
1453 request['state'] = 'fulfilled'
1454 nodes = []
1455 for node in request['node_types']:
1456 nodeid = self.makeNode(oid, node)
1457 nodes.append(nodeid)
1458 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -08001459
James E. Blaira38c28e2017-01-04 10:33:20 -08001460 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -08001461 path = self.REQUEST_ROOT + '/' + oid
Clint Byrumf322fe22017-05-10 20:53:12 -07001462 data = json.dumps(request).encode('utf8')
James E. Blairdce6cea2016-12-20 16:45:32 -08001463 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
James E. Blaircbbce0d2017-05-19 07:28:29 -07001464 try:
1465 self.client.set(path, data)
1466 except kazoo.exceptions.NoNodeError:
1467 self.log.debug("Node request %s %s disappeared" % (oid, data))
James E. Blairdce6cea2016-12-20 16:45:32 -08001468
1469
James E. Blair498059b2016-12-20 13:50:13 -08001470class ChrootedKazooFixture(fixtures.Fixture):
Clark Boylan621ec9a2017-04-07 17:41:33 -07001471 def __init__(self, test_id):
James E. Blair498059b2016-12-20 13:50:13 -08001472 super(ChrootedKazooFixture, self).__init__()
1473
1474 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1475 if ':' in zk_host:
1476 host, port = zk_host.split(':')
1477 else:
1478 host = zk_host
1479 port = None
1480
1481 self.zookeeper_host = host
1482
1483 if not port:
1484 self.zookeeper_port = 2181
1485 else:
1486 self.zookeeper_port = int(port)
1487
Clark Boylan621ec9a2017-04-07 17:41:33 -07001488 self.test_id = test_id
1489
James E. Blair498059b2016-12-20 13:50:13 -08001490 def _setUp(self):
1491 # Make sure the test chroot paths do not conflict
1492 random_bits = ''.join(random.choice(string.ascii_lowercase +
1493 string.ascii_uppercase)
1494 for x in range(8))
1495
Clark Boylan621ec9a2017-04-07 17:41:33 -07001496 rand_test_path = '%s_%s_%s' % (random_bits, os.getpid(), self.test_id)
James E. Blair498059b2016-12-20 13:50:13 -08001497 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1498
Clark Boylan7f5f1ec2017-04-07 17:39:56 -07001499 self.addCleanup(self._cleanup)
1500
James E. Blair498059b2016-12-20 13:50:13 -08001501 # Ensure the chroot path exists and clean up any pre-existing znodes.
1502 _tmp_client = kazoo.client.KazooClient(
1503 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1504 _tmp_client.start()
1505
1506 if _tmp_client.exists(self.zookeeper_chroot):
1507 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1508
1509 _tmp_client.ensure_path(self.zookeeper_chroot)
1510 _tmp_client.stop()
1511 _tmp_client.close()
1512
James E. Blair498059b2016-12-20 13:50:13 -08001513 def _cleanup(self):
1514 '''Remove the chroot path.'''
1515 # Need a non-chroot'ed client to remove the chroot path
1516 _tmp_client = kazoo.client.KazooClient(
1517 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1518 _tmp_client.start()
1519 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1520 _tmp_client.stop()
Clark Boylan7f5f1ec2017-04-07 17:39:56 -07001521 _tmp_client.close()
James E. Blair498059b2016-12-20 13:50:13 -08001522
1523
Joshua Heskethd78b4482015-09-14 16:56:34 -06001524class MySQLSchemaFixture(fixtures.Fixture):
1525 def setUp(self):
1526 super(MySQLSchemaFixture, self).setUp()
1527
1528 random_bits = ''.join(random.choice(string.ascii_lowercase +
1529 string.ascii_uppercase)
1530 for x in range(8))
1531 self.name = '%s_%s' % (random_bits, os.getpid())
1532 self.passwd = uuid.uuid4().hex
1533 db = pymysql.connect(host="localhost",
1534 user="openstack_citest",
1535 passwd="openstack_citest",
1536 db="openstack_citest")
1537 cur = db.cursor()
1538 cur.execute("create database %s" % self.name)
1539 cur.execute(
1540 "grant all on %s.* to '%s'@'localhost' identified by '%s'" %
1541 (self.name, self.name, self.passwd))
1542 cur.execute("flush privileges")
1543
1544 self.dburi = 'mysql+pymysql://%s:%s@localhost/%s' % (self.name,
1545 self.passwd,
1546 self.name)
1547 self.addDetail('dburi', testtools.content.text_content(self.dburi))
1548 self.addCleanup(self.cleanup)
1549
1550 def cleanup(self):
1551 db = pymysql.connect(host="localhost",
1552 user="openstack_citest",
1553 passwd="openstack_citest",
1554 db="openstack_citest")
1555 cur = db.cursor()
1556 cur.execute("drop database %s" % self.name)
1557 cur.execute("drop user '%s'@'localhost'" % self.name)
1558 cur.execute("flush privileges")
1559
1560
Maru Newby3fe5f852015-01-13 04:22:14 +00001561class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001562 log = logging.getLogger("zuul.test")
James E. Blair267e5162017-04-07 10:08:20 -07001563 wait_timeout = 30
Clark Boylanb640e052014-04-03 16:41:46 -07001564
James E. Blair1c236df2017-02-01 14:07:24 -08001565 def attachLogs(self, *args):
1566 def reader():
1567 self._log_stream.seek(0)
1568 while True:
1569 x = self._log_stream.read(4096)
1570 if not x:
1571 break
1572 yield x.encode('utf8')
1573 content = testtools.content.content_from_reader(
1574 reader,
1575 testtools.content_type.UTF8_TEXT,
1576 False)
1577 self.addDetail('logging', content)
1578
Clark Boylanb640e052014-04-03 16:41:46 -07001579 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001580 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001581 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1582 try:
1583 test_timeout = int(test_timeout)
1584 except ValueError:
1585 # If timeout value is invalid do not set a timeout.
1586 test_timeout = 0
1587 if test_timeout > 0:
1588 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1589
1590 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1591 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1592 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1593 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1594 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1595 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1596 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1597 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1598 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1599 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001600 self._log_stream = StringIO()
1601 self.addOnException(self.attachLogs)
1602 else:
1603 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001604
James E. Blair1c236df2017-02-01 14:07:24 -08001605 handler = logging.StreamHandler(self._log_stream)
1606 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1607 '%(levelname)-8s %(message)s')
1608 handler.setFormatter(formatter)
1609
1610 logger = logging.getLogger()
1611 logger.setLevel(logging.DEBUG)
1612 logger.addHandler(handler)
1613
Clark Boylan3410d532017-04-25 12:35:29 -07001614 # Make sure we don't carry old handlers around in process state
1615 # which slows down test runs
1616 self.addCleanup(logger.removeHandler, handler)
1617 self.addCleanup(handler.close)
1618 self.addCleanup(handler.flush)
1619
James E. Blair1c236df2017-02-01 14:07:24 -08001620 # NOTE(notmorgan): Extract logging overrides for specific
1621 # libraries from the OS_LOG_DEFAULTS env and create loggers
1622 # for each. This is used to limit the output during test runs
1623 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001624 log_defaults_from_env = os.environ.get(
1625 'OS_LOG_DEFAULTS',
Monty Taylor73db6492017-05-18 17:31:17 -05001626 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO,paste=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001627
James E. Blairdce6cea2016-12-20 16:45:32 -08001628 if log_defaults_from_env:
1629 for default in log_defaults_from_env.split(','):
1630 try:
1631 name, level_str = default.split('=', 1)
1632 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001633 logger = logging.getLogger(name)
1634 logger.setLevel(level)
1635 logger.addHandler(handler)
1636 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001637 except ValueError:
1638 # NOTE(notmorgan): Invalid format of the log default,
1639 # skip and don't try and apply a logger for the
1640 # specified module
1641 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001642
Maru Newby3fe5f852015-01-13 04:22:14 +00001643
1644class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001645 """A test case with a functioning Zuul.
1646
1647 The following class variables are used during test setup and can
1648 be overidden by subclasses but are effectively read-only once a
1649 test method starts running:
1650
1651 :cvar str config_file: This points to the main zuul config file
1652 within the fixtures directory. Subclasses may override this
1653 to obtain a different behavior.
1654
1655 :cvar str tenant_config_file: This is the tenant config file
1656 (which specifies from what git repos the configuration should
1657 be loaded). It defaults to the value specified in
1658 `config_file` but can be overidden by subclasses to obtain a
1659 different tenant/project layout while using the standard main
James E. Blair06cc3922017-04-19 10:08:10 -07001660 configuration. See also the :py:func:`simple_layout`
1661 decorator.
James E. Blaire7b99a02016-08-05 14:27:34 -07001662
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001663 :cvar bool create_project_keys: Indicates whether Zuul should
1664 auto-generate keys for each project, or whether the test
1665 infrastructure should insert dummy keys to save time during
1666 startup. Defaults to False.
1667
James E. Blaire7b99a02016-08-05 14:27:34 -07001668 The following are instance variables that are useful within test
1669 methods:
1670
1671 :ivar FakeGerritConnection fake_<connection>:
1672 A :py:class:`~tests.base.FakeGerritConnection` will be
1673 instantiated for each connection present in the config file
1674 and stored here. For instance, `fake_gerrit` will hold the
1675 FakeGerritConnection object for a connection named `gerrit`.
1676
1677 :ivar FakeGearmanServer gearman_server: An instance of
1678 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1679 server that all of the Zuul components in this test use to
1680 communicate with each other.
1681
Paul Belanger174a8272017-03-14 13:20:10 -04001682 :ivar RecordingExecutorServer executor_server: An instance of
1683 :py:class:`~tests.base.RecordingExecutorServer` which is the
1684 Ansible execute server used to run jobs for this test.
James E. Blaire7b99a02016-08-05 14:27:34 -07001685
1686 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1687 representing currently running builds. They are appended to
Paul Belanger174a8272017-03-14 13:20:10 -04001688 the list in the order they are executed, and removed from this
James E. Blaire7b99a02016-08-05 14:27:34 -07001689 list upon completion.
1690
1691 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1692 objects representing completed builds. They are appended to
1693 the list in the order they complete.
1694
1695 """
1696
James E. Blair83005782015-12-11 14:46:03 -08001697 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001698 run_ansible = False
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001699 create_project_keys = False
James E. Blair3f876d52016-07-22 13:07:14 -07001700
1701 def _startMerger(self):
1702 self.merge_server = zuul.merger.server.MergeServer(self.config,
1703 self.connections)
1704 self.merge_server.start()
1705
Maru Newby3fe5f852015-01-13 04:22:14 +00001706 def setUp(self):
1707 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001708
1709 self.setupZK()
1710
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001711 if not KEEP_TEMPDIRS:
James E. Blair97d902e2014-08-21 13:25:56 -07001712 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001713 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1714 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001715 else:
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001716 tmp_root = tempfile.mkdtemp(
1717 dir=os.environ.get("ZUUL_TEST_ROOT", None))
Clark Boylanb640e052014-04-03 16:41:46 -07001718 self.test_root = os.path.join(tmp_root, "zuul-test")
1719 self.upstream_root = os.path.join(self.test_root, "upstream")
Monty Taylord642d852017-02-23 14:05:42 -05001720 self.merger_src_root = os.path.join(self.test_root, "merger-git")
Paul Belanger174a8272017-03-14 13:20:10 -04001721 self.executor_src_root = os.path.join(self.test_root, "executor-git")
James E. Blairce8a2132016-05-19 15:21:52 -07001722 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001723
1724 if os.path.exists(self.test_root):
1725 shutil.rmtree(self.test_root)
1726 os.makedirs(self.test_root)
1727 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001728 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001729
1730 # Make per test copy of Configuration.
1731 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001732 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001733 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001734 self.config.get('zuul', 'tenant_config')))
Monty Taylord642d852017-02-23 14:05:42 -05001735 self.config.set('merger', 'git_dir', self.merger_src_root)
Paul Belanger174a8272017-03-14 13:20:10 -04001736 self.config.set('executor', 'git_dir', self.executor_src_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001737 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001738
Clark Boylanb640e052014-04-03 16:41:46 -07001739 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001740 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1741 # see: https://github.com/jsocol/pystatsd/issues/61
1742 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001743 os.environ['STATSD_PORT'] = str(self.statsd.port)
1744 self.statsd.start()
1745 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001746 reload_module(statsd)
1747 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001748
1749 self.gearman_server = FakeGearmanServer()
1750
1751 self.config.set('gearman', 'port', str(self.gearman_server.port))
James E. Blaire47eb772017-02-02 17:19:40 -08001752 self.log.info("Gearman server on port %s" %
1753 (self.gearman_server.port,))
Clark Boylanb640e052014-04-03 16:41:46 -07001754
James E. Blaire511d2f2016-12-08 15:22:26 -08001755 gerritsource.GerritSource.replication_timeout = 1.5
1756 gerritsource.GerritSource.replication_retry_interval = 0.5
1757 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001758
Joshua Hesketh352264b2015-08-11 23:42:08 +10001759 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001760
Jan Hruban7083edd2015-08-21 14:00:54 +02001761 self.webapp = zuul.webapp.WebApp(
1762 self.sched, port=0, listen_address='127.0.0.1')
1763
Jan Hruban6b71aff2015-10-22 16:58:08 +02001764 self.event_queues = [
1765 self.sched.result_event_queue,
James E. Blair646322f2017-01-27 15:50:34 -08001766 self.sched.trigger_event_queue,
1767 self.sched.management_event_queue
Jan Hruban6b71aff2015-10-22 16:58:08 +02001768 ]
1769
James E. Blairfef78942016-03-11 16:28:56 -08001770 self.configure_connections()
Jan Hruban7083edd2015-08-21 14:00:54 +02001771 self.sched.registerConnections(self.connections, self.webapp)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001772
Clark Boylanb640e052014-04-03 16:41:46 -07001773 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001774 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001775 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001776 return FakeURLOpener(self.upstream_root, *args, **kw)
1777
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001778 old_urlopen = urllib.request.urlopen
1779 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001780
Paul Belanger174a8272017-03-14 13:20:10 -04001781 self.executor_server = RecordingExecutorServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001782 self.config, self.connections,
James E. Blair854f8892017-02-02 11:25:39 -08001783 jobdir_root=self.test_root,
James E. Blaira92cbc82017-01-23 14:56:49 -08001784 _run_ansible=self.run_ansible,
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001785 _test_root=self.test_root,
1786 keep_jobdir=KEEP_TEMPDIRS)
Paul Belanger174a8272017-03-14 13:20:10 -04001787 self.executor_server.start()
1788 self.history = self.executor_server.build_history
1789 self.builds = self.executor_server.running_builds
James E. Blaire1767bc2016-08-02 10:00:27 -07001790
Paul Belanger174a8272017-03-14 13:20:10 -04001791 self.executor_client = zuul.executor.client.ExecutorClient(
James E. Blair92e953a2017-03-07 13:08:47 -08001792 self.config, self.sched)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001793 self.merge_client = zuul.merger.client.MergeClient(
1794 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001795 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001796 self.zk = zuul.zk.ZooKeeper()
James E. Blair0d5a36e2017-02-21 10:53:44 -05001797 self.zk.connect(self.zk_config)
James E. Blairdce6cea2016-12-20 16:45:32 -08001798
James E. Blair0d5a36e2017-02-21 10:53:44 -05001799 self.fake_nodepool = FakeNodepool(
1800 self.zk_chroot_fixture.zookeeper_host,
1801 self.zk_chroot_fixture.zookeeper_port,
1802 self.zk_chroot_fixture.zookeeper_chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001803
Paul Belanger174a8272017-03-14 13:20:10 -04001804 self.sched.setExecutor(self.executor_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001805 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001806 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001807 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001808
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001809 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001810
1811 self.sched.start()
Clark Boylanb640e052014-04-03 16:41:46 -07001812 self.webapp.start()
1813 self.rpc.start()
Paul Belanger174a8272017-03-14 13:20:10 -04001814 self.executor_client.gearman.waitForServer()
James E. Blaira002b032017-04-18 10:35:48 -07001815 # Cleanups are run in reverse order
1816 self.addCleanup(self.assertCleanShutdown)
Clark Boylanb640e052014-04-03 16:41:46 -07001817 self.addCleanup(self.shutdown)
James E. Blaira002b032017-04-18 10:35:48 -07001818 self.addCleanup(self.assertFinalState)
Clark Boylanb640e052014-04-03 16:41:46 -07001819
James E. Blairb9c0d772017-03-03 14:34:49 -08001820 self.sched.reconfigure(self.config)
1821 self.sched.resume()
1822
James E. Blairfef78942016-03-11 16:28:56 -08001823 def configure_connections(self):
James E. Blaire511d2f2016-12-08 15:22:26 -08001824 # Set up gerrit related fakes
1825 # Set a changes database so multiple FakeGerrit's can report back to
1826 # a virtual canonical database given by the configured hostname
1827 self.gerrit_changes_dbs = {}
1828
1829 def getGerritConnection(driver, name, config):
1830 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1831 con = FakeGerritConnection(driver, name, config,
1832 changes_db=db,
1833 upstream_root=self.upstream_root)
1834 self.event_queues.append(con.event_queue)
1835 setattr(self, 'fake_' + name, con)
1836 return con
1837
1838 self.useFixture(fixtures.MonkeyPatch(
1839 'zuul.driver.gerrit.GerritDriver.getConnection',
1840 getGerritConnection))
1841
Gregory Haynes4fc12542015-04-22 20:38:06 -07001842 def getGithubConnection(driver, name, config):
1843 con = FakeGithubConnection(driver, name, config,
1844 upstream_root=self.upstream_root)
1845 setattr(self, 'fake_' + name, con)
1846 return con
1847
1848 self.useFixture(fixtures.MonkeyPatch(
1849 'zuul.driver.github.GithubDriver.getConnection',
1850 getGithubConnection))
1851
James E. Blaire511d2f2016-12-08 15:22:26 -08001852 # Set up smtp related fakes
Joshua Heskethd78b4482015-09-14 16:56:34 -06001853 # TODO(jhesketh): This should come from lib.connections for better
1854 # coverage
Joshua Hesketh352264b2015-08-11 23:42:08 +10001855 # Register connections from the config
1856 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001857
Joshua Hesketh352264b2015-08-11 23:42:08 +10001858 def FakeSMTPFactory(*args, **kw):
1859 args = [self.smtp_messages] + list(args)
1860 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001861
Joshua Hesketh352264b2015-08-11 23:42:08 +10001862 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001863
James E. Blaire511d2f2016-12-08 15:22:26 -08001864 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001865 self.connections = zuul.lib.connections.ConnectionRegistry()
James E. Blaire511d2f2016-12-08 15:22:26 -08001866 self.connections.configure(self.config)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001867
James E. Blair83005782015-12-11 14:46:03 -08001868 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001869 # This creates the per-test configuration object. It can be
1870 # overriden by subclasses, but should not need to be since it
1871 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001872 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001873 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair06cc3922017-04-19 10:08:10 -07001874
1875 if not self.setupSimpleLayout():
1876 if hasattr(self, 'tenant_config_file'):
1877 self.config.set('zuul', 'tenant_config',
1878 self.tenant_config_file)
1879 git_path = os.path.join(
1880 os.path.dirname(
1881 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1882 'git')
1883 if os.path.exists(git_path):
1884 for reponame in os.listdir(git_path):
1885 project = reponame.replace('_', '/')
1886 self.copyDirToRepo(project,
1887 os.path.join(git_path, reponame))
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001888 self.setupAllProjectKeys()
1889
James E. Blair06cc3922017-04-19 10:08:10 -07001890 def setupSimpleLayout(self):
1891 # If the test method has been decorated with a simple_layout,
1892 # use that instead of the class tenant_config_file. Set up a
1893 # single config-project with the specified layout, and
1894 # initialize repos for all of the 'project' entries which
1895 # appear in the layout.
1896 test_name = self.id().split('.')[-1]
1897 test = getattr(self, test_name)
1898 if hasattr(test, '__simple_layout__'):
Jesse Keating436a5452017-04-20 11:48:41 -07001899 path, driver = getattr(test, '__simple_layout__')
James E. Blair06cc3922017-04-19 10:08:10 -07001900 else:
1901 return False
1902
James E. Blairb70e55a2017-04-19 12:57:02 -07001903 files = {}
James E. Blair06cc3922017-04-19 10:08:10 -07001904 path = os.path.join(FIXTURE_DIR, path)
1905 with open(path) as f:
James E. Blairb70e55a2017-04-19 12:57:02 -07001906 data = f.read()
1907 layout = yaml.safe_load(data)
1908 files['zuul.yaml'] = data
James E. Blair06cc3922017-04-19 10:08:10 -07001909 untrusted_projects = []
1910 for item in layout:
1911 if 'project' in item:
1912 name = item['project']['name']
1913 untrusted_projects.append(name)
1914 self.init_repo(name)
1915 self.addCommitToRepo(name, 'initial commit',
1916 files={'README': ''},
1917 branch='master', tag='init')
James E. Blairb70e55a2017-04-19 12:57:02 -07001918 if 'job' in item:
1919 jobname = item['job']['name']
1920 files['playbooks/%s.yaml' % jobname] = ''
James E. Blair06cc3922017-04-19 10:08:10 -07001921
1922 root = os.path.join(self.test_root, "config")
1923 if not os.path.exists(root):
1924 os.makedirs(root)
1925 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1926 config = [{'tenant':
1927 {'name': 'tenant-one',
Jesse Keating436a5452017-04-20 11:48:41 -07001928 'source': {driver:
James E. Blair06cc3922017-04-19 10:08:10 -07001929 {'config-projects': ['common-config'],
1930 'untrusted-projects': untrusted_projects}}}}]
Clint Byrumd52b7d72017-05-10 19:40:35 -07001931 f.write(yaml.dump(config).encode('utf8'))
James E. Blair06cc3922017-04-19 10:08:10 -07001932 f.close()
1933 self.config.set('zuul', 'tenant_config',
1934 os.path.join(FIXTURE_DIR, f.name))
1935
1936 self.init_repo('common-config')
James E. Blair06cc3922017-04-19 10:08:10 -07001937 self.addCommitToRepo('common-config', 'add content from fixture',
1938 files, branch='master', tag='init')
1939
1940 return True
1941
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001942 def setupAllProjectKeys(self):
1943 if self.create_project_keys:
1944 return
1945
1946 path = self.config.get('zuul', 'tenant_config')
1947 with open(os.path.join(FIXTURE_DIR, path)) as f:
1948 tenant_config = yaml.safe_load(f.read())
1949 for tenant in tenant_config:
1950 sources = tenant['tenant']['source']
1951 for source, conf in sources.items():
James E. Blair109da3f2017-04-04 14:39:43 -07001952 for project in conf.get('config-projects', []):
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001953 self.setupProjectKeys(source, project)
James E. Blair109da3f2017-04-04 14:39:43 -07001954 for project in conf.get('untrusted-projects', []):
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001955 self.setupProjectKeys(source, project)
1956
1957 def setupProjectKeys(self, source, project):
1958 # Make sure we set up an RSA key for the project so that we
1959 # don't spend time generating one:
1960
1961 key_root = os.path.join(self.state_root, 'keys')
1962 if not os.path.isdir(key_root):
1963 os.mkdir(key_root, 0o700)
1964 private_key_file = os.path.join(key_root, source, project + '.pem')
1965 private_key_dir = os.path.dirname(private_key_file)
1966 self.log.debug("Installing test keys for project %s at %s" % (
1967 project, private_key_file))
1968 if not os.path.isdir(private_key_dir):
1969 os.makedirs(private_key_dir)
1970 with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
1971 with open(private_key_file, 'w') as o:
1972 o.write(i.read())
James E. Blair96c6bf82016-01-15 16:20:40 -08001973
James E. Blair498059b2016-12-20 13:50:13 -08001974 def setupZK(self):
Clark Boylan621ec9a2017-04-07 17:41:33 -07001975 self.zk_chroot_fixture = self.useFixture(
1976 ChrootedKazooFixture(self.id()))
James E. Blair0d5a36e2017-02-21 10:53:44 -05001977 self.zk_config = '%s:%s%s' % (
James E. Blairdce6cea2016-12-20 16:45:32 -08001978 self.zk_chroot_fixture.zookeeper_host,
1979 self.zk_chroot_fixture.zookeeper_port,
1980 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08001981
James E. Blair96c6bf82016-01-15 16:20:40 -08001982 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001983 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08001984
1985 files = {}
1986 for (dirpath, dirnames, filenames) in os.walk(source_path):
1987 for filename in filenames:
1988 test_tree_filepath = os.path.join(dirpath, filename)
1989 common_path = os.path.commonprefix([test_tree_filepath,
1990 source_path])
1991 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1992 with open(test_tree_filepath, 'r') as f:
1993 content = f.read()
1994 files[relative_filepath] = content
1995 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001996 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08001997
James E. Blaire18d4602017-01-05 11:17:28 -08001998 def assertNodepoolState(self):
1999 # Make sure that there are no pending requests
2000
2001 requests = self.fake_nodepool.getNodeRequests()
2002 self.assertEqual(len(requests), 0)
2003
2004 nodes = self.fake_nodepool.getNodes()
2005 for node in nodes:
2006 self.assertFalse(node['_lock'], "Node %s is locked" %
2007 (node['_oid'],))
2008
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002009 def assertNoGeneratedKeys(self):
2010 # Make sure that Zuul did not generate any project keys
2011 # (unless it was supposed to).
2012
2013 if self.create_project_keys:
2014 return
2015
2016 with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
2017 test_key = i.read()
2018
2019 key_root = os.path.join(self.state_root, 'keys')
2020 for root, dirname, files in os.walk(key_root):
2021 for fn in files:
2022 with open(os.path.join(root, fn)) as f:
2023 self.assertEqual(test_key, f.read())
2024
Clark Boylanb640e052014-04-03 16:41:46 -07002025 def assertFinalState(self):
James E. Blaira002b032017-04-18 10:35:48 -07002026 self.log.debug("Assert final state")
2027 # Make sure no jobs are running
2028 self.assertEqual({}, self.executor_server.job_workers)
Clark Boylanb640e052014-04-03 16:41:46 -07002029 # Make sure that git.Repo objects have been garbage collected.
2030 repos = []
2031 gc.collect()
2032 for obj in gc.get_objects():
2033 if isinstance(obj, git.Repo):
James E. Blairc43525f2017-03-03 11:10:26 -08002034 self.log.debug("Leaked git repo object: %s" % repr(obj))
Clark Boylanb640e052014-04-03 16:41:46 -07002035 repos.append(obj)
2036 self.assertEqual(len(repos), 0)
2037 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08002038 self.assertNodepoolState()
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002039 self.assertNoGeneratedKeys()
James E. Blair83005782015-12-11 14:46:03 -08002040 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08002041 for tenant in self.sched.abide.tenants.values():
2042 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08002043 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08002044 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07002045
2046 def shutdown(self):
2047 self.log.debug("Shutting down after tests")
Paul Belanger174a8272017-03-14 13:20:10 -04002048 self.executor_client.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07002049 self.merge_client.stop()
Paul Belanger174a8272017-03-14 13:20:10 -04002050 self.executor_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07002051 self.sched.stop()
2052 self.sched.join()
2053 self.statsd.stop()
2054 self.statsd.join()
2055 self.webapp.stop()
2056 self.webapp.join()
2057 self.rpc.stop()
2058 self.rpc.join()
2059 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08002060 self.fake_nodepool.stop()
2061 self.zk.disconnect()
Clark Boylan8208c192017-04-24 18:08:08 -07002062 self.printHistory()
Clark Boylanf18e3b82017-04-24 17:34:13 -07002063 # we whitelist watchdog threads as they have relatively long delays
2064 # before noticing they should exit, but they should exit on their own.
2065 threads = [t for t in threading.enumerate()
2066 if t.name != 'executor-watchdog']
Clark Boylanb640e052014-04-03 16:41:46 -07002067 if len(threads) > 1:
Clark Boylan8208c192017-04-24 18:08:08 -07002068 log_str = ""
2069 for thread_id, stack_frame in sys._current_frames().items():
2070 log_str += "Thread: %s\n" % thread_id
2071 log_str += "".join(traceback.format_stack(stack_frame))
2072 self.log.debug(log_str)
2073 raise Exception("More than one thread is running: %s" % threads)
Clark Boylanb640e052014-04-03 16:41:46 -07002074
James E. Blaira002b032017-04-18 10:35:48 -07002075 def assertCleanShutdown(self):
2076 pass
2077
James E. Blairc4ba97a2017-04-19 16:26:24 -07002078 def init_repo(self, project, tag=None):
Clark Boylanb640e052014-04-03 16:41:46 -07002079 parts = project.split('/')
2080 path = os.path.join(self.upstream_root, *parts[:-1])
2081 if not os.path.exists(path):
2082 os.makedirs(path)
2083 path = os.path.join(self.upstream_root, project)
2084 repo = git.Repo.init(path)
2085
Morgan Fainberg78c301a2016-07-14 13:47:01 -07002086 with repo.config_writer() as config_writer:
2087 config_writer.set_value('user', 'email', 'user@example.com')
2088 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07002089
Clark Boylanb640e052014-04-03 16:41:46 -07002090 repo.index.commit('initial commit')
2091 master = repo.create_head('master')
James E. Blairc4ba97a2017-04-19 16:26:24 -07002092 if tag:
2093 repo.create_tag(tag)
Clark Boylanb640e052014-04-03 16:41:46 -07002094
James E. Blair97d902e2014-08-21 13:25:56 -07002095 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07002096 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07002097 repo.git.clean('-x', '-f', '-d')
2098
James E. Blair97d902e2014-08-21 13:25:56 -07002099 def create_branch(self, project, branch):
2100 path = os.path.join(self.upstream_root, project)
2101 repo = git.Repo.init(path)
2102 fn = os.path.join(path, 'README')
2103
2104 branch_head = repo.create_head(branch)
2105 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07002106 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07002107 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07002108 f.close()
2109 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07002110 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07002111
James E. Blair97d902e2014-08-21 13:25:56 -07002112 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07002113 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07002114 repo.git.clean('-x', '-f', '-d')
2115
Sachi King9f16d522016-03-16 12:20:45 +11002116 def create_commit(self, project):
2117 path = os.path.join(self.upstream_root, project)
2118 repo = git.Repo(path)
2119 repo.head.reference = repo.heads['master']
2120 file_name = os.path.join(path, 'README')
2121 with open(file_name, 'a') as f:
2122 f.write('creating fake commit\n')
2123 repo.index.add([file_name])
2124 commit = repo.index.commit('Creating a fake commit')
2125 return commit.hexsha
2126
James E. Blairf4a5f022017-04-18 14:01:10 -07002127 def orderedRelease(self, count=None):
James E. Blairb8c16472015-05-05 14:55:26 -07002128 # Run one build at a time to ensure non-race order:
James E. Blairf4a5f022017-04-18 14:01:10 -07002129 i = 0
James E. Blairb8c16472015-05-05 14:55:26 -07002130 while len(self.builds):
2131 self.release(self.builds[0])
2132 self.waitUntilSettled()
James E. Blairf4a5f022017-04-18 14:01:10 -07002133 i += 1
2134 if count is not None and i >= count:
2135 break
James E. Blairb8c16472015-05-05 14:55:26 -07002136
Clark Boylanb640e052014-04-03 16:41:46 -07002137 def release(self, job):
2138 if isinstance(job, FakeBuild):
2139 job.release()
2140 else:
2141 job.waiting = False
2142 self.log.debug("Queued job %s released" % job.unique)
2143 self.gearman_server.wakeConnections()
2144
2145 def getParameter(self, job, name):
2146 if isinstance(job, FakeBuild):
2147 return job.parameters[name]
2148 else:
2149 parameters = json.loads(job.arguments)
2150 return parameters[name]
2151
Clark Boylanb640e052014-04-03 16:41:46 -07002152 def haveAllBuildsReported(self):
2153 # See if Zuul is waiting on a meta job to complete
Paul Belanger174a8272017-03-14 13:20:10 -04002154 if self.executor_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07002155 return False
2156 # Find out if every build that the worker has completed has been
2157 # reported back to Zuul. If it hasn't then that means a Gearman
2158 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07002159 for build in self.history:
Paul Belanger174a8272017-03-14 13:20:10 -04002160 zbuild = self.executor_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07002161 if not zbuild:
2162 # It has already been reported
2163 continue
2164 # It hasn't been reported yet.
2165 return False
2166 # Make sure that none of the worker connections are in GRAB_WAIT
Paul Belanger174a8272017-03-14 13:20:10 -04002167 for connection in self.executor_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07002168 if connection.state == 'GRAB_WAIT':
2169 return False
2170 return True
2171
2172 def areAllBuildsWaiting(self):
Paul Belanger174a8272017-03-14 13:20:10 -04002173 builds = self.executor_client.builds.values()
James E. Blaira002b032017-04-18 10:35:48 -07002174 seen_builds = set()
Clark Boylanb640e052014-04-03 16:41:46 -07002175 for build in builds:
James E. Blaira002b032017-04-18 10:35:48 -07002176 seen_builds.add(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07002177 client_job = None
Paul Belanger174a8272017-03-14 13:20:10 -04002178 for conn in self.executor_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07002179 for j in conn.related_jobs.values():
2180 if j.unique == build.uuid:
2181 client_job = j
2182 break
2183 if not client_job:
2184 self.log.debug("%s is not known to the gearman client" %
2185 build)
James E. Blairf15139b2015-04-02 16:37:15 -07002186 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002187 if not client_job.handle:
2188 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002189 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002190 server_job = self.gearman_server.jobs.get(client_job.handle)
2191 if not server_job:
2192 self.log.debug("%s is not known to the gearman server" %
2193 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002194 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002195 if not hasattr(server_job, 'waiting'):
2196 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002197 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002198 if server_job.waiting:
2199 continue
James E. Blair17302972016-08-10 16:11:42 -07002200 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08002201 self.log.debug("%s has not reported start" % build)
2202 return False
Clint Byrumf322fe22017-05-10 20:53:12 -07002203 # using internal ServerJob which offers no Text interface
Paul Belanger174a8272017-03-14 13:20:10 -04002204 worker_build = self.executor_server.job_builds.get(
Clint Byrumf322fe22017-05-10 20:53:12 -07002205 server_job.unique.decode('utf8'))
James E. Blair962220f2016-08-03 11:22:38 -07002206 if worker_build:
2207 if worker_build.isWaiting():
2208 continue
2209 else:
2210 self.log.debug("%s is running" % worker_build)
2211 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002212 else:
James E. Blair962220f2016-08-03 11:22:38 -07002213 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002214 return False
James E. Blaira002b032017-04-18 10:35:48 -07002215 for (build_uuid, job_worker) in \
2216 self.executor_server.job_workers.items():
2217 if build_uuid not in seen_builds:
2218 self.log.debug("%s is not finalized" % build_uuid)
2219 return False
James E. Blairf15139b2015-04-02 16:37:15 -07002220 return True
Clark Boylanb640e052014-04-03 16:41:46 -07002221
James E. Blairdce6cea2016-12-20 16:45:32 -08002222 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08002223 if self.fake_nodepool.paused:
2224 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08002225 if self.sched.nodepool.requests:
2226 return False
2227 return True
2228
Jan Hruban6b71aff2015-10-22 16:58:08 +02002229 def eventQueuesEmpty(self):
2230 for queue in self.event_queues:
2231 yield queue.empty()
2232
2233 def eventQueuesJoin(self):
2234 for queue in self.event_queues:
2235 queue.join()
2236
Clark Boylanb640e052014-04-03 16:41:46 -07002237 def waitUntilSettled(self):
2238 self.log.debug("Waiting until settled...")
2239 start = time.time()
2240 while True:
Clint Byruma9626572017-02-22 14:04:00 -05002241 if time.time() - start > self.wait_timeout:
James E. Blair10fc1eb2016-12-21 16:16:25 -08002242 self.log.error("Timeout waiting for Zuul to settle")
2243 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07002244 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08002245 self.log.error(" %s: %s" % (queue, queue.empty()))
2246 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07002247 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08002248 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07002249 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08002250 self.log.error("All requests completed: %s" %
2251 (self.areAllNodeRequestsComplete(),))
2252 self.log.error("Merge client jobs: %s" %
2253 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07002254 raise Exception("Timeout waiting for Zuul to settle")
2255 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07002256
Paul Belanger174a8272017-03-14 13:20:10 -04002257 self.executor_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07002258 # have all build states propogated to zuul?
2259 if self.haveAllBuildsReported():
2260 # Join ensures that the queue is empty _and_ events have been
2261 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02002262 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07002263 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08002264 if (not self.merge_client.jobs and
Clark Boylanb640e052014-04-03 16:41:46 -07002265 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08002266 self.areAllBuildsWaiting() and
James E. Blair36c611a2017-02-06 15:59:43 -08002267 self.areAllNodeRequestsComplete() and
2268 all(self.eventQueuesEmpty())):
2269 # The queue empty check is placed at the end to
2270 # ensure that if a component adds an event between
2271 # when locked the run handler and checked that the
2272 # components were stable, we don't erroneously
2273 # report that we are settled.
Clark Boylanb640e052014-04-03 16:41:46 -07002274 self.sched.run_handler_lock.release()
Paul Belanger174a8272017-03-14 13:20:10 -04002275 self.executor_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07002276 self.log.debug("...settled.")
2277 return
2278 self.sched.run_handler_lock.release()
Paul Belanger174a8272017-03-14 13:20:10 -04002279 self.executor_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07002280 self.sched.wake_event.wait(0.1)
2281
2282 def countJobResults(self, jobs, result):
2283 jobs = filter(lambda x: x.result == result, jobs)
Clint Byrum1d0c7d12017-05-10 19:40:53 -07002284 return len(list(jobs))
Clark Boylanb640e052014-04-03 16:41:46 -07002285
James E. Blair96c6bf82016-01-15 16:20:40 -08002286 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07002287 for job in self.history:
2288 if (job.name == name and
2289 (project is None or
2290 job.parameters['ZUUL_PROJECT'] == project)):
2291 return job
Clark Boylanb640e052014-04-03 16:41:46 -07002292 raise Exception("Unable to find job %s in history" % name)
2293
2294 def assertEmptyQueues(self):
2295 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08002296 for tenant in self.sched.abide.tenants.values():
2297 for pipeline in tenant.layout.pipelines.values():
2298 for queue in pipeline.queues:
2299 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10002300 print('pipeline %s queue %s contents %s' % (
2301 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08002302 self.assertEqual(len(queue.queue), 0,
2303 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07002304
2305 def assertReportedStat(self, key, value=None, kind=None):
2306 start = time.time()
2307 while time.time() < (start + 5):
2308 for stat in self.statsd.stats:
Clint Byrumf322fe22017-05-10 20:53:12 -07002309 k, v = stat.decode('utf-8').split(':')
Clark Boylanb640e052014-04-03 16:41:46 -07002310 if key == k:
2311 if value is None and kind is None:
2312 return
2313 elif value:
2314 if value == v:
2315 return
2316 elif kind:
2317 if v.endswith('|' + kind):
2318 return
2319 time.sleep(0.1)
2320
Clark Boylanb640e052014-04-03 16:41:46 -07002321 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08002322
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002323 def assertBuilds(self, builds):
2324 """Assert that the running builds are as described.
2325
2326 The list of running builds is examined and must match exactly
2327 the list of builds described by the input.
2328
2329 :arg list builds: A list of dictionaries. Each item in the
2330 list must match the corresponding build in the build
2331 history, and each element of the dictionary must match the
2332 corresponding attribute of the build.
2333
2334 """
James E. Blair3158e282016-08-19 09:34:11 -07002335 try:
2336 self.assertEqual(len(self.builds), len(builds))
2337 for i, d in enumerate(builds):
2338 for k, v in d.items():
2339 self.assertEqual(
2340 getattr(self.builds[i], k), v,
2341 "Element %i in builds does not match" % (i,))
2342 except Exception:
2343 for build in self.builds:
2344 self.log.error("Running build: %s" % build)
2345 else:
2346 self.log.error("No running builds")
2347 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002348
James E. Blairb536ecc2016-08-31 10:11:42 -07002349 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002350 """Assert that the completed builds are as described.
2351
2352 The list of completed builds is examined and must match
2353 exactly the list of builds described by the input.
2354
2355 :arg list history: A list of dictionaries. Each item in the
2356 list must match the corresponding build in the build
2357 history, and each element of the dictionary must match the
2358 corresponding attribute of the build.
2359
James E. Blairb536ecc2016-08-31 10:11:42 -07002360 :arg bool ordered: If true, the history must match the order
2361 supplied, if false, the builds are permitted to have
2362 arrived in any order.
2363
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002364 """
James E. Blairb536ecc2016-08-31 10:11:42 -07002365 def matches(history_item, item):
2366 for k, v in item.items():
2367 if getattr(history_item, k) != v:
2368 return False
2369 return True
James E. Blair3158e282016-08-19 09:34:11 -07002370 try:
2371 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07002372 if ordered:
2373 for i, d in enumerate(history):
2374 if not matches(self.history[i], d):
2375 raise Exception(
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08002376 "Element %i in history does not match %s" %
2377 (i, self.history[i]))
James E. Blairb536ecc2016-08-31 10:11:42 -07002378 else:
2379 unseen = self.history[:]
2380 for i, d in enumerate(history):
2381 found = False
2382 for unseen_item in unseen:
2383 if matches(unseen_item, d):
2384 found = True
2385 unseen.remove(unseen_item)
2386 break
2387 if not found:
2388 raise Exception("No match found for element %i "
2389 "in history" % (i,))
2390 if unseen:
2391 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07002392 except Exception:
2393 for build in self.history:
2394 self.log.error("Completed build: %s" % build)
2395 else:
2396 self.log.error("No completed builds")
2397 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002398
James E. Blair6ac368c2016-12-22 18:07:20 -08002399 def printHistory(self):
2400 """Log the build history.
2401
2402 This can be useful during tests to summarize what jobs have
2403 completed.
2404
2405 """
2406 self.log.debug("Build history:")
2407 for build in self.history:
2408 self.log.debug(build)
2409
James E. Blair59fdbac2015-12-07 17:08:06 -08002410 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08002411 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
2412
James E. Blair9ea70072017-04-19 16:05:30 -07002413 def updateConfigLayout(self, path):
James E. Blairf84026c2015-12-08 16:11:46 -08002414 root = os.path.join(self.test_root, "config")
Clint Byrumead6c562017-02-01 16:34:04 -08002415 if not os.path.exists(root):
2416 os.makedirs(root)
James E. Blairf84026c2015-12-08 16:11:46 -08002417 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
2418 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05002419- tenant:
2420 name: openstack
2421 source:
2422 gerrit:
James E. Blair109da3f2017-04-04 14:39:43 -07002423 config-projects:
Paul Belanger66e95962016-11-11 12:11:06 -05002424 - %s
James E. Blair109da3f2017-04-04 14:39:43 -07002425 untrusted-projects:
James E. Blair0ffa0102017-03-30 13:11:33 -07002426 - org/project
2427 - org/project1
James E. Blair7cb84542017-04-19 13:35:05 -07002428 - org/project2\n""" % path)
James E. Blairf84026c2015-12-08 16:11:46 -08002429 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05002430 self.config.set('zuul', 'tenant_config',
2431 os.path.join(FIXTURE_DIR, f.name))
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002432 self.setupAllProjectKeys()
James E. Blair14abdf42015-12-09 16:11:53 -08002433
James E. Blair8b1dc3f2016-07-05 16:49:00 -07002434 def addCommitToRepo(self, project, message, files,
2435 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08002436 path = os.path.join(self.upstream_root, project)
2437 repo = git.Repo(path)
2438 repo.head.reference = branch
2439 zuul.merger.merger.reset_repo_to_head(repo)
2440 for fn, content in files.items():
2441 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08002442 try:
2443 os.makedirs(os.path.dirname(fn))
2444 except OSError:
2445 pass
James E. Blair14abdf42015-12-09 16:11:53 -08002446 with open(fn, 'w') as f:
2447 f.write(content)
2448 repo.index.add([fn])
2449 commit = repo.index.commit(message)
Clint Byrum58264dc2017-02-07 21:21:22 -08002450 before = repo.heads[branch].commit
James E. Blair14abdf42015-12-09 16:11:53 -08002451 repo.heads[branch].commit = commit
2452 repo.head.reference = branch
2453 repo.git.clean('-x', '-f', '-d')
2454 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07002455 if tag:
2456 repo.create_tag(tag)
Clint Byrum58264dc2017-02-07 21:21:22 -08002457 return before
2458
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002459 def commitConfigUpdate(self, project_name, source_name):
2460 """Commit an update to zuul.yaml
2461
2462 This overwrites the zuul.yaml in the specificed project with
2463 the contents specified.
2464
2465 :arg str project_name: The name of the project containing
2466 zuul.yaml (e.g., common-config)
2467
2468 :arg str source_name: The path to the file (underneath the
2469 test fixture directory) whose contents should be used to
2470 replace zuul.yaml.
2471 """
2472
2473 source_path = os.path.join(FIXTURE_DIR, source_name)
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002474 files = {}
2475 with open(source_path, 'r') as f:
2476 data = f.read()
2477 layout = yaml.safe_load(data)
2478 files['zuul.yaml'] = data
2479 for item in layout:
2480 if 'job' in item:
2481 jobname = item['job']['name']
2482 files['playbooks/%s.yaml' % jobname] = ''
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002483 before = self.addCommitToRepo(
2484 project_name, 'Pulling content from %s' % source_name,
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002485 files)
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002486 return before
2487
James E. Blair7fc8daa2016-08-08 15:37:15 -07002488 def addEvent(self, connection, event):
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002489
James E. Blair7fc8daa2016-08-08 15:37:15 -07002490 """Inject a Fake (Gerrit) event.
2491
2492 This method accepts a JSON-encoded event and simulates Zuul
2493 having received it from Gerrit. It could (and should)
2494 eventually apply to any connection type, but is currently only
2495 used with Gerrit connections. The name of the connection is
2496 used to look up the corresponding server, and the event is
2497 simulated as having been received by all Zuul connections
2498 attached to that server. So if two Gerrit connections in Zuul
2499 are connected to the same Gerrit server, and you invoke this
2500 method specifying the name of one of them, the event will be
2501 received by both.
2502
2503 .. note::
2504
2505 "self.fake_gerrit.addEvent" calls should be migrated to
2506 this method.
2507
2508 :arg str connection: The name of the connection corresponding
Clark Boylan500992b2017-04-03 14:28:24 -07002509 to the gerrit server.
James E. Blair7fc8daa2016-08-08 15:37:15 -07002510 :arg str event: The JSON-encoded event.
2511
2512 """
2513 specified_conn = self.connections.connections[connection]
2514 for conn in self.connections.connections.values():
2515 if (isinstance(conn, specified_conn.__class__) and
2516 specified_conn.server == conn.server):
2517 conn.addEvent(event)
2518
James E. Blair3f876d52016-07-22 13:07:14 -07002519
2520class AnsibleZuulTestCase(ZuulTestCase):
Paul Belanger174a8272017-03-14 13:20:10 -04002521 """ZuulTestCase but with an actual ansible executor running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07002522 run_ansible = True
Joshua Hesketh25695cb2017-03-06 12:50:04 +11002523
Joshua Heskethd78b4482015-09-14 16:56:34 -06002524
2525class ZuulDBTestCase(ZuulTestCase):
James E. Blair82844892017-03-06 10:55:26 -08002526 def setup_config(self):
2527 super(ZuulDBTestCase, self).setup_config()
Joshua Heskethd78b4482015-09-14 16:56:34 -06002528 for section_name in self.config.sections():
2529 con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
2530 section_name, re.I)
2531 if not con_match:
2532 continue
2533
2534 if self.config.get(section_name, 'driver') == 'sql':
2535 f = MySQLSchemaFixture()
2536 self.useFixture(f)
2537 if (self.config.get(section_name, 'dburi') ==
2538 '$MYSQL_FIXTURE_DBURI$'):
2539 self.config.set(section_name, 'dburi', f.dburi)