blob: 2bcd1ca19a0f232960c2f1fef76ce9738978cb18 [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Adam Gandelmand81dd762017-02-09 15:15:49 -080019import datetime
Clark Boylanb640e052014-04-03 16:41:46 -070020import gc
21import hashlib
22import json
23import logging
24import os
Christian Berendt12d4d722014-06-07 21:03:45 +020025from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070026from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070027import random
28import re
29import select
30import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030031from six.moves import reload_module
Clark Boylan21a2c812017-04-24 15:44:55 -070032try:
33 from cStringIO import StringIO
34except Exception:
35 from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070036import socket
37import string
38import subprocess
James E. Blair1c236df2017-02-01 14:07:24 -080039import sys
James E. Blairf84026c2015-12-08 16:11:46 -080040import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070041import threading
Clark Boylan8208c192017-04-24 18:08:08 -070042import traceback
Clark Boylanb640e052014-04-03 16:41:46 -070043import time
Joshua Heskethd78b4482015-09-14 16:56:34 -060044import uuid
45
Clark Boylanb640e052014-04-03 16:41:46 -070046
47import git
48import gear
49import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080050import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080051import kazoo.exceptions
Joshua Heskethd78b4482015-09-14 16:56:34 -060052import pymysql
Clark Boylanb640e052014-04-03 16:41:46 -070053import statsd
54import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080055import testtools.content
56import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080057from git.exc import NoSuchPathError
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +000058import yaml
Clark Boylanb640e052014-04-03 16:41:46 -070059
James E. Blaire511d2f2016-12-08 15:22:26 -080060import zuul.driver.gerrit.gerritsource as gerritsource
61import zuul.driver.gerrit.gerritconnection as gerritconnection
Gregory Haynes4fc12542015-04-22 20:38:06 -070062import zuul.driver.github.githubconnection as githubconnection
Clark Boylanb640e052014-04-03 16:41:46 -070063import zuul.scheduler
64import zuul.webapp
65import zuul.rpclistener
Paul Belanger174a8272017-03-14 13:20:10 -040066import zuul.executor.server
67import zuul.executor.client
James E. Blair83005782015-12-11 14:46:03 -080068import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070069import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070070import zuul.merger.merger
71import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070072import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080073import zuul.zk
Jan Hruban49bff072015-11-03 11:45:46 +010074from zuul.exceptions import MergeFailure
Clark Boylanb640e052014-04-03 16:41:46 -070075
76FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
77 'fixtures')
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -080078
79KEEP_TEMPDIRS = bool(os.environ.get('KEEP_TEMPDIRS', False))
Clark Boylanb640e052014-04-03 16:41:46 -070080
Clark Boylanb640e052014-04-03 16:41:46 -070081
82def repack_repo(path):
83 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
84 output = subprocess.Popen(cmd, close_fds=True,
85 stdout=subprocess.PIPE,
86 stderr=subprocess.PIPE)
87 out = output.communicate()
88 if output.returncode:
89 raise Exception("git repack returned %d" % output.returncode)
90 return out
91
92
93def random_sha1():
Clint Byrumc0923d52017-05-10 15:47:41 -040094 return hashlib.sha1(str(random.random()).encode('ascii')).hexdigest()
Clark Boylanb640e052014-04-03 16:41:46 -070095
96
James E. Blaira190f3b2015-01-05 14:56:54 -080097def iterate_timeout(max_seconds, purpose):
98 start = time.time()
99 count = 0
100 while (time.time() < start + max_seconds):
101 count += 1
102 yield count
103 time.sleep(0)
104 raise Exception("Timeout waiting for %s" % purpose)
105
106
Jesse Keating436a5452017-04-20 11:48:41 -0700107def simple_layout(path, driver='gerrit'):
James E. Blair06cc3922017-04-19 10:08:10 -0700108 """Specify a layout file for use by a test method.
109
110 :arg str path: The path to the layout file.
Jesse Keating436a5452017-04-20 11:48:41 -0700111 :arg str driver: The source driver to use, defaults to gerrit.
James E. Blair06cc3922017-04-19 10:08:10 -0700112
113 Some tests require only a very simple configuration. For those,
114 establishing a complete config directory hierachy is too much
115 work. In those cases, you can add a simple zuul.yaml file to the
116 test fixtures directory (in fixtures/layouts/foo.yaml) and use
117 this decorator to indicate the test method should use that rather
118 than the tenant config file specified by the test class.
119
120 The decorator will cause that layout file to be added to a
121 config-project called "common-config" and each "project" instance
122 referenced in the layout file will have a git repo automatically
123 initialized.
124 """
125
126 def decorator(test):
Jesse Keating436a5452017-04-20 11:48:41 -0700127 test.__simple_layout__ = (path, driver)
James E. Blair06cc3922017-04-19 10:08:10 -0700128 return test
129 return decorator
130
131
Gregory Haynes4fc12542015-04-22 20:38:06 -0700132class GerritChangeReference(git.Reference):
Clark Boylanb640e052014-04-03 16:41:46 -0700133 _common_path_default = "refs/changes"
134 _points_to_commits_only = True
135
136
Gregory Haynes4fc12542015-04-22 20:38:06 -0700137class FakeGerritChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700138 categories = {'approved': ('Approved', -1, 1),
139 'code-review': ('Code-Review', -2, 2),
140 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700141
142 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700143 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700144 self.gerrit = gerrit
Gregory Haynes4fc12542015-04-22 20:38:06 -0700145 self.source = gerrit
Clark Boylanb640e052014-04-03 16:41:46 -0700146 self.reported = 0
147 self.queried = 0
148 self.patchsets = []
149 self.number = number
150 self.project = project
151 self.branch = branch
152 self.subject = subject
153 self.latest_patchset = 0
154 self.depends_on_change = None
155 self.needed_by_changes = []
156 self.fail_merge = False
157 self.messages = []
158 self.data = {
159 'branch': branch,
160 'comments': [],
161 'commitMessage': subject,
162 'createdOn': time.time(),
163 'id': 'I' + random_sha1(),
164 'lastUpdated': time.time(),
165 'number': str(number),
166 'open': status == 'NEW',
167 'owner': {'email': 'user@example.com',
168 'name': 'User Name',
169 'username': 'username'},
170 'patchSets': self.patchsets,
171 'project': project,
172 'status': status,
173 'subject': subject,
174 'submitRecords': [],
175 'url': 'https://hostname/%s' % number}
176
177 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700178 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700179 self.data['submitRecords'] = self.getSubmitRecords()
180 self.open = status == 'NEW'
181
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700182 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700183 path = os.path.join(self.upstream_root, self.project)
184 repo = git.Repo(path)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700185 ref = GerritChangeReference.create(
186 repo, '1/%s/%s' % (self.number, self.latest_patchset),
187 'refs/tags/init')
Clark Boylanb640e052014-04-03 16:41:46 -0700188 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700189 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700190 repo.git.clean('-x', '-f', '-d')
191
192 path = os.path.join(self.upstream_root, self.project)
193 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700194 for fn, content in files.items():
195 fn = os.path.join(path, fn)
196 with open(fn, 'w') as f:
197 f.write(content)
198 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700199 else:
200 for fni in range(100):
201 fn = os.path.join(path, str(fni))
202 f = open(fn, 'w')
203 for ci in range(4096):
204 f.write(random.choice(string.printable))
205 f.close()
206 repo.index.add([fn])
207
208 r = repo.index.commit(msg)
209 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700210 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700211 repo.git.clean('-x', '-f', '-d')
212 repo.heads['master'].checkout()
213 return r
214
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700215 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700216 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700217 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700218 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700219 data = ("test %s %s %s\n" %
220 (self.branch, self.number, self.latest_patchset))
221 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700222 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700223 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700224 ps_files = [{'file': '/COMMIT_MSG',
225 'type': 'ADDED'},
226 {'file': 'README',
227 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700228 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700229 ps_files.append({'file': f, 'type': 'ADDED'})
230 d = {'approvals': [],
231 'createdOn': time.time(),
232 'files': ps_files,
233 'number': str(self.latest_patchset),
234 'ref': 'refs/changes/1/%s/%s' % (self.number,
235 self.latest_patchset),
236 'revision': c.hexsha,
237 'uploader': {'email': 'user@example.com',
238 'name': 'User name',
239 'username': 'user'}}
240 self.data['currentPatchSet'] = d
241 self.patchsets.append(d)
242 self.data['submitRecords'] = self.getSubmitRecords()
243
244 def getPatchsetCreatedEvent(self, patchset):
245 event = {"type": "patchset-created",
246 "change": {"project": self.project,
247 "branch": self.branch,
248 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
249 "number": str(self.number),
250 "subject": self.subject,
251 "owner": {"name": "User Name"},
252 "url": "https://hostname/3"},
253 "patchSet": self.patchsets[patchset - 1],
254 "uploader": {"name": "User Name"}}
255 return event
256
257 def getChangeRestoredEvent(self):
258 event = {"type": "change-restored",
259 "change": {"project": self.project,
260 "branch": self.branch,
261 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
262 "number": str(self.number),
263 "subject": self.subject,
264 "owner": {"name": "User Name"},
265 "url": "https://hostname/3"},
266 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100267 "patchSet": self.patchsets[-1],
268 "reason": ""}
269 return event
270
271 def getChangeAbandonedEvent(self):
272 event = {"type": "change-abandoned",
273 "change": {"project": self.project,
274 "branch": self.branch,
275 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
276 "number": str(self.number),
277 "subject": self.subject,
278 "owner": {"name": "User Name"},
279 "url": "https://hostname/3"},
280 "abandoner": {"name": "User Name"},
281 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700282 "reason": ""}
283 return event
284
285 def getChangeCommentEvent(self, patchset):
286 event = {"type": "comment-added",
287 "change": {"project": self.project,
288 "branch": self.branch,
289 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
290 "number": str(self.number),
291 "subject": self.subject,
292 "owner": {"name": "User Name"},
293 "url": "https://hostname/3"},
294 "patchSet": self.patchsets[patchset - 1],
295 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700296 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700297 "description": "Code-Review",
298 "value": "0"}],
299 "comment": "This is a comment"}
300 return event
301
James E. Blairc2a5ed72017-02-20 14:12:01 -0500302 def getChangeMergedEvent(self):
303 event = {"submitter": {"name": "Jenkins",
304 "username": "jenkins"},
305 "newRev": "29ed3b5f8f750a225c5be70235230e3a6ccb04d9",
306 "patchSet": self.patchsets[-1],
307 "change": self.data,
308 "type": "change-merged",
309 "eventCreatedOn": 1487613810}
310 return event
311
James E. Blair8cce42e2016-10-18 08:18:36 -0700312 def getRefUpdatedEvent(self):
313 path = os.path.join(self.upstream_root, self.project)
314 repo = git.Repo(path)
315 oldrev = repo.heads[self.branch].commit.hexsha
316
317 event = {
318 "type": "ref-updated",
319 "submitter": {
320 "name": "User Name",
321 },
322 "refUpdate": {
323 "oldRev": oldrev,
324 "newRev": self.patchsets[-1]['revision'],
325 "refName": self.branch,
326 "project": self.project,
327 }
328 }
329 return event
330
Joshua Hesketh642824b2014-07-01 17:54:59 +1000331 def addApproval(self, category, value, username='reviewer_john',
332 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700333 if not granted_on:
334 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000335 approval = {
336 'description': self.categories[category][0],
337 'type': category,
338 'value': str(value),
339 'by': {
340 'username': username,
341 'email': username + '@example.com',
342 },
343 'grantedOn': int(granted_on)
344 }
Clark Boylanb640e052014-04-03 16:41:46 -0700345 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
346 if x['by']['username'] == username and x['type'] == category:
347 del self.patchsets[-1]['approvals'][i]
348 self.patchsets[-1]['approvals'].append(approval)
349 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000350 'author': {'email': 'author@example.com',
351 'name': 'Patchset Author',
352 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700353 'change': {'branch': self.branch,
354 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
355 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000356 'owner': {'email': 'owner@example.com',
357 'name': 'Change Owner',
358 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700359 'project': self.project,
360 'subject': self.subject,
361 'topic': 'master',
362 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000363 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700364 'patchSet': self.patchsets[-1],
365 'type': 'comment-added'}
366 self.data['submitRecords'] = self.getSubmitRecords()
367 return json.loads(json.dumps(event))
368
369 def getSubmitRecords(self):
370 status = {}
371 for cat in self.categories.keys():
372 status[cat] = 0
373
374 for a in self.patchsets[-1]['approvals']:
375 cur = status[a['type']]
376 cat_min, cat_max = self.categories[a['type']][1:]
377 new = int(a['value'])
378 if new == cat_min:
379 cur = new
380 elif abs(new) > abs(cur):
381 cur = new
382 status[a['type']] = cur
383
384 labels = []
385 ok = True
386 for typ, cat in self.categories.items():
387 cur = status[typ]
388 cat_min, cat_max = cat[1:]
389 if cur == cat_min:
390 value = 'REJECT'
391 ok = False
392 elif cur == cat_max:
393 value = 'OK'
394 else:
395 value = 'NEED'
396 ok = False
397 labels.append({'label': cat[0], 'status': value})
398 if ok:
399 return [{'status': 'OK'}]
400 return [{'status': 'NOT_READY',
401 'labels': labels}]
402
403 def setDependsOn(self, other, patchset):
404 self.depends_on_change = other
405 d = {'id': other.data['id'],
406 'number': other.data['number'],
407 'ref': other.patchsets[patchset - 1]['ref']
408 }
409 self.data['dependsOn'] = [d]
410
411 other.needed_by_changes.append(self)
412 needed = other.data.get('neededBy', [])
413 d = {'id': self.data['id'],
414 'number': self.data['number'],
415 'ref': self.patchsets[patchset - 1]['ref'],
416 'revision': self.patchsets[patchset - 1]['revision']
417 }
418 needed.append(d)
419 other.data['neededBy'] = needed
420
421 def query(self):
422 self.queried += 1
423 d = self.data.get('dependsOn')
424 if d:
425 d = d[0]
426 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
427 d['isCurrentPatchSet'] = True
428 else:
429 d['isCurrentPatchSet'] = False
430 return json.loads(json.dumps(self.data))
431
432 def setMerged(self):
433 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000434 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700435 return
436 if self.fail_merge:
437 return
438 self.data['status'] = 'MERGED'
439 self.open = False
440
441 path = os.path.join(self.upstream_root, self.project)
442 repo = git.Repo(path)
443 repo.heads[self.branch].commit = \
444 repo.commit(self.patchsets[-1]['revision'])
445
446 def setReported(self):
447 self.reported += 1
448
449
James E. Blaire511d2f2016-12-08 15:22:26 -0800450class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700451 """A Fake Gerrit connection for use in tests.
452
453 This subclasses
454 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
455 ability for tests to add changes to the fake Gerrit it represents.
456 """
457
Joshua Hesketh352264b2015-08-11 23:42:08 +1000458 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700459
James E. Blaire511d2f2016-12-08 15:22:26 -0800460 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700461 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800462 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000463 connection_config)
464
James E. Blair7fc8daa2016-08-08 15:37:15 -0700465 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700466 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
467 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000468 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700469 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200470 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700471
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700472 def addFakeChange(self, project, branch, subject, status='NEW',
473 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700474 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700475 self.change_number += 1
Gregory Haynes4fc12542015-04-22 20:38:06 -0700476 c = FakeGerritChange(self, self.change_number, project, branch,
477 subject, upstream_root=self.upstream_root,
478 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700479 self.changes[self.change_number] = c
480 return c
481
Clark Boylanb640e052014-04-03 16:41:46 -0700482 def review(self, project, changeid, message, action):
483 number, ps = changeid.split(',')
484 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000485
486 # Add the approval back onto the change (ie simulate what gerrit would
487 # do).
488 # Usually when zuul leaves a review it'll create a feedback loop where
489 # zuul's review enters another gerrit event (which is then picked up by
490 # zuul). However, we can't mimic this behaviour (by adding this
491 # approval event into the queue) as it stops jobs from checking what
492 # happens before this event is triggered. If a job needs to see what
493 # happens they can add their own verified event into the queue.
494 # Nevertheless, we can update change with the new review in gerrit.
495
James E. Blair8b5408c2016-08-08 15:37:46 -0700496 for cat in action.keys():
497 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000498 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000499
Clark Boylanb640e052014-04-03 16:41:46 -0700500 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000501
Clark Boylanb640e052014-04-03 16:41:46 -0700502 if 'submit' in action:
503 change.setMerged()
504 if message:
505 change.setReported()
506
507 def query(self, number):
508 change = self.changes.get(int(number))
509 if change:
510 return change.query()
511 return {}
512
James E. Blairc494d542014-08-06 09:23:52 -0700513 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700514 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700515 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800516 if query.startswith('change:'):
517 # Query a specific changeid
518 changeid = query[len('change:'):]
519 l = [change.query() for change in self.changes.values()
520 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700521 elif query.startswith('message:'):
522 # Query the content of a commit message
523 msg = query[len('message:'):].strip()
524 l = [change.query() for change in self.changes.values()
525 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800526 else:
527 # Query all open changes
528 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700529 return l
James E. Blairc494d542014-08-06 09:23:52 -0700530
Joshua Hesketh352264b2015-08-11 23:42:08 +1000531 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700532 pass
533
Joshua Hesketh352264b2015-08-11 23:42:08 +1000534 def getGitUrl(self, project):
535 return os.path.join(self.upstream_root, project.name)
536
Clark Boylanb640e052014-04-03 16:41:46 -0700537
Gregory Haynes4fc12542015-04-22 20:38:06 -0700538class GithubChangeReference(git.Reference):
539 _common_path_default = "refs/pull"
540 _points_to_commits_only = True
541
542
543class FakeGithubPullRequest(object):
544
545 def __init__(self, github, number, project, branch,
Jesse Keatingae4cd272017-01-30 17:10:44 -0800546 subject, upstream_root, files=[], number_of_commits=1,
547 writers=[]):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700548 """Creates a new PR with several commits.
549 Sends an event about opened PR."""
550 self.github = github
551 self.source = github
552 self.number = number
553 self.project = project
554 self.branch = branch
Jan Hruban37615e52015-11-19 14:30:49 +0100555 self.subject = subject
556 self.number_of_commits = 0
Gregory Haynes4fc12542015-04-22 20:38:06 -0700557 self.upstream_root = upstream_root
Jan Hruban570d01c2016-03-10 21:51:32 +0100558 self.files = []
Gregory Haynes4fc12542015-04-22 20:38:06 -0700559 self.comments = []
Jan Hruban16ad31f2015-11-07 14:39:07 +0100560 self.labels = []
Jan Hrubane252a732017-01-03 15:03:09 +0100561 self.statuses = {}
Jesse Keatingae4cd272017-01-30 17:10:44 -0800562 self.reviews = []
563 self.writers = []
Gregory Haynes4fc12542015-04-22 20:38:06 -0700564 self.updated_at = None
565 self.head_sha = None
Jan Hruban49bff072015-11-03 11:45:46 +0100566 self.is_merged = False
Jan Hruban3b415922016-02-03 13:10:22 +0100567 self.merge_message = None
Gregory Haynes4fc12542015-04-22 20:38:06 -0700568 self._createPRRef()
Jan Hruban570d01c2016-03-10 21:51:32 +0100569 self._addCommitToRepo(files=files)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700570 self._updateTimeStamp()
571
Jan Hruban570d01c2016-03-10 21:51:32 +0100572 def addCommit(self, files=[]):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700573 """Adds a commit on top of the actual PR head."""
Jan Hruban570d01c2016-03-10 21:51:32 +0100574 self._addCommitToRepo(files=files)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700575 self._updateTimeStamp()
576
Jan Hruban570d01c2016-03-10 21:51:32 +0100577 def forcePush(self, files=[]):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700578 """Clears actual commits and add a commit on top of the base."""
Jan Hruban570d01c2016-03-10 21:51:32 +0100579 self._addCommitToRepo(files=files, reset=True)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700580 self._updateTimeStamp()
581
582 def getPullRequestOpenedEvent(self):
583 return self._getPullRequestEvent('opened')
584
585 def getPullRequestSynchronizeEvent(self):
586 return self._getPullRequestEvent('synchronize')
587
588 def getPullRequestReopenedEvent(self):
589 return self._getPullRequestEvent('reopened')
590
591 def getPullRequestClosedEvent(self):
592 return self._getPullRequestEvent('closed')
593
594 def addComment(self, message):
595 self.comments.append(message)
596 self._updateTimeStamp()
597
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200598 def getCommentAddedEvent(self, text):
599 name = 'issue_comment'
600 data = {
601 'action': 'created',
602 'issue': {
603 'number': self.number
604 },
605 'comment': {
606 'body': text
607 },
608 'repository': {
609 'full_name': self.project
Jan Hruban3b415922016-02-03 13:10:22 +0100610 },
611 'sender': {
612 'login': 'ghuser'
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200613 }
614 }
615 return (name, data)
616
Jesse Keating5c05a9f2017-01-12 14:44:58 -0800617 def getReviewAddedEvent(self, review):
618 name = 'pull_request_review'
619 data = {
620 'action': 'submitted',
621 'pull_request': {
622 'number': self.number,
623 'title': self.subject,
624 'updated_at': self.updated_at,
625 'base': {
626 'ref': self.branch,
627 'repo': {
628 'full_name': self.project
629 }
630 },
631 'head': {
632 'sha': self.head_sha
633 }
634 },
635 'review': {
636 'state': review
637 },
638 'repository': {
639 'full_name': self.project
640 },
641 'sender': {
642 'login': 'ghuser'
643 }
644 }
645 return (name, data)
646
Jan Hruban16ad31f2015-11-07 14:39:07 +0100647 def addLabel(self, name):
648 if name not in self.labels:
649 self.labels.append(name)
650 self._updateTimeStamp()
651 return self._getLabelEvent(name)
652
653 def removeLabel(self, name):
654 if name in self.labels:
655 self.labels.remove(name)
656 self._updateTimeStamp()
657 return self._getUnlabelEvent(name)
658
659 def _getLabelEvent(self, label):
660 name = 'pull_request'
661 data = {
662 'action': 'labeled',
663 'pull_request': {
664 'number': self.number,
665 'updated_at': self.updated_at,
666 'base': {
667 'ref': self.branch,
668 'repo': {
669 'full_name': self.project
670 }
671 },
672 'head': {
673 'sha': self.head_sha
674 }
675 },
676 'label': {
677 'name': label
Jan Hruban3b415922016-02-03 13:10:22 +0100678 },
679 'sender': {
680 'login': 'ghuser'
Jan Hruban16ad31f2015-11-07 14:39:07 +0100681 }
682 }
683 return (name, data)
684
685 def _getUnlabelEvent(self, label):
686 name = 'pull_request'
687 data = {
688 'action': 'unlabeled',
689 'pull_request': {
690 'number': self.number,
Jan Hruban3b415922016-02-03 13:10:22 +0100691 'title': self.subject,
Jan Hruban16ad31f2015-11-07 14:39:07 +0100692 'updated_at': self.updated_at,
693 'base': {
694 'ref': self.branch,
695 'repo': {
696 'full_name': self.project
697 }
698 },
699 'head': {
Jesse Keatingd96e5882017-01-19 13:55:50 -0800700 'sha': self.head_sha,
701 'repo': {
702 'full_name': self.project
703 }
Jan Hruban16ad31f2015-11-07 14:39:07 +0100704 }
705 },
706 'label': {
707 'name': label
Jan Hruban3b415922016-02-03 13:10:22 +0100708 },
709 'sender': {
710 'login': 'ghuser'
Jan Hruban16ad31f2015-11-07 14:39:07 +0100711 }
712 }
713 return (name, data)
714
Gregory Haynes4fc12542015-04-22 20:38:06 -0700715 def _getRepo(self):
716 repo_path = os.path.join(self.upstream_root, self.project)
717 return git.Repo(repo_path)
718
719 def _createPRRef(self):
720 repo = self._getRepo()
721 GithubChangeReference.create(
722 repo, self._getPRReference(), 'refs/tags/init')
723
Jan Hruban570d01c2016-03-10 21:51:32 +0100724 def _addCommitToRepo(self, files=[], reset=False):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700725 repo = self._getRepo()
726 ref = repo.references[self._getPRReference()]
727 if reset:
Jan Hruban37615e52015-11-19 14:30:49 +0100728 self.number_of_commits = 0
Gregory Haynes4fc12542015-04-22 20:38:06 -0700729 ref.set_object('refs/tags/init')
Jan Hruban37615e52015-11-19 14:30:49 +0100730 self.number_of_commits += 1
Gregory Haynes4fc12542015-04-22 20:38:06 -0700731 repo.head.reference = ref
732 zuul.merger.merger.reset_repo_to_head(repo)
733 repo.git.clean('-x', '-f', '-d')
734
Jan Hruban570d01c2016-03-10 21:51:32 +0100735 if files:
736 fn = files[0]
737 self.files = files
738 else:
739 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
740 self.files = [fn]
Jan Hruban37615e52015-11-19 14:30:49 +0100741 msg = self.subject + '-' + str(self.number_of_commits)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700742 fn = os.path.join(repo.working_dir, fn)
743 f = open(fn, 'w')
744 with open(fn, 'w') as f:
745 f.write("test %s %s\n" %
746 (self.branch, self.number))
747 repo.index.add([fn])
748
749 self.head_sha = repo.index.commit(msg).hexsha
Jesse Keatingd96e5882017-01-19 13:55:50 -0800750 # Create an empty set of statuses for the given sha,
751 # each sha on a PR may have a status set on it
752 self.statuses[self.head_sha] = []
Gregory Haynes4fc12542015-04-22 20:38:06 -0700753 repo.head.reference = 'master'
754 zuul.merger.merger.reset_repo_to_head(repo)
755 repo.git.clean('-x', '-f', '-d')
756 repo.heads['master'].checkout()
757
758 def _updateTimeStamp(self):
759 self.updated_at = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
760
761 def getPRHeadSha(self):
762 repo = self._getRepo()
763 return repo.references[self._getPRReference()].commit.hexsha
764
Adam Gandelman8c6eeb52017-01-23 16:31:06 -0800765 def setStatus(self, sha, state, url, description, context, user='zuul'):
Jesse Keatingd96e5882017-01-19 13:55:50 -0800766 # Since we're bypassing github API, which would require a user, we
767 # hard set the user as 'zuul' here.
Jesse Keatingd96e5882017-01-19 13:55:50 -0800768 # insert the status at the top of the list, to simulate that it
769 # is the most recent set status
770 self.statuses[sha].insert(0, ({
Jan Hrubane252a732017-01-03 15:03:09 +0100771 'state': state,
772 'url': url,
Jesse Keatingd96e5882017-01-19 13:55:50 -0800773 'description': description,
774 'context': context,
775 'creator': {
776 'login': user
777 }
778 }))
Jan Hrubane252a732017-01-03 15:03:09 +0100779
Jesse Keatingae4cd272017-01-30 17:10:44 -0800780 def addReview(self, user, state, granted_on=None):
Adam Gandelmand81dd762017-02-09 15:15:49 -0800781 gh_time_format = '%Y-%m-%dT%H:%M:%SZ'
782 # convert the timestamp to a str format that would be returned
783 # from github as 'submitted_at' in the API response
Jesse Keatingae4cd272017-01-30 17:10:44 -0800784
Adam Gandelmand81dd762017-02-09 15:15:49 -0800785 if granted_on:
786 granted_on = datetime.datetime.utcfromtimestamp(granted_on)
787 submitted_at = time.strftime(
788 gh_time_format, granted_on.timetuple())
789 else:
790 # github timestamps only down to the second, so we need to make
791 # sure reviews that tests add appear to be added over a period of
792 # time in the past and not all at once.
793 if not self.reviews:
794 # the first review happens 10 mins ago
795 offset = 600
796 else:
797 # subsequent reviews happen 1 minute closer to now
798 offset = 600 - (len(self.reviews) * 60)
799
800 granted_on = datetime.datetime.utcfromtimestamp(
801 time.time() - offset)
802 submitted_at = time.strftime(
803 gh_time_format, granted_on.timetuple())
804
Jesse Keatingae4cd272017-01-30 17:10:44 -0800805 self.reviews.append({
806 'state': state,
807 'user': {
808 'login': user,
809 'email': user + "@derp.com",
810 },
Adam Gandelmand81dd762017-02-09 15:15:49 -0800811 'submitted_at': submitted_at,
Jesse Keatingae4cd272017-01-30 17:10:44 -0800812 })
813
Gregory Haynes4fc12542015-04-22 20:38:06 -0700814 def _getPRReference(self):
815 return '%s/head' % self.number
816
817 def _getPullRequestEvent(self, action):
818 name = 'pull_request'
819 data = {
820 'action': action,
821 'number': self.number,
822 'pull_request': {
823 'number': self.number,
Jan Hruban3b415922016-02-03 13:10:22 +0100824 'title': self.subject,
Gregory Haynes4fc12542015-04-22 20:38:06 -0700825 'updated_at': self.updated_at,
826 'base': {
827 'ref': self.branch,
828 'repo': {
829 'full_name': self.project
830 }
831 },
832 'head': {
Jesse Keatingd96e5882017-01-19 13:55:50 -0800833 'sha': self.head_sha,
834 'repo': {
835 'full_name': self.project
836 }
Gregory Haynes4fc12542015-04-22 20:38:06 -0700837 }
Jan Hruban3b415922016-02-03 13:10:22 +0100838 },
839 'sender': {
840 'login': 'ghuser'
Gregory Haynes4fc12542015-04-22 20:38:06 -0700841 }
842 }
843 return (name, data)
844
Adam Gandelman8c6eeb52017-01-23 16:31:06 -0800845 def getCommitStatusEvent(self, context, state='success', user='zuul'):
846 name = 'status'
847 data = {
848 'state': state,
849 'sha': self.head_sha,
850 'description': 'Test results for %s: %s' % (self.head_sha, state),
851 'target_url': 'http://zuul/%s' % self.head_sha,
852 'branches': [],
853 'context': context,
854 'sender': {
855 'login': user
856 }
857 }
858 return (name, data)
859
Gregory Haynes4fc12542015-04-22 20:38:06 -0700860
861class FakeGithubConnection(githubconnection.GithubConnection):
862 log = logging.getLogger("zuul.test.FakeGithubConnection")
863
864 def __init__(self, driver, connection_name, connection_config,
865 upstream_root=None):
866 super(FakeGithubConnection, self).__init__(driver, connection_name,
867 connection_config)
868 self.connection_name = connection_name
869 self.pr_number = 0
870 self.pull_requests = []
871 self.upstream_root = upstream_root
Jan Hruban49bff072015-11-03 11:45:46 +0100872 self.merge_failure = False
873 self.merge_not_allowed_count = 0
Gregory Haynes4fc12542015-04-22 20:38:06 -0700874
Jan Hruban570d01c2016-03-10 21:51:32 +0100875 def openFakePullRequest(self, project, branch, subject, files=[]):
Gregory Haynes4fc12542015-04-22 20:38:06 -0700876 self.pr_number += 1
877 pull_request = FakeGithubPullRequest(
Jan Hruban570d01c2016-03-10 21:51:32 +0100878 self, self.pr_number, project, branch, subject, self.upstream_root,
879 files=files)
Gregory Haynes4fc12542015-04-22 20:38:06 -0700880 self.pull_requests.append(pull_request)
881 return pull_request
882
Wayne1a78c612015-06-11 17:14:13 -0700883 def getPushEvent(self, project, ref, old_rev=None, new_rev=None):
884 if not old_rev:
885 old_rev = '00000000000000000000000000000000'
886 if not new_rev:
887 new_rev = random_sha1()
888 name = 'push'
889 data = {
890 'ref': ref,
891 'before': old_rev,
892 'after': new_rev,
893 'repository': {
894 'full_name': project
895 }
896 }
897 return (name, data)
898
Gregory Haynes4fc12542015-04-22 20:38:06 -0700899 def emitEvent(self, event):
900 """Emulates sending the GitHub webhook event to the connection."""
901 port = self.webapp.server.socket.getsockname()[1]
902 name, data = event
Clint Byrum607d10e2017-05-18 12:05:13 -0700903 payload = json.dumps(data).encode('utf8')
Gregory Haynes4fc12542015-04-22 20:38:06 -0700904 headers = {'X-Github-Event': name}
905 req = urllib.request.Request(
906 'http://localhost:%s/connection/%s/payload'
907 % (port, self.connection_name),
908 data=payload, headers=headers)
909 urllib.request.urlopen(req)
910
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200911 def getPull(self, project, number):
912 pr = self.pull_requests[number - 1]
913 data = {
914 'number': number,
Jan Hruban3b415922016-02-03 13:10:22 +0100915 'title': pr.subject,
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200916 'updated_at': pr.updated_at,
917 'base': {
918 'repo': {
919 'full_name': pr.project
920 },
921 'ref': pr.branch,
922 },
Jan Hruban37615e52015-11-19 14:30:49 +0100923 'mergeable': True,
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200924 'head': {
Jesse Keatingd96e5882017-01-19 13:55:50 -0800925 'sha': pr.head_sha,
926 'repo': {
927 'full_name': pr.project
928 }
Jan Hrubanc7ab1602015-10-14 15:29:33 +0200929 }
930 }
931 return data
932
Adam Gandelman8c6eeb52017-01-23 16:31:06 -0800933 def getPullBySha(self, sha):
934 prs = list(set([p for p in self.pull_requests if sha == p.head_sha]))
935 if len(prs) > 1:
936 raise Exception('Multiple pulls found with head sha: %s' % sha)
937 pr = prs[0]
938 return self.getPull(pr.project, pr.number)
939
Jan Hruban570d01c2016-03-10 21:51:32 +0100940 def getPullFileNames(self, project, number):
941 pr = self.pull_requests[number - 1]
942 return pr.files
943
Jesse Keatingae4cd272017-01-30 17:10:44 -0800944 def _getPullReviews(self, owner, project, number):
945 pr = self.pull_requests[number - 1]
946 return pr.reviews
947
Jan Hruban3b415922016-02-03 13:10:22 +0100948 def getUser(self, login):
949 data = {
950 'username': login,
951 'name': 'Github User',
952 'email': 'github.user@example.com'
953 }
954 return data
955
Jesse Keatingae4cd272017-01-30 17:10:44 -0800956 def getRepoPermission(self, project, login):
957 owner, proj = project.split('/')
958 for pr in self.pull_requests:
959 pr_owner, pr_project = pr.project.split('/')
960 if (pr_owner == owner and proj == pr_project):
961 if login in pr.writers:
962 return 'write'
963 else:
964 return 'read'
965
Gregory Haynes4fc12542015-04-22 20:38:06 -0700966 def getGitUrl(self, project):
967 return os.path.join(self.upstream_root, str(project))
968
Jan Hruban6d53c5e2015-10-24 03:03:34 +0200969 def real_getGitUrl(self, project):
970 return super(FakeGithubConnection, self).getGitUrl(project)
971
Gregory Haynes4fc12542015-04-22 20:38:06 -0700972 def getProjectBranches(self, project):
973 """Masks getProjectBranches since we don't have a real github"""
974
975 # just returns master for now
976 return ['master']
977
Jan Hrubane252a732017-01-03 15:03:09 +0100978 def commentPull(self, project, pr_number, message):
Wayne40f40042015-06-12 16:56:30 -0700979 pull_request = self.pull_requests[pr_number - 1]
980 pull_request.addComment(message)
981
Jan Hruban3b415922016-02-03 13:10:22 +0100982 def mergePull(self, project, pr_number, commit_message='', sha=None):
Jan Hruban49bff072015-11-03 11:45:46 +0100983 pull_request = self.pull_requests[pr_number - 1]
984 if self.merge_failure:
985 raise Exception('Pull request was not merged')
986 if self.merge_not_allowed_count > 0:
987 self.merge_not_allowed_count -= 1
988 raise MergeFailure('Merge was not successful due to mergeability'
989 ' conflict')
990 pull_request.is_merged = True
Jan Hruban3b415922016-02-03 13:10:22 +0100991 pull_request.merge_message = commit_message
Jan Hruban49bff072015-11-03 11:45:46 +0100992
Jesse Keatingd96e5882017-01-19 13:55:50 -0800993 def getCommitStatuses(self, project, sha):
994 owner, proj = project.split('/')
995 for pr in self.pull_requests:
996 pr_owner, pr_project = pr.project.split('/')
997 if (pr_owner == owner and pr_project == proj and
998 pr.head_sha == sha):
999 return pr.statuses[sha]
1000
Jan Hrubane252a732017-01-03 15:03:09 +01001001 def setCommitStatus(self, project, sha, state,
1002 url='', description='', context=''):
1003 owner, proj = project.split('/')
1004 for pr in self.pull_requests:
1005 pr_owner, pr_project = pr.project.split('/')
1006 if (pr_owner == owner and pr_project == proj and
1007 pr.head_sha == sha):
Jesse Keatingd96e5882017-01-19 13:55:50 -08001008 pr.setStatus(sha, state, url, description, context)
Jan Hrubane252a732017-01-03 15:03:09 +01001009
Jan Hruban16ad31f2015-11-07 14:39:07 +01001010 def labelPull(self, project, pr_number, label):
1011 pull_request = self.pull_requests[pr_number - 1]
1012 pull_request.addLabel(label)
1013
1014 def unlabelPull(self, project, pr_number, label):
1015 pull_request = self.pull_requests[pr_number - 1]
1016 pull_request.removeLabel(label)
1017
Gregory Haynes4fc12542015-04-22 20:38:06 -07001018
Clark Boylanb640e052014-04-03 16:41:46 -07001019class BuildHistory(object):
1020 def __init__(self, **kw):
1021 self.__dict__.update(kw)
1022
1023 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -07001024 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
1025 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -07001026
1027
1028class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +02001029 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -07001030 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -07001031 self.url = url
1032
1033 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001034 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -07001035 path = res.path
1036 project = '/'.join(path.split('/')[2:-2])
1037 ret = '001e# service=git-upload-pack\n'
1038 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
1039 'multi_ack thin-pack side-band side-band-64k ofs-delta '
1040 'shallow no-progress include-tag multi_ack_detailed no-done\n')
1041 path = os.path.join(self.upstream_root, project)
1042 repo = git.Repo(path)
1043 for ref in repo.refs:
1044 r = ref.object.hexsha + ' ' + ref.path + '\n'
1045 ret += '%04x%s' % (len(r) + 4, r)
1046 ret += '0000'
1047 return ret
1048
1049
Clark Boylanb640e052014-04-03 16:41:46 -07001050class FakeStatsd(threading.Thread):
1051 def __init__(self):
1052 threading.Thread.__init__(self)
1053 self.daemon = True
1054 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
1055 self.sock.bind(('', 0))
1056 self.port = self.sock.getsockname()[1]
1057 self.wake_read, self.wake_write = os.pipe()
1058 self.stats = []
1059
1060 def run(self):
1061 while True:
1062 poll = select.poll()
1063 poll.register(self.sock, select.POLLIN)
1064 poll.register(self.wake_read, select.POLLIN)
1065 ret = poll.poll()
1066 for (fd, event) in ret:
1067 if fd == self.sock.fileno():
1068 data = self.sock.recvfrom(1024)
1069 if not data:
1070 return
1071 self.stats.append(data[0])
1072 if fd == self.wake_read:
1073 return
1074
1075 def stop(self):
Clint Byrumf322fe22017-05-10 20:53:12 -07001076 os.write(self.wake_write, b'1\n')
Clark Boylanb640e052014-04-03 16:41:46 -07001077
1078
James E. Blaire1767bc2016-08-02 10:00:27 -07001079class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -07001080 log = logging.getLogger("zuul.test")
1081
Paul Belanger174a8272017-03-14 13:20:10 -04001082 def __init__(self, executor_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -07001083 self.daemon = True
Paul Belanger174a8272017-03-14 13:20:10 -04001084 self.executor_server = executor_server
Clark Boylanb640e052014-04-03 16:41:46 -07001085 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -07001086 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -07001087 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -07001088 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -07001089 # TODOv3(jeblair): self.node is really "the image of the node
1090 # assigned". We should rename it (self.node_image?) if we
1091 # keep using it like this, or we may end up exposing more of
1092 # the complexity around multi-node jobs here
1093 # (self.nodes[0].image?)
1094 self.node = None
1095 if len(self.parameters.get('nodes')) == 1:
1096 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -07001097 self.unique = self.parameters['ZUUL_UUID']
Jamie Lennoxd2e37332016-12-05 15:26:19 +11001098 self.pipeline = self.parameters['ZUUL_PIPELINE']
1099 self.project = self.parameters['ZUUL_PROJECT']
James E. Blair3f876d52016-07-22 13:07:14 -07001100 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -07001101 self.wait_condition = threading.Condition()
1102 self.waiting = False
1103 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -05001104 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -07001105 self.created = time.time()
James E. Blaire1767bc2016-08-02 10:00:27 -07001106 self.changes = None
1107 if 'ZUUL_CHANGE_IDS' in self.parameters:
1108 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -07001109
James E. Blair3158e282016-08-19 09:34:11 -07001110 def __repr__(self):
1111 waiting = ''
1112 if self.waiting:
1113 waiting = ' [waiting]'
Jamie Lennoxd2e37332016-12-05 15:26:19 +11001114 return '<FakeBuild %s:%s %s%s>' % (self.pipeline, self.name,
1115 self.changes, waiting)
James E. Blair3158e282016-08-19 09:34:11 -07001116
Clark Boylanb640e052014-04-03 16:41:46 -07001117 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001118 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -07001119 self.wait_condition.acquire()
1120 self.wait_condition.notify()
1121 self.waiting = False
1122 self.log.debug("Build %s released" % self.unique)
1123 self.wait_condition.release()
1124
1125 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001126 """Return whether this build is being held.
1127
1128 :returns: Whether the build is being held.
1129 :rtype: bool
1130 """
1131
Clark Boylanb640e052014-04-03 16:41:46 -07001132 self.wait_condition.acquire()
1133 if self.waiting:
1134 ret = True
1135 else:
1136 ret = False
1137 self.wait_condition.release()
1138 return ret
1139
1140 def _wait(self):
1141 self.wait_condition.acquire()
1142 self.waiting = True
1143 self.log.debug("Build %s waiting" % self.unique)
1144 self.wait_condition.wait()
1145 self.wait_condition.release()
1146
1147 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001148 self.log.debug('Running build %s' % self.unique)
1149
Paul Belanger174a8272017-03-14 13:20:10 -04001150 if self.executor_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -07001151 self.log.debug('Holding build %s' % self.unique)
1152 self._wait()
1153 self.log.debug("Build %s continuing" % self.unique)
1154
James E. Blair412fba82017-01-26 15:00:50 -08001155 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
James E. Blaira5dba232016-08-08 15:53:24 -07001156 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
James E. Blair412fba82017-01-26 15:00:50 -08001157 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
Clark Boylanb640e052014-04-03 16:41:46 -07001158 if self.aborted:
James E. Blair412fba82017-01-26 15:00:50 -08001159 result = (RecordingAnsibleJob.RESULT_ABORTED, None)
Paul Belanger71d98172016-11-08 10:56:31 -05001160 if self.requeue:
James E. Blair412fba82017-01-26 15:00:50 -08001161 result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
Clark Boylanb640e052014-04-03 16:41:46 -07001162
James E. Blaire1767bc2016-08-02 10:00:27 -07001163 return result
Clark Boylanb640e052014-04-03 16:41:46 -07001164
James E. Blaira5dba232016-08-08 15:53:24 -07001165 def shouldFail(self):
Paul Belanger174a8272017-03-14 13:20:10 -04001166 changes = self.executor_server.fail_tests.get(self.name, [])
James E. Blaira5dba232016-08-08 15:53:24 -07001167 for change in changes:
1168 if self.hasChanges(change):
1169 return True
1170 return False
1171
James E. Blaire7b99a02016-08-05 14:27:34 -07001172 def hasChanges(self, *changes):
1173 """Return whether this build has certain changes in its git repos.
1174
1175 :arg FakeChange changes: One or more changes (varargs) that
Clark Boylan500992b2017-04-03 14:28:24 -07001176 are expected to be present (in order) in the git repository of
1177 the active project.
James E. Blaire7b99a02016-08-05 14:27:34 -07001178
1179 :returns: Whether the build has the indicated changes.
1180 :rtype: bool
1181
1182 """
Clint Byrum3343e3e2016-11-15 16:05:03 -08001183 for change in changes:
Gregory Haynes4fc12542015-04-22 20:38:06 -07001184 hostname = change.source.canonical_hostname
James E. Blair2a535672017-04-27 12:03:15 -07001185 path = os.path.join(self.jobdir.src_root, hostname, change.project)
Clint Byrum3343e3e2016-11-15 16:05:03 -08001186 try:
1187 repo = git.Repo(path)
1188 except NoSuchPathError as e:
1189 self.log.debug('%s' % e)
1190 return False
1191 ref = self.parameters['ZUUL_REF']
1192 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
1193 commit_message = '%s-1' % change.subject
1194 self.log.debug("Checking if build %s has changes; commit_message "
1195 "%s; repo_messages %s" % (self, commit_message,
1196 repo_messages))
1197 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -07001198 self.log.debug(" messages do not match")
1199 return False
1200 self.log.debug(" OK")
1201 return True
1202
Clark Boylanb640e052014-04-03 16:41:46 -07001203
Paul Belanger174a8272017-03-14 13:20:10 -04001204class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1205 """An Ansible executor to be used in tests.
James E. Blaire7b99a02016-08-05 14:27:34 -07001206
Paul Belanger174a8272017-03-14 13:20:10 -04001207 :ivar bool hold_jobs_in_build: If true, when jobs are executed
James E. Blaire7b99a02016-08-05 14:27:34 -07001208 they will report that they have started but then pause until
1209 released before reporting completion. This attribute may be
1210 changed at any time and will take effect for subsequently
Paul Belanger174a8272017-03-14 13:20:10 -04001211 executed builds, but previously held builds will still need to
James E. Blaire7b99a02016-08-05 14:27:34 -07001212 be explicitly released.
1213
1214 """
James E. Blairf5dbd002015-12-23 15:26:17 -08001215 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -07001216 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -08001217 self._test_root = kw.pop('_test_root', False)
Paul Belanger174a8272017-03-14 13:20:10 -04001218 super(RecordingExecutorServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -07001219 self.hold_jobs_in_build = False
1220 self.lock = threading.Lock()
1221 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -07001222 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -07001223 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -07001224 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -08001225
James E. Blaira5dba232016-08-08 15:53:24 -07001226 def failJob(self, name, change):
Paul Belanger174a8272017-03-14 13:20:10 -04001227 """Instruct the executor to report matching builds as failures.
James E. Blaire7b99a02016-08-05 14:27:34 -07001228
1229 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -07001230 :arg Change change: The :py:class:`~tests.base.FakeChange`
1231 instance which should cause the job to fail. This job
1232 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -07001233
1234 """
James E. Blaire1767bc2016-08-02 10:00:27 -07001235 l = self.fail_tests.get(name, [])
1236 l.append(change)
1237 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -08001238
James E. Blair962220f2016-08-03 11:22:38 -07001239 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -07001240 """Release a held build.
1241
1242 :arg str regex: A regular expression which, if supplied, will
1243 cause only builds with matching names to be released. If
1244 not supplied, all builds will be released.
1245
1246 """
James E. Blair962220f2016-08-03 11:22:38 -07001247 builds = self.running_builds[:]
1248 self.log.debug("Releasing build %s (%s)" % (regex,
1249 len(self.running_builds)))
1250 for build in builds:
1251 if not regex or re.match(regex, build.name):
1252 self.log.debug("Releasing build %s" %
1253 (build.parameters['ZUUL_UUID']))
1254 build.release()
1255 else:
1256 self.log.debug("Not releasing build %s" %
1257 (build.parameters['ZUUL_UUID']))
1258 self.log.debug("Done releasing builds %s (%s)" %
1259 (regex, len(self.running_builds)))
1260
Paul Belanger174a8272017-03-14 13:20:10 -04001261 def executeJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -07001262 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -07001263 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -07001264 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -07001265 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -08001266 args = json.loads(job.arguments)
James E. Blair490cf042017-02-24 23:07:21 -05001267 args['vars']['zuul']['_test'] = dict(test_root=self._test_root)
James E. Blaira92cbc82017-01-23 14:56:49 -08001268 job.arguments = json.dumps(args)
Joshua Hesketh50c21782016-10-13 21:34:14 +11001269 self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
1270 self.job_workers[job.unique].run()
James E. Blair17302972016-08-10 16:11:42 -07001271
1272 def stopJob(self, job):
1273 self.log.debug("handle stop")
1274 parameters = json.loads(job.arguments)
1275 uuid = parameters['uuid']
1276 for build in self.running_builds:
1277 if build.unique == uuid:
1278 build.aborted = True
1279 build.release()
Paul Belanger174a8272017-03-14 13:20:10 -04001280 super(RecordingExecutorServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -07001281
James E. Blaira002b032017-04-18 10:35:48 -07001282 def stop(self):
1283 for build in self.running_builds:
1284 build.release()
1285 super(RecordingExecutorServer, self).stop()
1286
Joshua Hesketh50c21782016-10-13 21:34:14 +11001287
Paul Belanger174a8272017-03-14 13:20:10 -04001288class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001289 def doMergeChanges(self, items):
1290 # Get a merger in order to update the repos involved in this job.
1291 commit = super(RecordingAnsibleJob, self).doMergeChanges(items)
1292 if not commit: # merge conflict
1293 self.recordResult('MERGER_FAILURE')
1294 return commit
1295
1296 def recordResult(self, result):
Paul Belanger174a8272017-03-14 13:20:10 -04001297 build = self.executor_server.job_builds[self.job.unique]
Paul Belanger174a8272017-03-14 13:20:10 -04001298 self.executor_server.lock.acquire()
1299 self.executor_server.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -07001300 BuildHistory(name=build.name, result=result, changes=build.changes,
1301 node=build.node, uuid=build.unique,
K Jonathan Harker2c1a6232017-02-21 14:34:08 -08001302 parameters=build.parameters, jobdir=build.jobdir,
James E. Blaire1767bc2016-08-02 10:00:27 -07001303 pipeline=build.parameters['ZUUL_PIPELINE'])
1304 )
Paul Belanger174a8272017-03-14 13:20:10 -04001305 self.executor_server.running_builds.remove(build)
1306 del self.executor_server.job_builds[self.job.unique]
1307 self.executor_server.lock.release()
K Jonathan Harkerae04e4c2017-03-15 19:07:11 -07001308
1309 def runPlaybooks(self, args):
1310 build = self.executor_server.job_builds[self.job.unique]
1311 build.jobdir = self.jobdir
1312
1313 result = super(RecordingAnsibleJob, self).runPlaybooks(args)
1314 self.recordResult(result)
James E. Blair412fba82017-01-26 15:00:50 -08001315 return result
1316
Monty Taylore6562aa2017-02-20 07:37:39 -05001317 def runAnsible(self, cmd, timeout, trusted=False):
Paul Belanger174a8272017-03-14 13:20:10 -04001318 build = self.executor_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -08001319
Paul Belanger174a8272017-03-14 13:20:10 -04001320 if self.executor_server._run_ansible:
Monty Taylorc231d932017-02-03 09:57:15 -06001321 result = super(RecordingAnsibleJob, self).runAnsible(
Monty Taylore6562aa2017-02-20 07:37:39 -05001322 cmd, timeout, trusted=trusted)
James E. Blair412fba82017-01-26 15:00:50 -08001323 else:
1324 result = build.run()
James E. Blaire1767bc2016-08-02 10:00:27 -07001325 return result
James E. Blairf5dbd002015-12-23 15:26:17 -08001326
James E. Blairad8dca02017-02-21 11:48:32 -05001327 def getHostList(self, args):
1328 self.log.debug("hostlist")
1329 hosts = super(RecordingAnsibleJob, self).getHostList(args)
Paul Belangerc5bf3752017-03-16 19:38:43 -04001330 for host in hosts:
1331 host['host_vars']['ansible_connection'] = 'local'
1332
1333 hosts.append(dict(
1334 name='localhost',
1335 host_vars=dict(ansible_connection='local'),
1336 host_keys=[]))
James E. Blairad8dca02017-02-21 11:48:32 -05001337 return hosts
1338
James E. Blairf5dbd002015-12-23 15:26:17 -08001339
Clark Boylanb640e052014-04-03 16:41:46 -07001340class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -07001341 """A Gearman server for use in tests.
1342
1343 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
1344 added to the queue but will not be distributed to workers
1345 until released. This attribute may be changed at any time and
1346 will take effect for subsequently enqueued jobs, but
1347 previously held jobs will still need to be explicitly
1348 released.
1349
1350 """
1351
Clark Boylanb640e052014-04-03 16:41:46 -07001352 def __init__(self):
1353 self.hold_jobs_in_queue = False
1354 super(FakeGearmanServer, self).__init__(0)
1355
1356 def getJobForConnection(self, connection, peek=False):
1357 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
1358 for job in queue:
1359 if not hasattr(job, 'waiting'):
Clint Byrumf322fe22017-05-10 20:53:12 -07001360 if job.name.startswith(b'executor:execute'):
Clark Boylanb640e052014-04-03 16:41:46 -07001361 job.waiting = self.hold_jobs_in_queue
1362 else:
1363 job.waiting = False
1364 if job.waiting:
1365 continue
1366 if job.name in connection.functions:
1367 if not peek:
1368 queue.remove(job)
1369 connection.related_jobs[job.handle] = job
1370 job.worker_connection = connection
1371 job.running = True
1372 return job
1373 return None
1374
1375 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -07001376 """Release a held job.
1377
1378 :arg str regex: A regular expression which, if supplied, will
1379 cause only jobs with matching names to be released. If
1380 not supplied, all jobs will be released.
1381 """
Clark Boylanb640e052014-04-03 16:41:46 -07001382 released = False
1383 qlen = (len(self.high_queue) + len(self.normal_queue) +
1384 len(self.low_queue))
1385 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
1386 for job in self.getQueue():
Paul Belanger174a8272017-03-14 13:20:10 -04001387 if job.name != 'executor:execute':
Clark Boylanb640e052014-04-03 16:41:46 -07001388 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -05001389 parameters = json.loads(job.arguments)
1390 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -07001391 self.log.debug("releasing queued job %s" %
1392 job.unique)
1393 job.waiting = False
1394 released = True
1395 else:
1396 self.log.debug("not releasing queued job %s" %
1397 job.unique)
1398 if released:
1399 self.wakeConnections()
1400 qlen = (len(self.high_queue) + len(self.normal_queue) +
1401 len(self.low_queue))
1402 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
1403
1404
1405class FakeSMTP(object):
1406 log = logging.getLogger('zuul.FakeSMTP')
1407
1408 def __init__(self, messages, server, port):
1409 self.server = server
1410 self.port = port
1411 self.messages = messages
1412
1413 def sendmail(self, from_email, to_email, msg):
1414 self.log.info("Sending email from %s, to %s, with msg %s" % (
1415 from_email, to_email, msg))
1416
1417 headers = msg.split('\n\n', 1)[0]
1418 body = msg.split('\n\n', 1)[1]
1419
1420 self.messages.append(dict(
1421 from_email=from_email,
1422 to_email=to_email,
1423 msg=msg,
1424 headers=headers,
1425 body=body,
1426 ))
1427
1428 return True
1429
1430 def quit(self):
1431 return True
1432
1433
James E. Blairdce6cea2016-12-20 16:45:32 -08001434class FakeNodepool(object):
1435 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -08001436 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -08001437
1438 log = logging.getLogger("zuul.test.FakeNodepool")
1439
1440 def __init__(self, host, port, chroot):
1441 self.client = kazoo.client.KazooClient(
1442 hosts='%s:%s%s' % (host, port, chroot))
1443 self.client.start()
1444 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -08001445 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001446 self.thread = threading.Thread(target=self.run)
1447 self.thread.daemon = True
1448 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -08001449 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -08001450
1451 def stop(self):
1452 self._running = False
1453 self.thread.join()
1454 self.client.stop()
1455 self.client.close()
1456
1457 def run(self):
1458 while self._running:
James E. Blaircbbce0d2017-05-19 07:28:29 -07001459 try:
1460 self._run()
1461 except Exception:
1462 self.log.exception("Error in fake nodepool:")
James E. Blairdce6cea2016-12-20 16:45:32 -08001463 time.sleep(0.1)
1464
1465 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001466 if self.paused:
1467 return
James E. Blairdce6cea2016-12-20 16:45:32 -08001468 for req in self.getNodeRequests():
1469 self.fulfillRequest(req)
1470
1471 def getNodeRequests(self):
1472 try:
1473 reqids = self.client.get_children(self.REQUEST_ROOT)
1474 except kazoo.exceptions.NoNodeError:
1475 return []
1476 reqs = []
1477 for oid in sorted(reqids):
1478 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -08001479 try:
1480 data, stat = self.client.get(path)
Clint Byrumf322fe22017-05-10 20:53:12 -07001481 data = json.loads(data.decode('utf8'))
James E. Blair0ef64f82017-02-02 11:25:16 -08001482 data['_oid'] = oid
1483 reqs.append(data)
1484 except kazoo.exceptions.NoNodeError:
1485 pass
James E. Blairdce6cea2016-12-20 16:45:32 -08001486 return reqs
1487
James E. Blaire18d4602017-01-05 11:17:28 -08001488 def getNodes(self):
1489 try:
1490 nodeids = self.client.get_children(self.NODE_ROOT)
1491 except kazoo.exceptions.NoNodeError:
1492 return []
1493 nodes = []
1494 for oid in sorted(nodeids):
1495 path = self.NODE_ROOT + '/' + oid
1496 data, stat = self.client.get(path)
Clint Byrumf322fe22017-05-10 20:53:12 -07001497 data = json.loads(data.decode('utf8'))
James E. Blaire18d4602017-01-05 11:17:28 -08001498 data['_oid'] = oid
1499 try:
1500 lockfiles = self.client.get_children(path + '/lock')
1501 except kazoo.exceptions.NoNodeError:
1502 lockfiles = []
1503 if lockfiles:
1504 data['_lock'] = True
1505 else:
1506 data['_lock'] = False
1507 nodes.append(data)
1508 return nodes
1509
James E. Blaira38c28e2017-01-04 10:33:20 -08001510 def makeNode(self, request_id, node_type):
1511 now = time.time()
1512 path = '/nodepool/nodes/'
1513 data = dict(type=node_type,
1514 provider='test-provider',
1515 region='test-region',
Paul Belanger30ba93a2017-03-16 16:28:10 -04001516 az='test-az',
Monty Taylor56f61332017-04-11 05:38:12 -05001517 interface_ip='127.0.0.1',
James E. Blaira38c28e2017-01-04 10:33:20 -08001518 public_ipv4='127.0.0.1',
1519 private_ipv4=None,
1520 public_ipv6=None,
1521 allocated_to=request_id,
1522 state='ready',
1523 state_time=now,
1524 created_time=now,
1525 updated_time=now,
1526 image_id=None,
Paul Belangerc5bf3752017-03-16 19:38:43 -04001527 host_keys=["fake-key1", "fake-key2"],
Paul Belanger174a8272017-03-14 13:20:10 -04001528 executor='fake-nodepool')
Clint Byrumf322fe22017-05-10 20:53:12 -07001529 data = json.dumps(data).encode('utf8')
James E. Blaira38c28e2017-01-04 10:33:20 -08001530 path = self.client.create(path, data,
1531 makepath=True,
1532 sequence=True)
1533 nodeid = path.split("/")[-1]
1534 return nodeid
1535
James E. Blair6ab79e02017-01-06 10:10:17 -08001536 def addFailRequest(self, request):
1537 self.fail_requests.add(request['_oid'])
1538
James E. Blairdce6cea2016-12-20 16:45:32 -08001539 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -08001540 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -08001541 return
1542 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -08001543 oid = request['_oid']
1544 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -08001545
James E. Blair6ab79e02017-01-06 10:10:17 -08001546 if oid in self.fail_requests:
1547 request['state'] = 'failed'
1548 else:
1549 request['state'] = 'fulfilled'
1550 nodes = []
1551 for node in request['node_types']:
1552 nodeid = self.makeNode(oid, node)
1553 nodes.append(nodeid)
1554 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -08001555
James E. Blaira38c28e2017-01-04 10:33:20 -08001556 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -08001557 path = self.REQUEST_ROOT + '/' + oid
Clint Byrumf322fe22017-05-10 20:53:12 -07001558 data = json.dumps(request).encode('utf8')
James E. Blairdce6cea2016-12-20 16:45:32 -08001559 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
James E. Blaircbbce0d2017-05-19 07:28:29 -07001560 try:
1561 self.client.set(path, data)
1562 except kazoo.exceptions.NoNodeError:
1563 self.log.debug("Node request %s %s disappeared" % (oid, data))
James E. Blairdce6cea2016-12-20 16:45:32 -08001564
1565
James E. Blair498059b2016-12-20 13:50:13 -08001566class ChrootedKazooFixture(fixtures.Fixture):
Clark Boylan621ec9a2017-04-07 17:41:33 -07001567 def __init__(self, test_id):
James E. Blair498059b2016-12-20 13:50:13 -08001568 super(ChrootedKazooFixture, self).__init__()
1569
1570 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1571 if ':' in zk_host:
1572 host, port = zk_host.split(':')
1573 else:
1574 host = zk_host
1575 port = None
1576
1577 self.zookeeper_host = host
1578
1579 if not port:
1580 self.zookeeper_port = 2181
1581 else:
1582 self.zookeeper_port = int(port)
1583
Clark Boylan621ec9a2017-04-07 17:41:33 -07001584 self.test_id = test_id
1585
James E. Blair498059b2016-12-20 13:50:13 -08001586 def _setUp(self):
1587 # Make sure the test chroot paths do not conflict
1588 random_bits = ''.join(random.choice(string.ascii_lowercase +
1589 string.ascii_uppercase)
1590 for x in range(8))
1591
Clark Boylan621ec9a2017-04-07 17:41:33 -07001592 rand_test_path = '%s_%s_%s' % (random_bits, os.getpid(), self.test_id)
James E. Blair498059b2016-12-20 13:50:13 -08001593 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1594
Clark Boylan7f5f1ec2017-04-07 17:39:56 -07001595 self.addCleanup(self._cleanup)
1596
James E. Blair498059b2016-12-20 13:50:13 -08001597 # Ensure the chroot path exists and clean up any pre-existing znodes.
1598 _tmp_client = kazoo.client.KazooClient(
1599 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1600 _tmp_client.start()
1601
1602 if _tmp_client.exists(self.zookeeper_chroot):
1603 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1604
1605 _tmp_client.ensure_path(self.zookeeper_chroot)
1606 _tmp_client.stop()
1607 _tmp_client.close()
1608
James E. Blair498059b2016-12-20 13:50:13 -08001609 def _cleanup(self):
1610 '''Remove the chroot path.'''
1611 # Need a non-chroot'ed client to remove the chroot path
1612 _tmp_client = kazoo.client.KazooClient(
1613 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1614 _tmp_client.start()
1615 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1616 _tmp_client.stop()
Clark Boylan7f5f1ec2017-04-07 17:39:56 -07001617 _tmp_client.close()
James E. Blair498059b2016-12-20 13:50:13 -08001618
1619
Joshua Heskethd78b4482015-09-14 16:56:34 -06001620class MySQLSchemaFixture(fixtures.Fixture):
1621 def setUp(self):
1622 super(MySQLSchemaFixture, self).setUp()
1623
1624 random_bits = ''.join(random.choice(string.ascii_lowercase +
1625 string.ascii_uppercase)
1626 for x in range(8))
1627 self.name = '%s_%s' % (random_bits, os.getpid())
1628 self.passwd = uuid.uuid4().hex
1629 db = pymysql.connect(host="localhost",
1630 user="openstack_citest",
1631 passwd="openstack_citest",
1632 db="openstack_citest")
1633 cur = db.cursor()
1634 cur.execute("create database %s" % self.name)
1635 cur.execute(
1636 "grant all on %s.* to '%s'@'localhost' identified by '%s'" %
1637 (self.name, self.name, self.passwd))
1638 cur.execute("flush privileges")
1639
1640 self.dburi = 'mysql+pymysql://%s:%s@localhost/%s' % (self.name,
1641 self.passwd,
1642 self.name)
1643 self.addDetail('dburi', testtools.content.text_content(self.dburi))
1644 self.addCleanup(self.cleanup)
1645
1646 def cleanup(self):
1647 db = pymysql.connect(host="localhost",
1648 user="openstack_citest",
1649 passwd="openstack_citest",
1650 db="openstack_citest")
1651 cur = db.cursor()
1652 cur.execute("drop database %s" % self.name)
1653 cur.execute("drop user '%s'@'localhost'" % self.name)
1654 cur.execute("flush privileges")
1655
1656
Maru Newby3fe5f852015-01-13 04:22:14 +00001657class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001658 log = logging.getLogger("zuul.test")
James E. Blair267e5162017-04-07 10:08:20 -07001659 wait_timeout = 30
Clark Boylanb640e052014-04-03 16:41:46 -07001660
James E. Blair1c236df2017-02-01 14:07:24 -08001661 def attachLogs(self, *args):
1662 def reader():
1663 self._log_stream.seek(0)
1664 while True:
1665 x = self._log_stream.read(4096)
1666 if not x:
1667 break
1668 yield x.encode('utf8')
1669 content = testtools.content.content_from_reader(
1670 reader,
1671 testtools.content_type.UTF8_TEXT,
1672 False)
1673 self.addDetail('logging', content)
1674
Clark Boylanb640e052014-04-03 16:41:46 -07001675 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001676 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001677 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1678 try:
1679 test_timeout = int(test_timeout)
1680 except ValueError:
1681 # If timeout value is invalid do not set a timeout.
1682 test_timeout = 0
1683 if test_timeout > 0:
1684 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1685
1686 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1687 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1688 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1689 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1690 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1691 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1692 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1693 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1694 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1695 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001696 self._log_stream = StringIO()
1697 self.addOnException(self.attachLogs)
1698 else:
1699 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001700
James E. Blair73b41772017-05-22 13:22:55 -07001701 # NOTE(jeblair): this is temporary extra debugging to try to
1702 # track down a possible leak.
1703 orig_git_repo_init = git.Repo.__init__
1704
1705 def git_repo_init(myself, *args, **kw):
1706 orig_git_repo_init(myself, *args, **kw)
1707 self.log.debug("Created git repo 0x%x %s" %
1708 (id(myself), repr(myself)))
1709
1710 self.useFixture(fixtures.MonkeyPatch('git.Repo.__init__',
1711 git_repo_init))
1712
James E. Blair1c236df2017-02-01 14:07:24 -08001713 handler = logging.StreamHandler(self._log_stream)
1714 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1715 '%(levelname)-8s %(message)s')
1716 handler.setFormatter(formatter)
1717
1718 logger = logging.getLogger()
1719 logger.setLevel(logging.DEBUG)
1720 logger.addHandler(handler)
1721
Clark Boylan3410d532017-04-25 12:35:29 -07001722 # Make sure we don't carry old handlers around in process state
1723 # which slows down test runs
1724 self.addCleanup(logger.removeHandler, handler)
1725 self.addCleanup(handler.close)
1726 self.addCleanup(handler.flush)
1727
James E. Blair1c236df2017-02-01 14:07:24 -08001728 # NOTE(notmorgan): Extract logging overrides for specific
1729 # libraries from the OS_LOG_DEFAULTS env and create loggers
1730 # for each. This is used to limit the output during test runs
1731 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001732 log_defaults_from_env = os.environ.get(
1733 'OS_LOG_DEFAULTS',
Monty Taylor73db6492017-05-18 17:31:17 -05001734 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO,paste=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001735
James E. Blairdce6cea2016-12-20 16:45:32 -08001736 if log_defaults_from_env:
1737 for default in log_defaults_from_env.split(','):
1738 try:
1739 name, level_str = default.split('=', 1)
1740 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001741 logger = logging.getLogger(name)
1742 logger.setLevel(level)
1743 logger.addHandler(handler)
1744 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001745 except ValueError:
1746 # NOTE(notmorgan): Invalid format of the log default,
1747 # skip and don't try and apply a logger for the
1748 # specified module
1749 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001750
Maru Newby3fe5f852015-01-13 04:22:14 +00001751
1752class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001753 """A test case with a functioning Zuul.
1754
1755 The following class variables are used during test setup and can
1756 be overidden by subclasses but are effectively read-only once a
1757 test method starts running:
1758
1759 :cvar str config_file: This points to the main zuul config file
1760 within the fixtures directory. Subclasses may override this
1761 to obtain a different behavior.
1762
1763 :cvar str tenant_config_file: This is the tenant config file
1764 (which specifies from what git repos the configuration should
1765 be loaded). It defaults to the value specified in
1766 `config_file` but can be overidden by subclasses to obtain a
1767 different tenant/project layout while using the standard main
James E. Blair06cc3922017-04-19 10:08:10 -07001768 configuration. See also the :py:func:`simple_layout`
1769 decorator.
James E. Blaire7b99a02016-08-05 14:27:34 -07001770
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001771 :cvar bool create_project_keys: Indicates whether Zuul should
1772 auto-generate keys for each project, or whether the test
1773 infrastructure should insert dummy keys to save time during
1774 startup. Defaults to False.
1775
James E. Blaire7b99a02016-08-05 14:27:34 -07001776 The following are instance variables that are useful within test
1777 methods:
1778
1779 :ivar FakeGerritConnection fake_<connection>:
1780 A :py:class:`~tests.base.FakeGerritConnection` will be
1781 instantiated for each connection present in the config file
1782 and stored here. For instance, `fake_gerrit` will hold the
1783 FakeGerritConnection object for a connection named `gerrit`.
1784
1785 :ivar FakeGearmanServer gearman_server: An instance of
1786 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1787 server that all of the Zuul components in this test use to
1788 communicate with each other.
1789
Paul Belanger174a8272017-03-14 13:20:10 -04001790 :ivar RecordingExecutorServer executor_server: An instance of
1791 :py:class:`~tests.base.RecordingExecutorServer` which is the
1792 Ansible execute server used to run jobs for this test.
James E. Blaire7b99a02016-08-05 14:27:34 -07001793
1794 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1795 representing currently running builds. They are appended to
Paul Belanger174a8272017-03-14 13:20:10 -04001796 the list in the order they are executed, and removed from this
James E. Blaire7b99a02016-08-05 14:27:34 -07001797 list upon completion.
1798
1799 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1800 objects representing completed builds. They are appended to
1801 the list in the order they complete.
1802
1803 """
1804
James E. Blair83005782015-12-11 14:46:03 -08001805 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001806 run_ansible = False
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001807 create_project_keys = False
James E. Blair3f876d52016-07-22 13:07:14 -07001808
1809 def _startMerger(self):
1810 self.merge_server = zuul.merger.server.MergeServer(self.config,
1811 self.connections)
1812 self.merge_server.start()
1813
Maru Newby3fe5f852015-01-13 04:22:14 +00001814 def setUp(self):
1815 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001816
1817 self.setupZK()
1818
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001819 if not KEEP_TEMPDIRS:
James E. Blair97d902e2014-08-21 13:25:56 -07001820 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001821 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1822 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001823 else:
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001824 tmp_root = tempfile.mkdtemp(
1825 dir=os.environ.get("ZUUL_TEST_ROOT", None))
Clark Boylanb640e052014-04-03 16:41:46 -07001826 self.test_root = os.path.join(tmp_root, "zuul-test")
1827 self.upstream_root = os.path.join(self.test_root, "upstream")
Monty Taylord642d852017-02-23 14:05:42 -05001828 self.merger_src_root = os.path.join(self.test_root, "merger-git")
Paul Belanger174a8272017-03-14 13:20:10 -04001829 self.executor_src_root = os.path.join(self.test_root, "executor-git")
James E. Blairce8a2132016-05-19 15:21:52 -07001830 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001831
1832 if os.path.exists(self.test_root):
1833 shutil.rmtree(self.test_root)
1834 os.makedirs(self.test_root)
1835 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001836 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001837
1838 # Make per test copy of Configuration.
1839 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001840 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001841 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001842 self.config.get('zuul', 'tenant_config')))
Monty Taylord642d852017-02-23 14:05:42 -05001843 self.config.set('merger', 'git_dir', self.merger_src_root)
Paul Belanger174a8272017-03-14 13:20:10 -04001844 self.config.set('executor', 'git_dir', self.executor_src_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001845 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001846
Clark Boylanb640e052014-04-03 16:41:46 -07001847 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001848 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1849 # see: https://github.com/jsocol/pystatsd/issues/61
1850 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001851 os.environ['STATSD_PORT'] = str(self.statsd.port)
1852 self.statsd.start()
1853 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001854 reload_module(statsd)
1855 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001856
1857 self.gearman_server = FakeGearmanServer()
1858
1859 self.config.set('gearman', 'port', str(self.gearman_server.port))
James E. Blaire47eb772017-02-02 17:19:40 -08001860 self.log.info("Gearman server on port %s" %
1861 (self.gearman_server.port,))
Clark Boylanb640e052014-04-03 16:41:46 -07001862
James E. Blaire511d2f2016-12-08 15:22:26 -08001863 gerritsource.GerritSource.replication_timeout = 1.5
1864 gerritsource.GerritSource.replication_retry_interval = 0.5
1865 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001866
Joshua Hesketh352264b2015-08-11 23:42:08 +10001867 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001868
Jan Hruban7083edd2015-08-21 14:00:54 +02001869 self.webapp = zuul.webapp.WebApp(
1870 self.sched, port=0, listen_address='127.0.0.1')
1871
Jan Hruban6b71aff2015-10-22 16:58:08 +02001872 self.event_queues = [
1873 self.sched.result_event_queue,
James E. Blair646322f2017-01-27 15:50:34 -08001874 self.sched.trigger_event_queue,
1875 self.sched.management_event_queue
Jan Hruban6b71aff2015-10-22 16:58:08 +02001876 ]
1877
James E. Blairfef78942016-03-11 16:28:56 -08001878 self.configure_connections()
Jan Hruban7083edd2015-08-21 14:00:54 +02001879 self.sched.registerConnections(self.connections, self.webapp)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001880
Clark Boylanb640e052014-04-03 16:41:46 -07001881 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001882 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001883 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001884 return FakeURLOpener(self.upstream_root, *args, **kw)
1885
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001886 old_urlopen = urllib.request.urlopen
1887 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001888
Paul Belanger174a8272017-03-14 13:20:10 -04001889 self.executor_server = RecordingExecutorServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001890 self.config, self.connections,
James E. Blair854f8892017-02-02 11:25:39 -08001891 jobdir_root=self.test_root,
James E. Blaira92cbc82017-01-23 14:56:49 -08001892 _run_ansible=self.run_ansible,
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001893 _test_root=self.test_root,
1894 keep_jobdir=KEEP_TEMPDIRS)
Paul Belanger174a8272017-03-14 13:20:10 -04001895 self.executor_server.start()
1896 self.history = self.executor_server.build_history
1897 self.builds = self.executor_server.running_builds
James E. Blaire1767bc2016-08-02 10:00:27 -07001898
Paul Belanger174a8272017-03-14 13:20:10 -04001899 self.executor_client = zuul.executor.client.ExecutorClient(
James E. Blair92e953a2017-03-07 13:08:47 -08001900 self.config, self.sched)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001901 self.merge_client = zuul.merger.client.MergeClient(
1902 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001903 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001904 self.zk = zuul.zk.ZooKeeper()
James E. Blair0d5a36e2017-02-21 10:53:44 -05001905 self.zk.connect(self.zk_config)
James E. Blairdce6cea2016-12-20 16:45:32 -08001906
James E. Blair0d5a36e2017-02-21 10:53:44 -05001907 self.fake_nodepool = FakeNodepool(
1908 self.zk_chroot_fixture.zookeeper_host,
1909 self.zk_chroot_fixture.zookeeper_port,
1910 self.zk_chroot_fixture.zookeeper_chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001911
Paul Belanger174a8272017-03-14 13:20:10 -04001912 self.sched.setExecutor(self.executor_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001913 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001914 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001915 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001916
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001917 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001918
1919 self.sched.start()
Clark Boylanb640e052014-04-03 16:41:46 -07001920 self.webapp.start()
1921 self.rpc.start()
Paul Belanger174a8272017-03-14 13:20:10 -04001922 self.executor_client.gearman.waitForServer()
James E. Blaira002b032017-04-18 10:35:48 -07001923 # Cleanups are run in reverse order
1924 self.addCleanup(self.assertCleanShutdown)
Clark Boylanb640e052014-04-03 16:41:46 -07001925 self.addCleanup(self.shutdown)
James E. Blaira002b032017-04-18 10:35:48 -07001926 self.addCleanup(self.assertFinalState)
Clark Boylanb640e052014-04-03 16:41:46 -07001927
James E. Blairb9c0d772017-03-03 14:34:49 -08001928 self.sched.reconfigure(self.config)
1929 self.sched.resume()
1930
Tobias Henkel7df274b2017-05-26 17:41:11 +02001931 def configure_connections(self, source_only=False):
James E. Blaire511d2f2016-12-08 15:22:26 -08001932 # Set up gerrit related fakes
1933 # Set a changes database so multiple FakeGerrit's can report back to
1934 # a virtual canonical database given by the configured hostname
1935 self.gerrit_changes_dbs = {}
1936
1937 def getGerritConnection(driver, name, config):
1938 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1939 con = FakeGerritConnection(driver, name, config,
1940 changes_db=db,
1941 upstream_root=self.upstream_root)
1942 self.event_queues.append(con.event_queue)
1943 setattr(self, 'fake_' + name, con)
1944 return con
1945
1946 self.useFixture(fixtures.MonkeyPatch(
1947 'zuul.driver.gerrit.GerritDriver.getConnection',
1948 getGerritConnection))
1949
Gregory Haynes4fc12542015-04-22 20:38:06 -07001950 def getGithubConnection(driver, name, config):
1951 con = FakeGithubConnection(driver, name, config,
1952 upstream_root=self.upstream_root)
1953 setattr(self, 'fake_' + name, con)
1954 return con
1955
1956 self.useFixture(fixtures.MonkeyPatch(
1957 'zuul.driver.github.GithubDriver.getConnection',
1958 getGithubConnection))
1959
James E. Blaire511d2f2016-12-08 15:22:26 -08001960 # Set up smtp related fakes
Joshua Heskethd78b4482015-09-14 16:56:34 -06001961 # TODO(jhesketh): This should come from lib.connections for better
1962 # coverage
Joshua Hesketh352264b2015-08-11 23:42:08 +10001963 # Register connections from the config
1964 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001965
Joshua Hesketh352264b2015-08-11 23:42:08 +10001966 def FakeSMTPFactory(*args, **kw):
1967 args = [self.smtp_messages] + list(args)
1968 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001969
Joshua Hesketh352264b2015-08-11 23:42:08 +10001970 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001971
James E. Blaire511d2f2016-12-08 15:22:26 -08001972 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001973 self.connections = zuul.lib.connections.ConnectionRegistry()
Tobias Henkel7df274b2017-05-26 17:41:11 +02001974 self.connections.configure(self.config, source_only=source_only)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001975
James E. Blair83005782015-12-11 14:46:03 -08001976 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001977 # This creates the per-test configuration object. It can be
1978 # overriden by subclasses, but should not need to be since it
1979 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001980 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001981 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair06cc3922017-04-19 10:08:10 -07001982
1983 if not self.setupSimpleLayout():
1984 if hasattr(self, 'tenant_config_file'):
1985 self.config.set('zuul', 'tenant_config',
1986 self.tenant_config_file)
1987 git_path = os.path.join(
1988 os.path.dirname(
1989 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1990 'git')
1991 if os.path.exists(git_path):
1992 for reponame in os.listdir(git_path):
1993 project = reponame.replace('_', '/')
1994 self.copyDirToRepo(project,
1995 os.path.join(git_path, reponame))
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001996 self.setupAllProjectKeys()
1997
James E. Blair06cc3922017-04-19 10:08:10 -07001998 def setupSimpleLayout(self):
1999 # If the test method has been decorated with a simple_layout,
2000 # use that instead of the class tenant_config_file. Set up a
2001 # single config-project with the specified layout, and
2002 # initialize repos for all of the 'project' entries which
2003 # appear in the layout.
2004 test_name = self.id().split('.')[-1]
2005 test = getattr(self, test_name)
2006 if hasattr(test, '__simple_layout__'):
Jesse Keating436a5452017-04-20 11:48:41 -07002007 path, driver = getattr(test, '__simple_layout__')
James E. Blair06cc3922017-04-19 10:08:10 -07002008 else:
2009 return False
2010
James E. Blairb70e55a2017-04-19 12:57:02 -07002011 files = {}
James E. Blair06cc3922017-04-19 10:08:10 -07002012 path = os.path.join(FIXTURE_DIR, path)
2013 with open(path) as f:
James E. Blairb70e55a2017-04-19 12:57:02 -07002014 data = f.read()
2015 layout = yaml.safe_load(data)
2016 files['zuul.yaml'] = data
James E. Blair06cc3922017-04-19 10:08:10 -07002017 untrusted_projects = []
2018 for item in layout:
2019 if 'project' in item:
2020 name = item['project']['name']
2021 untrusted_projects.append(name)
2022 self.init_repo(name)
2023 self.addCommitToRepo(name, 'initial commit',
2024 files={'README': ''},
2025 branch='master', tag='init')
James E. Blairb70e55a2017-04-19 12:57:02 -07002026 if 'job' in item:
2027 jobname = item['job']['name']
2028 files['playbooks/%s.yaml' % jobname] = ''
James E. Blair06cc3922017-04-19 10:08:10 -07002029
2030 root = os.path.join(self.test_root, "config")
2031 if not os.path.exists(root):
2032 os.makedirs(root)
2033 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
2034 config = [{'tenant':
2035 {'name': 'tenant-one',
Jesse Keating436a5452017-04-20 11:48:41 -07002036 'source': {driver:
James E. Blair06cc3922017-04-19 10:08:10 -07002037 {'config-projects': ['common-config'],
2038 'untrusted-projects': untrusted_projects}}}}]
Clint Byrumd52b7d72017-05-10 19:40:35 -07002039 f.write(yaml.dump(config).encode('utf8'))
James E. Blair06cc3922017-04-19 10:08:10 -07002040 f.close()
2041 self.config.set('zuul', 'tenant_config',
2042 os.path.join(FIXTURE_DIR, f.name))
2043
2044 self.init_repo('common-config')
James E. Blair06cc3922017-04-19 10:08:10 -07002045 self.addCommitToRepo('common-config', 'add content from fixture',
2046 files, branch='master', tag='init')
2047
2048 return True
2049
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002050 def setupAllProjectKeys(self):
2051 if self.create_project_keys:
2052 return
2053
2054 path = self.config.get('zuul', 'tenant_config')
2055 with open(os.path.join(FIXTURE_DIR, path)) as f:
2056 tenant_config = yaml.safe_load(f.read())
2057 for tenant in tenant_config:
2058 sources = tenant['tenant']['source']
2059 for source, conf in sources.items():
James E. Blair109da3f2017-04-04 14:39:43 -07002060 for project in conf.get('config-projects', []):
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002061 self.setupProjectKeys(source, project)
James E. Blair109da3f2017-04-04 14:39:43 -07002062 for project in conf.get('untrusted-projects', []):
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002063 self.setupProjectKeys(source, project)
2064
2065 def setupProjectKeys(self, source, project):
2066 # Make sure we set up an RSA key for the project so that we
2067 # don't spend time generating one:
2068
2069 key_root = os.path.join(self.state_root, 'keys')
2070 if not os.path.isdir(key_root):
2071 os.mkdir(key_root, 0o700)
2072 private_key_file = os.path.join(key_root, source, project + '.pem')
2073 private_key_dir = os.path.dirname(private_key_file)
2074 self.log.debug("Installing test keys for project %s at %s" % (
2075 project, private_key_file))
2076 if not os.path.isdir(private_key_dir):
2077 os.makedirs(private_key_dir)
2078 with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
2079 with open(private_key_file, 'w') as o:
2080 o.write(i.read())
James E. Blair96c6bf82016-01-15 16:20:40 -08002081
James E. Blair498059b2016-12-20 13:50:13 -08002082 def setupZK(self):
Clark Boylan621ec9a2017-04-07 17:41:33 -07002083 self.zk_chroot_fixture = self.useFixture(
2084 ChrootedKazooFixture(self.id()))
James E. Blair0d5a36e2017-02-21 10:53:44 -05002085 self.zk_config = '%s:%s%s' % (
James E. Blairdce6cea2016-12-20 16:45:32 -08002086 self.zk_chroot_fixture.zookeeper_host,
2087 self.zk_chroot_fixture.zookeeper_port,
2088 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08002089
James E. Blair96c6bf82016-01-15 16:20:40 -08002090 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07002091 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08002092
2093 files = {}
2094 for (dirpath, dirnames, filenames) in os.walk(source_path):
2095 for filename in filenames:
2096 test_tree_filepath = os.path.join(dirpath, filename)
2097 common_path = os.path.commonprefix([test_tree_filepath,
2098 source_path])
2099 relative_filepath = test_tree_filepath[len(common_path) + 1:]
2100 with open(test_tree_filepath, 'r') as f:
2101 content = f.read()
2102 files[relative_filepath] = content
2103 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07002104 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08002105
James E. Blaire18d4602017-01-05 11:17:28 -08002106 def assertNodepoolState(self):
2107 # Make sure that there are no pending requests
2108
2109 requests = self.fake_nodepool.getNodeRequests()
2110 self.assertEqual(len(requests), 0)
2111
2112 nodes = self.fake_nodepool.getNodes()
2113 for node in nodes:
2114 self.assertFalse(node['_lock'], "Node %s is locked" %
2115 (node['_oid'],))
2116
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002117 def assertNoGeneratedKeys(self):
2118 # Make sure that Zuul did not generate any project keys
2119 # (unless it was supposed to).
2120
2121 if self.create_project_keys:
2122 return
2123
2124 with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
2125 test_key = i.read()
2126
2127 key_root = os.path.join(self.state_root, 'keys')
2128 for root, dirname, files in os.walk(key_root):
2129 for fn in files:
2130 with open(os.path.join(root, fn)) as f:
2131 self.assertEqual(test_key, f.read())
2132
Clark Boylanb640e052014-04-03 16:41:46 -07002133 def assertFinalState(self):
James E. Blaira002b032017-04-18 10:35:48 -07002134 self.log.debug("Assert final state")
2135 # Make sure no jobs are running
2136 self.assertEqual({}, self.executor_server.job_workers)
Clark Boylanb640e052014-04-03 16:41:46 -07002137 # Make sure that git.Repo objects have been garbage collected.
2138 repos = []
James E. Blair73b41772017-05-22 13:22:55 -07002139 gc.disable()
Clark Boylanb640e052014-04-03 16:41:46 -07002140 gc.collect()
2141 for obj in gc.get_objects():
2142 if isinstance(obj, git.Repo):
James E. Blair73b41772017-05-22 13:22:55 -07002143 self.log.debug("Leaked git repo object: 0x%x %s" %
2144 (id(obj), repr(obj)))
2145 for ref in gc.get_referrers(obj):
2146 self.log.debug(" Referrer %s" % (repr(ref)))
Clark Boylanb640e052014-04-03 16:41:46 -07002147 repos.append(obj)
James E. Blair73b41772017-05-22 13:22:55 -07002148 if repos:
2149 for obj in gc.garbage:
2150 self.log.debug(" Garbage %s" % (repr(obj)))
2151 gc.enable()
Clark Boylanb640e052014-04-03 16:41:46 -07002152 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08002153 self.assertNodepoolState()
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002154 self.assertNoGeneratedKeys()
James E. Blair83005782015-12-11 14:46:03 -08002155 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08002156 for tenant in self.sched.abide.tenants.values():
2157 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08002158 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08002159 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07002160
2161 def shutdown(self):
2162 self.log.debug("Shutting down after tests")
Paul Belanger174a8272017-03-14 13:20:10 -04002163 self.executor_client.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07002164 self.merge_client.stop()
Paul Belanger174a8272017-03-14 13:20:10 -04002165 self.executor_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07002166 self.sched.stop()
2167 self.sched.join()
2168 self.statsd.stop()
2169 self.statsd.join()
2170 self.webapp.stop()
2171 self.webapp.join()
2172 self.rpc.stop()
2173 self.rpc.join()
2174 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08002175 self.fake_nodepool.stop()
2176 self.zk.disconnect()
Clark Boylan8208c192017-04-24 18:08:08 -07002177 self.printHistory()
Clark Boylanf18e3b82017-04-24 17:34:13 -07002178 # we whitelist watchdog threads as they have relatively long delays
2179 # before noticing they should exit, but they should exit on their own.
2180 threads = [t for t in threading.enumerate()
2181 if t.name != 'executor-watchdog']
Clark Boylanb640e052014-04-03 16:41:46 -07002182 if len(threads) > 1:
Clark Boylan8208c192017-04-24 18:08:08 -07002183 log_str = ""
2184 for thread_id, stack_frame in sys._current_frames().items():
2185 log_str += "Thread: %s\n" % thread_id
2186 log_str += "".join(traceback.format_stack(stack_frame))
2187 self.log.debug(log_str)
2188 raise Exception("More than one thread is running: %s" % threads)
Clark Boylanb640e052014-04-03 16:41:46 -07002189
James E. Blaira002b032017-04-18 10:35:48 -07002190 def assertCleanShutdown(self):
2191 pass
2192
James E. Blairc4ba97a2017-04-19 16:26:24 -07002193 def init_repo(self, project, tag=None):
Clark Boylanb640e052014-04-03 16:41:46 -07002194 parts = project.split('/')
2195 path = os.path.join(self.upstream_root, *parts[:-1])
2196 if not os.path.exists(path):
2197 os.makedirs(path)
2198 path = os.path.join(self.upstream_root, project)
2199 repo = git.Repo.init(path)
2200
Morgan Fainberg78c301a2016-07-14 13:47:01 -07002201 with repo.config_writer() as config_writer:
2202 config_writer.set_value('user', 'email', 'user@example.com')
2203 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07002204
Clark Boylanb640e052014-04-03 16:41:46 -07002205 repo.index.commit('initial commit')
2206 master = repo.create_head('master')
James E. Blairc4ba97a2017-04-19 16:26:24 -07002207 if tag:
2208 repo.create_tag(tag)
Clark Boylanb640e052014-04-03 16:41:46 -07002209
James E. Blair97d902e2014-08-21 13:25:56 -07002210 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07002211 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07002212 repo.git.clean('-x', '-f', '-d')
2213
James E. Blair97d902e2014-08-21 13:25:56 -07002214 def create_branch(self, project, branch):
2215 path = os.path.join(self.upstream_root, project)
2216 repo = git.Repo.init(path)
2217 fn = os.path.join(path, 'README')
2218
2219 branch_head = repo.create_head(branch)
2220 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07002221 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07002222 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07002223 f.close()
2224 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07002225 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07002226
James E. Blair97d902e2014-08-21 13:25:56 -07002227 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07002228 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07002229 repo.git.clean('-x', '-f', '-d')
2230
Sachi King9f16d522016-03-16 12:20:45 +11002231 def create_commit(self, project):
2232 path = os.path.join(self.upstream_root, project)
2233 repo = git.Repo(path)
2234 repo.head.reference = repo.heads['master']
2235 file_name = os.path.join(path, 'README')
2236 with open(file_name, 'a') as f:
2237 f.write('creating fake commit\n')
2238 repo.index.add([file_name])
2239 commit = repo.index.commit('Creating a fake commit')
2240 return commit.hexsha
2241
James E. Blairf4a5f022017-04-18 14:01:10 -07002242 def orderedRelease(self, count=None):
James E. Blairb8c16472015-05-05 14:55:26 -07002243 # Run one build at a time to ensure non-race order:
James E. Blairf4a5f022017-04-18 14:01:10 -07002244 i = 0
James E. Blairb8c16472015-05-05 14:55:26 -07002245 while len(self.builds):
2246 self.release(self.builds[0])
2247 self.waitUntilSettled()
James E. Blairf4a5f022017-04-18 14:01:10 -07002248 i += 1
2249 if count is not None and i >= count:
2250 break
James E. Blairb8c16472015-05-05 14:55:26 -07002251
Clark Boylanb640e052014-04-03 16:41:46 -07002252 def release(self, job):
2253 if isinstance(job, FakeBuild):
2254 job.release()
2255 else:
2256 job.waiting = False
2257 self.log.debug("Queued job %s released" % job.unique)
2258 self.gearman_server.wakeConnections()
2259
2260 def getParameter(self, job, name):
2261 if isinstance(job, FakeBuild):
2262 return job.parameters[name]
2263 else:
2264 parameters = json.loads(job.arguments)
2265 return parameters[name]
2266
Clark Boylanb640e052014-04-03 16:41:46 -07002267 def haveAllBuildsReported(self):
2268 # See if Zuul is waiting on a meta job to complete
Paul Belanger174a8272017-03-14 13:20:10 -04002269 if self.executor_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07002270 return False
2271 # Find out if every build that the worker has completed has been
2272 # reported back to Zuul. If it hasn't then that means a Gearman
2273 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07002274 for build in self.history:
Paul Belanger174a8272017-03-14 13:20:10 -04002275 zbuild = self.executor_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07002276 if not zbuild:
2277 # It has already been reported
2278 continue
2279 # It hasn't been reported yet.
2280 return False
2281 # Make sure that none of the worker connections are in GRAB_WAIT
Paul Belanger174a8272017-03-14 13:20:10 -04002282 for connection in self.executor_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07002283 if connection.state == 'GRAB_WAIT':
2284 return False
2285 return True
2286
2287 def areAllBuildsWaiting(self):
Paul Belanger174a8272017-03-14 13:20:10 -04002288 builds = self.executor_client.builds.values()
James E. Blaira002b032017-04-18 10:35:48 -07002289 seen_builds = set()
Clark Boylanb640e052014-04-03 16:41:46 -07002290 for build in builds:
James E. Blaira002b032017-04-18 10:35:48 -07002291 seen_builds.add(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07002292 client_job = None
Paul Belanger174a8272017-03-14 13:20:10 -04002293 for conn in self.executor_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07002294 for j in conn.related_jobs.values():
2295 if j.unique == build.uuid:
2296 client_job = j
2297 break
2298 if not client_job:
2299 self.log.debug("%s is not known to the gearman client" %
2300 build)
James E. Blairf15139b2015-04-02 16:37:15 -07002301 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002302 if not client_job.handle:
2303 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002304 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002305 server_job = self.gearman_server.jobs.get(client_job.handle)
2306 if not server_job:
2307 self.log.debug("%s is not known to the gearman server" %
2308 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002309 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002310 if not hasattr(server_job, 'waiting'):
2311 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002312 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002313 if server_job.waiting:
2314 continue
James E. Blair17302972016-08-10 16:11:42 -07002315 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08002316 self.log.debug("%s has not reported start" % build)
2317 return False
Clint Byrumf322fe22017-05-10 20:53:12 -07002318 # using internal ServerJob which offers no Text interface
Paul Belanger174a8272017-03-14 13:20:10 -04002319 worker_build = self.executor_server.job_builds.get(
Clint Byrumf322fe22017-05-10 20:53:12 -07002320 server_job.unique.decode('utf8'))
James E. Blair962220f2016-08-03 11:22:38 -07002321 if worker_build:
2322 if worker_build.isWaiting():
2323 continue
2324 else:
2325 self.log.debug("%s is running" % worker_build)
2326 return False
Clark Boylanb640e052014-04-03 16:41:46 -07002327 else:
James E. Blair962220f2016-08-03 11:22:38 -07002328 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07002329 return False
James E. Blaira002b032017-04-18 10:35:48 -07002330 for (build_uuid, job_worker) in \
2331 self.executor_server.job_workers.items():
2332 if build_uuid not in seen_builds:
2333 self.log.debug("%s is not finalized" % build_uuid)
2334 return False
James E. Blairf15139b2015-04-02 16:37:15 -07002335 return True
Clark Boylanb640e052014-04-03 16:41:46 -07002336
James E. Blairdce6cea2016-12-20 16:45:32 -08002337 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08002338 if self.fake_nodepool.paused:
2339 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08002340 if self.sched.nodepool.requests:
2341 return False
2342 return True
2343
Jan Hruban6b71aff2015-10-22 16:58:08 +02002344 def eventQueuesEmpty(self):
2345 for queue in self.event_queues:
2346 yield queue.empty()
2347
2348 def eventQueuesJoin(self):
2349 for queue in self.event_queues:
2350 queue.join()
2351
Clark Boylanb640e052014-04-03 16:41:46 -07002352 def waitUntilSettled(self):
2353 self.log.debug("Waiting until settled...")
2354 start = time.time()
2355 while True:
Clint Byruma9626572017-02-22 14:04:00 -05002356 if time.time() - start > self.wait_timeout:
James E. Blair10fc1eb2016-12-21 16:16:25 -08002357 self.log.error("Timeout waiting for Zuul to settle")
2358 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07002359 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08002360 self.log.error(" %s: %s" % (queue, queue.empty()))
2361 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07002362 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08002363 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07002364 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08002365 self.log.error("All requests completed: %s" %
2366 (self.areAllNodeRequestsComplete(),))
2367 self.log.error("Merge client jobs: %s" %
2368 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07002369 raise Exception("Timeout waiting for Zuul to settle")
2370 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07002371
Paul Belanger174a8272017-03-14 13:20:10 -04002372 self.executor_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07002373 # have all build states propogated to zuul?
2374 if self.haveAllBuildsReported():
2375 # Join ensures that the queue is empty _and_ events have been
2376 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02002377 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07002378 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08002379 if (not self.merge_client.jobs and
Clark Boylanb640e052014-04-03 16:41:46 -07002380 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08002381 self.areAllBuildsWaiting() and
James E. Blair36c611a2017-02-06 15:59:43 -08002382 self.areAllNodeRequestsComplete() and
2383 all(self.eventQueuesEmpty())):
2384 # The queue empty check is placed at the end to
2385 # ensure that if a component adds an event between
2386 # when locked the run handler and checked that the
2387 # components were stable, we don't erroneously
2388 # report that we are settled.
Clark Boylanb640e052014-04-03 16:41:46 -07002389 self.sched.run_handler_lock.release()
Paul Belanger174a8272017-03-14 13:20:10 -04002390 self.executor_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07002391 self.log.debug("...settled.")
2392 return
2393 self.sched.run_handler_lock.release()
Paul Belanger174a8272017-03-14 13:20:10 -04002394 self.executor_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07002395 self.sched.wake_event.wait(0.1)
2396
2397 def countJobResults(self, jobs, result):
2398 jobs = filter(lambda x: x.result == result, jobs)
Clint Byrum1d0c7d12017-05-10 19:40:53 -07002399 return len(list(jobs))
Clark Boylanb640e052014-04-03 16:41:46 -07002400
James E. Blair96c6bf82016-01-15 16:20:40 -08002401 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07002402 for job in self.history:
2403 if (job.name == name and
2404 (project is None or
2405 job.parameters['ZUUL_PROJECT'] == project)):
2406 return job
Clark Boylanb640e052014-04-03 16:41:46 -07002407 raise Exception("Unable to find job %s in history" % name)
2408
2409 def assertEmptyQueues(self):
2410 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08002411 for tenant in self.sched.abide.tenants.values():
2412 for pipeline in tenant.layout.pipelines.values():
2413 for queue in pipeline.queues:
2414 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10002415 print('pipeline %s queue %s contents %s' % (
2416 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08002417 self.assertEqual(len(queue.queue), 0,
2418 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07002419
2420 def assertReportedStat(self, key, value=None, kind=None):
2421 start = time.time()
2422 while time.time() < (start + 5):
2423 for stat in self.statsd.stats:
Clint Byrumf322fe22017-05-10 20:53:12 -07002424 k, v = stat.decode('utf-8').split(':')
Clark Boylanb640e052014-04-03 16:41:46 -07002425 if key == k:
2426 if value is None and kind is None:
2427 return
2428 elif value:
2429 if value == v:
2430 return
2431 elif kind:
2432 if v.endswith('|' + kind):
2433 return
2434 time.sleep(0.1)
2435
Clark Boylanb640e052014-04-03 16:41:46 -07002436 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08002437
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002438 def assertBuilds(self, builds):
2439 """Assert that the running builds are as described.
2440
2441 The list of running builds is examined and must match exactly
2442 the list of builds described by the input.
2443
2444 :arg list builds: A list of dictionaries. Each item in the
2445 list must match the corresponding build in the build
2446 history, and each element of the dictionary must match the
2447 corresponding attribute of the build.
2448
2449 """
James E. Blair3158e282016-08-19 09:34:11 -07002450 try:
2451 self.assertEqual(len(self.builds), len(builds))
2452 for i, d in enumerate(builds):
2453 for k, v in d.items():
2454 self.assertEqual(
2455 getattr(self.builds[i], k), v,
2456 "Element %i in builds does not match" % (i,))
2457 except Exception:
2458 for build in self.builds:
2459 self.log.error("Running build: %s" % build)
2460 else:
2461 self.log.error("No running builds")
2462 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002463
James E. Blairb536ecc2016-08-31 10:11:42 -07002464 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002465 """Assert that the completed builds are as described.
2466
2467 The list of completed builds is examined and must match
2468 exactly the list of builds described by the input.
2469
2470 :arg list history: A list of dictionaries. Each item in the
2471 list must match the corresponding build in the build
2472 history, and each element of the dictionary must match the
2473 corresponding attribute of the build.
2474
James E. Blairb536ecc2016-08-31 10:11:42 -07002475 :arg bool ordered: If true, the history must match the order
2476 supplied, if false, the builds are permitted to have
2477 arrived in any order.
2478
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002479 """
James E. Blairb536ecc2016-08-31 10:11:42 -07002480 def matches(history_item, item):
2481 for k, v in item.items():
2482 if getattr(history_item, k) != v:
2483 return False
2484 return True
James E. Blair3158e282016-08-19 09:34:11 -07002485 try:
2486 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07002487 if ordered:
2488 for i, d in enumerate(history):
2489 if not matches(self.history[i], d):
2490 raise Exception(
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08002491 "Element %i in history does not match %s" %
2492 (i, self.history[i]))
James E. Blairb536ecc2016-08-31 10:11:42 -07002493 else:
2494 unseen = self.history[:]
2495 for i, d in enumerate(history):
2496 found = False
2497 for unseen_item in unseen:
2498 if matches(unseen_item, d):
2499 found = True
2500 unseen.remove(unseen_item)
2501 break
2502 if not found:
2503 raise Exception("No match found for element %i "
2504 "in history" % (i,))
2505 if unseen:
2506 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07002507 except Exception:
2508 for build in self.history:
2509 self.log.error("Completed build: %s" % build)
2510 else:
2511 self.log.error("No completed builds")
2512 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07002513
James E. Blair6ac368c2016-12-22 18:07:20 -08002514 def printHistory(self):
2515 """Log the build history.
2516
2517 This can be useful during tests to summarize what jobs have
2518 completed.
2519
2520 """
2521 self.log.debug("Build history:")
2522 for build in self.history:
2523 self.log.debug(build)
2524
James E. Blair59fdbac2015-12-07 17:08:06 -08002525 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08002526 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
2527
James E. Blair9ea70072017-04-19 16:05:30 -07002528 def updateConfigLayout(self, path):
James E. Blairf84026c2015-12-08 16:11:46 -08002529 root = os.path.join(self.test_root, "config")
Clint Byrumead6c562017-02-01 16:34:04 -08002530 if not os.path.exists(root):
2531 os.makedirs(root)
James E. Blairf84026c2015-12-08 16:11:46 -08002532 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
2533 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05002534- tenant:
2535 name: openstack
2536 source:
2537 gerrit:
James E. Blair109da3f2017-04-04 14:39:43 -07002538 config-projects:
Paul Belanger66e95962016-11-11 12:11:06 -05002539 - %s
James E. Blair109da3f2017-04-04 14:39:43 -07002540 untrusted-projects:
James E. Blair0ffa0102017-03-30 13:11:33 -07002541 - org/project
2542 - org/project1
James E. Blair7cb84542017-04-19 13:35:05 -07002543 - org/project2\n""" % path)
James E. Blairf84026c2015-12-08 16:11:46 -08002544 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05002545 self.config.set('zuul', 'tenant_config',
2546 os.path.join(FIXTURE_DIR, f.name))
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00002547 self.setupAllProjectKeys()
James E. Blair14abdf42015-12-09 16:11:53 -08002548
James E. Blair8b1dc3f2016-07-05 16:49:00 -07002549 def addCommitToRepo(self, project, message, files,
2550 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08002551 path = os.path.join(self.upstream_root, project)
2552 repo = git.Repo(path)
2553 repo.head.reference = branch
2554 zuul.merger.merger.reset_repo_to_head(repo)
2555 for fn, content in files.items():
2556 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08002557 try:
2558 os.makedirs(os.path.dirname(fn))
2559 except OSError:
2560 pass
James E. Blair14abdf42015-12-09 16:11:53 -08002561 with open(fn, 'w') as f:
2562 f.write(content)
2563 repo.index.add([fn])
2564 commit = repo.index.commit(message)
Clint Byrum58264dc2017-02-07 21:21:22 -08002565 before = repo.heads[branch].commit
James E. Blair14abdf42015-12-09 16:11:53 -08002566 repo.heads[branch].commit = commit
2567 repo.head.reference = branch
2568 repo.git.clean('-x', '-f', '-d')
2569 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07002570 if tag:
2571 repo.create_tag(tag)
Clint Byrum58264dc2017-02-07 21:21:22 -08002572 return before
2573
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002574 def commitConfigUpdate(self, project_name, source_name):
2575 """Commit an update to zuul.yaml
2576
2577 This overwrites the zuul.yaml in the specificed project with
2578 the contents specified.
2579
2580 :arg str project_name: The name of the project containing
2581 zuul.yaml (e.g., common-config)
2582
2583 :arg str source_name: The path to the file (underneath the
2584 test fixture directory) whose contents should be used to
2585 replace zuul.yaml.
2586 """
2587
2588 source_path = os.path.join(FIXTURE_DIR, source_name)
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002589 files = {}
2590 with open(source_path, 'r') as f:
2591 data = f.read()
2592 layout = yaml.safe_load(data)
2593 files['zuul.yaml'] = data
2594 for item in layout:
2595 if 'job' in item:
2596 jobname = item['job']['name']
2597 files['playbooks/%s.yaml' % jobname] = ''
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002598 before = self.addCommitToRepo(
2599 project_name, 'Pulling content from %s' % source_name,
James E. Blairdfdfcfc2017-04-20 10:19:20 -07002600 files)
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002601 return before
2602
James E. Blair7fc8daa2016-08-08 15:37:15 -07002603 def addEvent(self, connection, event):
James E. Blair9ea0d0b2017-04-20 09:27:15 -07002604
James E. Blair7fc8daa2016-08-08 15:37:15 -07002605 """Inject a Fake (Gerrit) event.
2606
2607 This method accepts a JSON-encoded event and simulates Zuul
2608 having received it from Gerrit. It could (and should)
2609 eventually apply to any connection type, but is currently only
2610 used with Gerrit connections. The name of the connection is
2611 used to look up the corresponding server, and the event is
2612 simulated as having been received by all Zuul connections
2613 attached to that server. So if two Gerrit connections in Zuul
2614 are connected to the same Gerrit server, and you invoke this
2615 method specifying the name of one of them, the event will be
2616 received by both.
2617
2618 .. note::
2619
2620 "self.fake_gerrit.addEvent" calls should be migrated to
2621 this method.
2622
2623 :arg str connection: The name of the connection corresponding
Clark Boylan500992b2017-04-03 14:28:24 -07002624 to the gerrit server.
James E. Blair7fc8daa2016-08-08 15:37:15 -07002625 :arg str event: The JSON-encoded event.
2626
2627 """
2628 specified_conn = self.connections.connections[connection]
2629 for conn in self.connections.connections.values():
2630 if (isinstance(conn, specified_conn.__class__) and
2631 specified_conn.server == conn.server):
2632 conn.addEvent(event)
2633
James E. Blair3f876d52016-07-22 13:07:14 -07002634
2635class AnsibleZuulTestCase(ZuulTestCase):
Paul Belanger174a8272017-03-14 13:20:10 -04002636 """ZuulTestCase but with an actual ansible executor running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07002637 run_ansible = True
Joshua Hesketh25695cb2017-03-06 12:50:04 +11002638
Joshua Heskethd78b4482015-09-14 16:56:34 -06002639
2640class ZuulDBTestCase(ZuulTestCase):
James E. Blair82844892017-03-06 10:55:26 -08002641 def setup_config(self):
2642 super(ZuulDBTestCase, self).setup_config()
Joshua Heskethd78b4482015-09-14 16:56:34 -06002643 for section_name in self.config.sections():
2644 con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
2645 section_name, re.I)
2646 if not con_match:
2647 continue
2648
2649 if self.config.get(section_name, 'driver') == 'sql':
2650 f = MySQLSchemaFixture()
2651 self.useFixture(f)
2652 if (self.config.get(section_name, 'dburi') ==
2653 '$MYSQL_FIXTURE_DBURI$'):
2654 self.config.set(section_name, 'dburi', f.dburi)