blob: 2816b9f77af598193d63f5d9a334ab5be6b5ce4e [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070019import gc
20import hashlib
21import json
22import logging
23import os
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070025from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070026import random
27import re
28import select
29import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030030from six.moves import reload_module
James E. Blair1c236df2017-02-01 14:07:24 -080031from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070032import socket
33import string
34import subprocess
James E. Blair1c236df2017-02-01 14:07:24 -080035import sys
James E. Blairf84026c2015-12-08 16:11:46 -080036import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070037import threading
38import time
Joshua Heskethd78b4482015-09-14 16:56:34 -060039import uuid
40
Clark Boylanb640e052014-04-03 16:41:46 -070041
42import git
43import gear
44import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080045import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080046import kazoo.exceptions
Joshua Heskethd78b4482015-09-14 16:56:34 -060047import pymysql
Clark Boylanb640e052014-04-03 16:41:46 -070048import statsd
49import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080050import testtools.content
51import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080052from git.exc import NoSuchPathError
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +000053import yaml
Clark Boylanb640e052014-04-03 16:41:46 -070054
James E. Blaire511d2f2016-12-08 15:22:26 -080055import zuul.driver.gerrit.gerritsource as gerritsource
56import zuul.driver.gerrit.gerritconnection as gerritconnection
Clark Boylanb640e052014-04-03 16:41:46 -070057import zuul.scheduler
58import zuul.webapp
59import zuul.rpclistener
Paul Belanger174a8272017-03-14 13:20:10 -040060import zuul.executor.server
61import zuul.executor.client
James E. Blair83005782015-12-11 14:46:03 -080062import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070063import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070064import zuul.merger.merger
65import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070066import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080067import zuul.zk
Clark Boylanb640e052014-04-03 16:41:46 -070068
69FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
70 'fixtures')
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -080071
72KEEP_TEMPDIRS = bool(os.environ.get('KEEP_TEMPDIRS', False))
Clark Boylanb640e052014-04-03 16:41:46 -070073
Clark Boylanb640e052014-04-03 16:41:46 -070074
75def repack_repo(path):
76 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
77 output = subprocess.Popen(cmd, close_fds=True,
78 stdout=subprocess.PIPE,
79 stderr=subprocess.PIPE)
80 out = output.communicate()
81 if output.returncode:
82 raise Exception("git repack returned %d" % output.returncode)
83 return out
84
85
86def random_sha1():
87 return hashlib.sha1(str(random.random())).hexdigest()
88
89
James E. Blaira190f3b2015-01-05 14:56:54 -080090def iterate_timeout(max_seconds, purpose):
91 start = time.time()
92 count = 0
93 while (time.time() < start + max_seconds):
94 count += 1
95 yield count
96 time.sleep(0)
97 raise Exception("Timeout waiting for %s" % purpose)
98
99
Clark Boylanb640e052014-04-03 16:41:46 -0700100class ChangeReference(git.Reference):
101 _common_path_default = "refs/changes"
102 _points_to_commits_only = True
103
104
105class FakeChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700106 categories = {'approved': ('Approved', -1, 1),
107 'code-review': ('Code-Review', -2, 2),
108 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700109
110 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700111 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700112 self.gerrit = gerrit
113 self.reported = 0
114 self.queried = 0
115 self.patchsets = []
116 self.number = number
117 self.project = project
118 self.branch = branch
119 self.subject = subject
120 self.latest_patchset = 0
121 self.depends_on_change = None
122 self.needed_by_changes = []
123 self.fail_merge = False
124 self.messages = []
125 self.data = {
126 'branch': branch,
127 'comments': [],
128 'commitMessage': subject,
129 'createdOn': time.time(),
130 'id': 'I' + random_sha1(),
131 'lastUpdated': time.time(),
132 'number': str(number),
133 'open': status == 'NEW',
134 'owner': {'email': 'user@example.com',
135 'name': 'User Name',
136 'username': 'username'},
137 'patchSets': self.patchsets,
138 'project': project,
139 'status': status,
140 'subject': subject,
141 'submitRecords': [],
142 'url': 'https://hostname/%s' % number}
143
144 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700145 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700146 self.data['submitRecords'] = self.getSubmitRecords()
147 self.open = status == 'NEW'
148
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700149 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700150 path = os.path.join(self.upstream_root, self.project)
151 repo = git.Repo(path)
152 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
153 self.latest_patchset),
154 'refs/tags/init')
155 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700156 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700157 repo.git.clean('-x', '-f', '-d')
158
159 path = os.path.join(self.upstream_root, self.project)
160 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700161 for fn, content in files.items():
162 fn = os.path.join(path, fn)
163 with open(fn, 'w') as f:
164 f.write(content)
165 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700166 else:
167 for fni in range(100):
168 fn = os.path.join(path, str(fni))
169 f = open(fn, 'w')
170 for ci in range(4096):
171 f.write(random.choice(string.printable))
172 f.close()
173 repo.index.add([fn])
174
175 r = repo.index.commit(msg)
176 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700177 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700178 repo.git.clean('-x', '-f', '-d')
179 repo.heads['master'].checkout()
180 return r
181
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700182 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700183 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700184 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700185 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700186 data = ("test %s %s %s\n" %
187 (self.branch, self.number, self.latest_patchset))
188 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700189 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700190 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700191 ps_files = [{'file': '/COMMIT_MSG',
192 'type': 'ADDED'},
193 {'file': 'README',
194 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700195 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700196 ps_files.append({'file': f, 'type': 'ADDED'})
197 d = {'approvals': [],
198 'createdOn': time.time(),
199 'files': ps_files,
200 'number': str(self.latest_patchset),
201 'ref': 'refs/changes/1/%s/%s' % (self.number,
202 self.latest_patchset),
203 'revision': c.hexsha,
204 'uploader': {'email': 'user@example.com',
205 'name': 'User name',
206 'username': 'user'}}
207 self.data['currentPatchSet'] = d
208 self.patchsets.append(d)
209 self.data['submitRecords'] = self.getSubmitRecords()
210
211 def getPatchsetCreatedEvent(self, patchset):
212 event = {"type": "patchset-created",
213 "change": {"project": self.project,
214 "branch": self.branch,
215 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
216 "number": str(self.number),
217 "subject": self.subject,
218 "owner": {"name": "User Name"},
219 "url": "https://hostname/3"},
220 "patchSet": self.patchsets[patchset - 1],
221 "uploader": {"name": "User Name"}}
222 return event
223
224 def getChangeRestoredEvent(self):
225 event = {"type": "change-restored",
226 "change": {"project": self.project,
227 "branch": self.branch,
228 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
229 "number": str(self.number),
230 "subject": self.subject,
231 "owner": {"name": "User Name"},
232 "url": "https://hostname/3"},
233 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100234 "patchSet": self.patchsets[-1],
235 "reason": ""}
236 return event
237
238 def getChangeAbandonedEvent(self):
239 event = {"type": "change-abandoned",
240 "change": {"project": self.project,
241 "branch": self.branch,
242 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
243 "number": str(self.number),
244 "subject": self.subject,
245 "owner": {"name": "User Name"},
246 "url": "https://hostname/3"},
247 "abandoner": {"name": "User Name"},
248 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700249 "reason": ""}
250 return event
251
252 def getChangeCommentEvent(self, patchset):
253 event = {"type": "comment-added",
254 "change": {"project": self.project,
255 "branch": self.branch,
256 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
257 "number": str(self.number),
258 "subject": self.subject,
259 "owner": {"name": "User Name"},
260 "url": "https://hostname/3"},
261 "patchSet": self.patchsets[patchset - 1],
262 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700263 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700264 "description": "Code-Review",
265 "value": "0"}],
266 "comment": "This is a comment"}
267 return event
268
James E. Blairc2a5ed72017-02-20 14:12:01 -0500269 def getChangeMergedEvent(self):
270 event = {"submitter": {"name": "Jenkins",
271 "username": "jenkins"},
272 "newRev": "29ed3b5f8f750a225c5be70235230e3a6ccb04d9",
273 "patchSet": self.patchsets[-1],
274 "change": self.data,
275 "type": "change-merged",
276 "eventCreatedOn": 1487613810}
277 return event
278
James E. Blair8cce42e2016-10-18 08:18:36 -0700279 def getRefUpdatedEvent(self):
280 path = os.path.join(self.upstream_root, self.project)
281 repo = git.Repo(path)
282 oldrev = repo.heads[self.branch].commit.hexsha
283
284 event = {
285 "type": "ref-updated",
286 "submitter": {
287 "name": "User Name",
288 },
289 "refUpdate": {
290 "oldRev": oldrev,
291 "newRev": self.patchsets[-1]['revision'],
292 "refName": self.branch,
293 "project": self.project,
294 }
295 }
296 return event
297
Joshua Hesketh642824b2014-07-01 17:54:59 +1000298 def addApproval(self, category, value, username='reviewer_john',
299 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700300 if not granted_on:
301 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000302 approval = {
303 'description': self.categories[category][0],
304 'type': category,
305 'value': str(value),
306 'by': {
307 'username': username,
308 'email': username + '@example.com',
309 },
310 'grantedOn': int(granted_on)
311 }
Clark Boylanb640e052014-04-03 16:41:46 -0700312 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
313 if x['by']['username'] == username and x['type'] == category:
314 del self.patchsets[-1]['approvals'][i]
315 self.patchsets[-1]['approvals'].append(approval)
316 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000317 'author': {'email': 'author@example.com',
318 'name': 'Patchset Author',
319 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700320 'change': {'branch': self.branch,
321 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
322 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000323 'owner': {'email': 'owner@example.com',
324 'name': 'Change Owner',
325 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700326 'project': self.project,
327 'subject': self.subject,
328 'topic': 'master',
329 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000330 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700331 'patchSet': self.patchsets[-1],
332 'type': 'comment-added'}
333 self.data['submitRecords'] = self.getSubmitRecords()
334 return json.loads(json.dumps(event))
335
336 def getSubmitRecords(self):
337 status = {}
338 for cat in self.categories.keys():
339 status[cat] = 0
340
341 for a in self.patchsets[-1]['approvals']:
342 cur = status[a['type']]
343 cat_min, cat_max = self.categories[a['type']][1:]
344 new = int(a['value'])
345 if new == cat_min:
346 cur = new
347 elif abs(new) > abs(cur):
348 cur = new
349 status[a['type']] = cur
350
351 labels = []
352 ok = True
353 for typ, cat in self.categories.items():
354 cur = status[typ]
355 cat_min, cat_max = cat[1:]
356 if cur == cat_min:
357 value = 'REJECT'
358 ok = False
359 elif cur == cat_max:
360 value = 'OK'
361 else:
362 value = 'NEED'
363 ok = False
364 labels.append({'label': cat[0], 'status': value})
365 if ok:
366 return [{'status': 'OK'}]
367 return [{'status': 'NOT_READY',
368 'labels': labels}]
369
370 def setDependsOn(self, other, patchset):
371 self.depends_on_change = other
372 d = {'id': other.data['id'],
373 'number': other.data['number'],
374 'ref': other.patchsets[patchset - 1]['ref']
375 }
376 self.data['dependsOn'] = [d]
377
378 other.needed_by_changes.append(self)
379 needed = other.data.get('neededBy', [])
380 d = {'id': self.data['id'],
381 'number': self.data['number'],
382 'ref': self.patchsets[patchset - 1]['ref'],
383 'revision': self.patchsets[patchset - 1]['revision']
384 }
385 needed.append(d)
386 other.data['neededBy'] = needed
387
388 def query(self):
389 self.queried += 1
390 d = self.data.get('dependsOn')
391 if d:
392 d = d[0]
393 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
394 d['isCurrentPatchSet'] = True
395 else:
396 d['isCurrentPatchSet'] = False
397 return json.loads(json.dumps(self.data))
398
399 def setMerged(self):
400 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000401 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700402 return
403 if self.fail_merge:
404 return
405 self.data['status'] = 'MERGED'
406 self.open = False
407
408 path = os.path.join(self.upstream_root, self.project)
409 repo = git.Repo(path)
410 repo.heads[self.branch].commit = \
411 repo.commit(self.patchsets[-1]['revision'])
412
413 def setReported(self):
414 self.reported += 1
415
416
James E. Blaire511d2f2016-12-08 15:22:26 -0800417class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700418 """A Fake Gerrit connection for use in tests.
419
420 This subclasses
421 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
422 ability for tests to add changes to the fake Gerrit it represents.
423 """
424
Joshua Hesketh352264b2015-08-11 23:42:08 +1000425 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700426
James E. Blaire511d2f2016-12-08 15:22:26 -0800427 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700428 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800429 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000430 connection_config)
431
James E. Blair7fc8daa2016-08-08 15:37:15 -0700432 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700433 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
434 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000435 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700436 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200437 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700438
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700439 def addFakeChange(self, project, branch, subject, status='NEW',
440 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700441 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700442 self.change_number += 1
443 c = FakeChange(self, self.change_number, project, branch, subject,
444 upstream_root=self.upstream_root,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700445 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700446 self.changes[self.change_number] = c
447 return c
448
Clark Boylanb640e052014-04-03 16:41:46 -0700449 def review(self, project, changeid, message, action):
450 number, ps = changeid.split(',')
451 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000452
453 # Add the approval back onto the change (ie simulate what gerrit would
454 # do).
455 # Usually when zuul leaves a review it'll create a feedback loop where
456 # zuul's review enters another gerrit event (which is then picked up by
457 # zuul). However, we can't mimic this behaviour (by adding this
458 # approval event into the queue) as it stops jobs from checking what
459 # happens before this event is triggered. If a job needs to see what
460 # happens they can add their own verified event into the queue.
461 # Nevertheless, we can update change with the new review in gerrit.
462
James E. Blair8b5408c2016-08-08 15:37:46 -0700463 for cat in action.keys():
464 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000465 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000466
James E. Blair8b5408c2016-08-08 15:37:46 -0700467 # TODOv3(jeblair): can this be removed?
Joshua Hesketh642824b2014-07-01 17:54:59 +1000468 if 'label' in action:
469 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000470 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000471
Clark Boylanb640e052014-04-03 16:41:46 -0700472 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000473
Clark Boylanb640e052014-04-03 16:41:46 -0700474 if 'submit' in action:
475 change.setMerged()
476 if message:
477 change.setReported()
478
479 def query(self, number):
480 change = self.changes.get(int(number))
481 if change:
482 return change.query()
483 return {}
484
James E. Blairc494d542014-08-06 09:23:52 -0700485 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700486 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700487 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800488 if query.startswith('change:'):
489 # Query a specific changeid
490 changeid = query[len('change:'):]
491 l = [change.query() for change in self.changes.values()
492 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700493 elif query.startswith('message:'):
494 # Query the content of a commit message
495 msg = query[len('message:'):].strip()
496 l = [change.query() for change in self.changes.values()
497 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800498 else:
499 # Query all open changes
500 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700501 return l
James E. Blairc494d542014-08-06 09:23:52 -0700502
Joshua Hesketh352264b2015-08-11 23:42:08 +1000503 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700504 pass
505
Joshua Hesketh352264b2015-08-11 23:42:08 +1000506 def getGitUrl(self, project):
507 return os.path.join(self.upstream_root, project.name)
508
Clark Boylanb640e052014-04-03 16:41:46 -0700509
510class BuildHistory(object):
511 def __init__(self, **kw):
512 self.__dict__.update(kw)
513
514 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -0700515 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
516 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -0700517
518
519class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200520 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700521 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700522 self.url = url
523
524 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -0700525 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -0700526 path = res.path
527 project = '/'.join(path.split('/')[2:-2])
528 ret = '001e# service=git-upload-pack\n'
529 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
530 'multi_ack thin-pack side-band side-band-64k ofs-delta '
531 'shallow no-progress include-tag multi_ack_detailed no-done\n')
532 path = os.path.join(self.upstream_root, project)
533 repo = git.Repo(path)
534 for ref in repo.refs:
535 r = ref.object.hexsha + ' ' + ref.path + '\n'
536 ret += '%04x%s' % (len(r) + 4, r)
537 ret += '0000'
538 return ret
539
540
Clark Boylanb640e052014-04-03 16:41:46 -0700541class FakeStatsd(threading.Thread):
542 def __init__(self):
543 threading.Thread.__init__(self)
544 self.daemon = True
545 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
546 self.sock.bind(('', 0))
547 self.port = self.sock.getsockname()[1]
548 self.wake_read, self.wake_write = os.pipe()
549 self.stats = []
550
551 def run(self):
552 while True:
553 poll = select.poll()
554 poll.register(self.sock, select.POLLIN)
555 poll.register(self.wake_read, select.POLLIN)
556 ret = poll.poll()
557 for (fd, event) in ret:
558 if fd == self.sock.fileno():
559 data = self.sock.recvfrom(1024)
560 if not data:
561 return
562 self.stats.append(data[0])
563 if fd == self.wake_read:
564 return
565
566 def stop(self):
567 os.write(self.wake_write, '1\n')
568
569
James E. Blaire1767bc2016-08-02 10:00:27 -0700570class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -0700571 log = logging.getLogger("zuul.test")
572
Paul Belanger174a8272017-03-14 13:20:10 -0400573 def __init__(self, executor_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -0700574 self.daemon = True
Paul Belanger174a8272017-03-14 13:20:10 -0400575 self.executor_server = executor_server
Clark Boylanb640e052014-04-03 16:41:46 -0700576 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -0700577 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -0700578 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -0700579 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -0700580 # TODOv3(jeblair): self.node is really "the image of the node
581 # assigned". We should rename it (self.node_image?) if we
582 # keep using it like this, or we may end up exposing more of
583 # the complexity around multi-node jobs here
584 # (self.nodes[0].image?)
585 self.node = None
586 if len(self.parameters.get('nodes')) == 1:
587 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -0700588 self.unique = self.parameters['ZUUL_UUID']
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100589 self.pipeline = self.parameters['ZUUL_PIPELINE']
590 self.project = self.parameters['ZUUL_PROJECT']
James E. Blair3f876d52016-07-22 13:07:14 -0700591 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -0700592 self.wait_condition = threading.Condition()
593 self.waiting = False
594 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -0500595 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -0700596 self.created = time.time()
James E. Blaire1767bc2016-08-02 10:00:27 -0700597 self.changes = None
598 if 'ZUUL_CHANGE_IDS' in self.parameters:
599 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -0700600
James E. Blair3158e282016-08-19 09:34:11 -0700601 def __repr__(self):
602 waiting = ''
603 if self.waiting:
604 waiting = ' [waiting]'
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100605 return '<FakeBuild %s:%s %s%s>' % (self.pipeline, self.name,
606 self.changes, waiting)
James E. Blair3158e282016-08-19 09:34:11 -0700607
Clark Boylanb640e052014-04-03 16:41:46 -0700608 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700609 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -0700610 self.wait_condition.acquire()
611 self.wait_condition.notify()
612 self.waiting = False
613 self.log.debug("Build %s released" % self.unique)
614 self.wait_condition.release()
615
616 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700617 """Return whether this build is being held.
618
619 :returns: Whether the build is being held.
620 :rtype: bool
621 """
622
Clark Boylanb640e052014-04-03 16:41:46 -0700623 self.wait_condition.acquire()
624 if self.waiting:
625 ret = True
626 else:
627 ret = False
628 self.wait_condition.release()
629 return ret
630
631 def _wait(self):
632 self.wait_condition.acquire()
633 self.waiting = True
634 self.log.debug("Build %s waiting" % self.unique)
635 self.wait_condition.wait()
636 self.wait_condition.release()
637
638 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -0700639 self.log.debug('Running build %s' % self.unique)
640
Paul Belanger174a8272017-03-14 13:20:10 -0400641 if self.executor_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -0700642 self.log.debug('Holding build %s' % self.unique)
643 self._wait()
644 self.log.debug("Build %s continuing" % self.unique)
645
James E. Blair412fba82017-01-26 15:00:50 -0800646 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
James E. Blaira5dba232016-08-08 15:53:24 -0700647 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
James E. Blair412fba82017-01-26 15:00:50 -0800648 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
Clark Boylanb640e052014-04-03 16:41:46 -0700649 if self.aborted:
James E. Blair412fba82017-01-26 15:00:50 -0800650 result = (RecordingAnsibleJob.RESULT_ABORTED, None)
Paul Belanger71d98172016-11-08 10:56:31 -0500651 if self.requeue:
James E. Blair412fba82017-01-26 15:00:50 -0800652 result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
Clark Boylanb640e052014-04-03 16:41:46 -0700653
James E. Blaire1767bc2016-08-02 10:00:27 -0700654 return result
Clark Boylanb640e052014-04-03 16:41:46 -0700655
James E. Blaira5dba232016-08-08 15:53:24 -0700656 def shouldFail(self):
Paul Belanger174a8272017-03-14 13:20:10 -0400657 changes = self.executor_server.fail_tests.get(self.name, [])
James E. Blaira5dba232016-08-08 15:53:24 -0700658 for change in changes:
659 if self.hasChanges(change):
660 return True
661 return False
662
James E. Blaire7b99a02016-08-05 14:27:34 -0700663 def hasChanges(self, *changes):
664 """Return whether this build has certain changes in its git repos.
665
666 :arg FakeChange changes: One or more changes (varargs) that
667 are expected to be present (in order) in the git repository of
668 the active project.
669
670 :returns: Whether the build has the indicated changes.
671 :rtype: bool
672
673 """
Clint Byrum3343e3e2016-11-15 16:05:03 -0800674 for change in changes:
Monty Taylord642d852017-02-23 14:05:42 -0500675 path = os.path.join(self.jobdir.src_root, change.project)
Clint Byrum3343e3e2016-11-15 16:05:03 -0800676 try:
677 repo = git.Repo(path)
678 except NoSuchPathError as e:
679 self.log.debug('%s' % e)
680 return False
681 ref = self.parameters['ZUUL_REF']
682 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
683 commit_message = '%s-1' % change.subject
684 self.log.debug("Checking if build %s has changes; commit_message "
685 "%s; repo_messages %s" % (self, commit_message,
686 repo_messages))
687 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -0700688 self.log.debug(" messages do not match")
689 return False
690 self.log.debug(" OK")
691 return True
692
Clark Boylanb640e052014-04-03 16:41:46 -0700693
Paul Belanger174a8272017-03-14 13:20:10 -0400694class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
695 """An Ansible executor to be used in tests.
James E. Blaire7b99a02016-08-05 14:27:34 -0700696
Paul Belanger174a8272017-03-14 13:20:10 -0400697 :ivar bool hold_jobs_in_build: If true, when jobs are executed
James E. Blaire7b99a02016-08-05 14:27:34 -0700698 they will report that they have started but then pause until
699 released before reporting completion. This attribute may be
700 changed at any time and will take effect for subsequently
Paul Belanger174a8272017-03-14 13:20:10 -0400701 executed builds, but previously held builds will still need to
James E. Blaire7b99a02016-08-05 14:27:34 -0700702 be explicitly released.
703
704 """
James E. Blairf5dbd002015-12-23 15:26:17 -0800705 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -0700706 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -0800707 self._test_root = kw.pop('_test_root', False)
Paul Belanger174a8272017-03-14 13:20:10 -0400708 super(RecordingExecutorServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -0700709 self.hold_jobs_in_build = False
710 self.lock = threading.Lock()
711 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -0700712 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -0700713 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -0700714 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -0800715
James E. Blaira5dba232016-08-08 15:53:24 -0700716 def failJob(self, name, change):
Paul Belanger174a8272017-03-14 13:20:10 -0400717 """Instruct the executor to report matching builds as failures.
James E. Blaire7b99a02016-08-05 14:27:34 -0700718
719 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -0700720 :arg Change change: The :py:class:`~tests.base.FakeChange`
721 instance which should cause the job to fail. This job
722 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -0700723
724 """
James E. Blaire1767bc2016-08-02 10:00:27 -0700725 l = self.fail_tests.get(name, [])
726 l.append(change)
727 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -0800728
James E. Blair962220f2016-08-03 11:22:38 -0700729 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700730 """Release a held build.
731
732 :arg str regex: A regular expression which, if supplied, will
733 cause only builds with matching names to be released. If
734 not supplied, all builds will be released.
735
736 """
James E. Blair962220f2016-08-03 11:22:38 -0700737 builds = self.running_builds[:]
738 self.log.debug("Releasing build %s (%s)" % (regex,
739 len(self.running_builds)))
740 for build in builds:
741 if not regex or re.match(regex, build.name):
742 self.log.debug("Releasing build %s" %
743 (build.parameters['ZUUL_UUID']))
744 build.release()
745 else:
746 self.log.debug("Not releasing build %s" %
747 (build.parameters['ZUUL_UUID']))
748 self.log.debug("Done releasing builds %s (%s)" %
749 (regex, len(self.running_builds)))
750
Paul Belanger174a8272017-03-14 13:20:10 -0400751 def executeJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -0700752 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -0700753 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -0700754 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -0700755 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -0800756 args = json.loads(job.arguments)
James E. Blair490cf042017-02-24 23:07:21 -0500757 args['vars']['zuul']['_test'] = dict(test_root=self._test_root)
James E. Blaira92cbc82017-01-23 14:56:49 -0800758 job.arguments = json.dumps(args)
Joshua Hesketh50c21782016-10-13 21:34:14 +1100759 self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
760 self.job_workers[job.unique].run()
James E. Blair17302972016-08-10 16:11:42 -0700761
762 def stopJob(self, job):
763 self.log.debug("handle stop")
764 parameters = json.loads(job.arguments)
765 uuid = parameters['uuid']
766 for build in self.running_builds:
767 if build.unique == uuid:
768 build.aborted = True
769 build.release()
Paul Belanger174a8272017-03-14 13:20:10 -0400770 super(RecordingExecutorServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -0700771
Joshua Hesketh50c21782016-10-13 21:34:14 +1100772
Paul Belanger174a8272017-03-14 13:20:10 -0400773class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
Paul Belanger96618ed2017-03-01 09:42:33 -0500774 def runPlaybooks(self, args):
Paul Belanger174a8272017-03-14 13:20:10 -0400775 build = self.executor_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -0800776 build.jobdir = self.jobdir
James E. Blaire1767bc2016-08-02 10:00:27 -0700777
Paul Belanger96618ed2017-03-01 09:42:33 -0500778 result = super(RecordingAnsibleJob, self).runPlaybooks(args)
James E. Blair412fba82017-01-26 15:00:50 -0800779
Paul Belanger174a8272017-03-14 13:20:10 -0400780 self.executor_server.lock.acquire()
781 self.executor_server.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -0700782 BuildHistory(name=build.name, result=result, changes=build.changes,
783 node=build.node, uuid=build.unique,
K Jonathan Harker2c1a6232017-02-21 14:34:08 -0800784 parameters=build.parameters, jobdir=build.jobdir,
James E. Blaire1767bc2016-08-02 10:00:27 -0700785 pipeline=build.parameters['ZUUL_PIPELINE'])
786 )
Paul Belanger174a8272017-03-14 13:20:10 -0400787 self.executor_server.running_builds.remove(build)
788 del self.executor_server.job_builds[self.job.unique]
789 self.executor_server.lock.release()
James E. Blair412fba82017-01-26 15:00:50 -0800790 return result
791
Monty Taylore6562aa2017-02-20 07:37:39 -0500792 def runAnsible(self, cmd, timeout, trusted=False):
Paul Belanger174a8272017-03-14 13:20:10 -0400793 build = self.executor_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -0800794
Paul Belanger174a8272017-03-14 13:20:10 -0400795 if self.executor_server._run_ansible:
Monty Taylorc231d932017-02-03 09:57:15 -0600796 result = super(RecordingAnsibleJob, self).runAnsible(
Monty Taylore6562aa2017-02-20 07:37:39 -0500797 cmd, timeout, trusted=trusted)
James E. Blair412fba82017-01-26 15:00:50 -0800798 else:
799 result = build.run()
James E. Blaire1767bc2016-08-02 10:00:27 -0700800 return result
James E. Blairf5dbd002015-12-23 15:26:17 -0800801
James E. Blairad8dca02017-02-21 11:48:32 -0500802 def getHostList(self, args):
803 self.log.debug("hostlist")
804 hosts = super(RecordingAnsibleJob, self).getHostList(args)
Paul Belangerc5bf3752017-03-16 19:38:43 -0400805 for host in hosts:
806 host['host_vars']['ansible_connection'] = 'local'
807
808 hosts.append(dict(
809 name='localhost',
810 host_vars=dict(ansible_connection='local'),
811 host_keys=[]))
James E. Blairad8dca02017-02-21 11:48:32 -0500812 return hosts
813
James E. Blairf5dbd002015-12-23 15:26:17 -0800814
Clark Boylanb640e052014-04-03 16:41:46 -0700815class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -0700816 """A Gearman server for use in tests.
817
818 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
819 added to the queue but will not be distributed to workers
820 until released. This attribute may be changed at any time and
821 will take effect for subsequently enqueued jobs, but
822 previously held jobs will still need to be explicitly
823 released.
824
825 """
826
Clark Boylanb640e052014-04-03 16:41:46 -0700827 def __init__(self):
828 self.hold_jobs_in_queue = False
829 super(FakeGearmanServer, self).__init__(0)
830
831 def getJobForConnection(self, connection, peek=False):
832 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
833 for job in queue:
834 if not hasattr(job, 'waiting'):
Paul Belanger174a8272017-03-14 13:20:10 -0400835 if job.name.startswith('executor:execute'):
Clark Boylanb640e052014-04-03 16:41:46 -0700836 job.waiting = self.hold_jobs_in_queue
837 else:
838 job.waiting = False
839 if job.waiting:
840 continue
841 if job.name in connection.functions:
842 if not peek:
843 queue.remove(job)
844 connection.related_jobs[job.handle] = job
845 job.worker_connection = connection
846 job.running = True
847 return job
848 return None
849
850 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700851 """Release a held job.
852
853 :arg str regex: A regular expression which, if supplied, will
854 cause only jobs with matching names to be released. If
855 not supplied, all jobs will be released.
856 """
Clark Boylanb640e052014-04-03 16:41:46 -0700857 released = False
858 qlen = (len(self.high_queue) + len(self.normal_queue) +
859 len(self.low_queue))
860 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
861 for job in self.getQueue():
Paul Belanger174a8272017-03-14 13:20:10 -0400862 if job.name != 'executor:execute':
Clark Boylanb640e052014-04-03 16:41:46 -0700863 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -0500864 parameters = json.loads(job.arguments)
865 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -0700866 self.log.debug("releasing queued job %s" %
867 job.unique)
868 job.waiting = False
869 released = True
870 else:
871 self.log.debug("not releasing queued job %s" %
872 job.unique)
873 if released:
874 self.wakeConnections()
875 qlen = (len(self.high_queue) + len(self.normal_queue) +
876 len(self.low_queue))
877 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
878
879
880class FakeSMTP(object):
881 log = logging.getLogger('zuul.FakeSMTP')
882
883 def __init__(self, messages, server, port):
884 self.server = server
885 self.port = port
886 self.messages = messages
887
888 def sendmail(self, from_email, to_email, msg):
889 self.log.info("Sending email from %s, to %s, with msg %s" % (
890 from_email, to_email, msg))
891
892 headers = msg.split('\n\n', 1)[0]
893 body = msg.split('\n\n', 1)[1]
894
895 self.messages.append(dict(
896 from_email=from_email,
897 to_email=to_email,
898 msg=msg,
899 headers=headers,
900 body=body,
901 ))
902
903 return True
904
905 def quit(self):
906 return True
907
908
James E. Blairdce6cea2016-12-20 16:45:32 -0800909class FakeNodepool(object):
910 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -0800911 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -0800912
913 log = logging.getLogger("zuul.test.FakeNodepool")
914
915 def __init__(self, host, port, chroot):
916 self.client = kazoo.client.KazooClient(
917 hosts='%s:%s%s' % (host, port, chroot))
918 self.client.start()
919 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -0800920 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -0800921 self.thread = threading.Thread(target=self.run)
922 self.thread.daemon = True
923 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -0800924 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -0800925
926 def stop(self):
927 self._running = False
928 self.thread.join()
929 self.client.stop()
930 self.client.close()
931
932 def run(self):
933 while self._running:
934 self._run()
935 time.sleep(0.1)
936
937 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -0800938 if self.paused:
939 return
James E. Blairdce6cea2016-12-20 16:45:32 -0800940 for req in self.getNodeRequests():
941 self.fulfillRequest(req)
942
943 def getNodeRequests(self):
944 try:
945 reqids = self.client.get_children(self.REQUEST_ROOT)
946 except kazoo.exceptions.NoNodeError:
947 return []
948 reqs = []
949 for oid in sorted(reqids):
950 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -0800951 try:
952 data, stat = self.client.get(path)
953 data = json.loads(data)
954 data['_oid'] = oid
955 reqs.append(data)
956 except kazoo.exceptions.NoNodeError:
957 pass
James E. Blairdce6cea2016-12-20 16:45:32 -0800958 return reqs
959
James E. Blaire18d4602017-01-05 11:17:28 -0800960 def getNodes(self):
961 try:
962 nodeids = self.client.get_children(self.NODE_ROOT)
963 except kazoo.exceptions.NoNodeError:
964 return []
965 nodes = []
966 for oid in sorted(nodeids):
967 path = self.NODE_ROOT + '/' + oid
968 data, stat = self.client.get(path)
969 data = json.loads(data)
970 data['_oid'] = oid
971 try:
972 lockfiles = self.client.get_children(path + '/lock')
973 except kazoo.exceptions.NoNodeError:
974 lockfiles = []
975 if lockfiles:
976 data['_lock'] = True
977 else:
978 data['_lock'] = False
979 nodes.append(data)
980 return nodes
981
James E. Blaira38c28e2017-01-04 10:33:20 -0800982 def makeNode(self, request_id, node_type):
983 now = time.time()
984 path = '/nodepool/nodes/'
985 data = dict(type=node_type,
986 provider='test-provider',
987 region='test-region',
Paul Belanger30ba93a2017-03-16 16:28:10 -0400988 az='test-az',
James E. Blaira38c28e2017-01-04 10:33:20 -0800989 public_ipv4='127.0.0.1',
990 private_ipv4=None,
991 public_ipv6=None,
992 allocated_to=request_id,
993 state='ready',
994 state_time=now,
995 created_time=now,
996 updated_time=now,
997 image_id=None,
Paul Belangerc5bf3752017-03-16 19:38:43 -0400998 host_keys=["fake-key1", "fake-key2"],
Paul Belanger174a8272017-03-14 13:20:10 -0400999 executor='fake-nodepool')
James E. Blaira38c28e2017-01-04 10:33:20 -08001000 data = json.dumps(data)
1001 path = self.client.create(path, data,
1002 makepath=True,
1003 sequence=True)
1004 nodeid = path.split("/")[-1]
1005 return nodeid
1006
James E. Blair6ab79e02017-01-06 10:10:17 -08001007 def addFailRequest(self, request):
1008 self.fail_requests.add(request['_oid'])
1009
James E. Blairdce6cea2016-12-20 16:45:32 -08001010 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -08001011 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -08001012 return
1013 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -08001014 oid = request['_oid']
1015 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -08001016
James E. Blair6ab79e02017-01-06 10:10:17 -08001017 if oid in self.fail_requests:
1018 request['state'] = 'failed'
1019 else:
1020 request['state'] = 'fulfilled'
1021 nodes = []
1022 for node in request['node_types']:
1023 nodeid = self.makeNode(oid, node)
1024 nodes.append(nodeid)
1025 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -08001026
James E. Blaira38c28e2017-01-04 10:33:20 -08001027 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -08001028 path = self.REQUEST_ROOT + '/' + oid
1029 data = json.dumps(request)
1030 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
1031 self.client.set(path, data)
1032
1033
James E. Blair498059b2016-12-20 13:50:13 -08001034class ChrootedKazooFixture(fixtures.Fixture):
1035 def __init__(self):
1036 super(ChrootedKazooFixture, self).__init__()
1037
1038 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1039 if ':' in zk_host:
1040 host, port = zk_host.split(':')
1041 else:
1042 host = zk_host
1043 port = None
1044
1045 self.zookeeper_host = host
1046
1047 if not port:
1048 self.zookeeper_port = 2181
1049 else:
1050 self.zookeeper_port = int(port)
1051
1052 def _setUp(self):
1053 # Make sure the test chroot paths do not conflict
1054 random_bits = ''.join(random.choice(string.ascii_lowercase +
1055 string.ascii_uppercase)
1056 for x in range(8))
1057
1058 rand_test_path = '%s_%s' % (random_bits, os.getpid())
1059 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1060
1061 # Ensure the chroot path exists and clean up any pre-existing znodes.
1062 _tmp_client = kazoo.client.KazooClient(
1063 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1064 _tmp_client.start()
1065
1066 if _tmp_client.exists(self.zookeeper_chroot):
1067 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1068
1069 _tmp_client.ensure_path(self.zookeeper_chroot)
1070 _tmp_client.stop()
1071 _tmp_client.close()
1072
1073 self.addCleanup(self._cleanup)
1074
1075 def _cleanup(self):
1076 '''Remove the chroot path.'''
1077 # Need a non-chroot'ed client to remove the chroot path
1078 _tmp_client = kazoo.client.KazooClient(
1079 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1080 _tmp_client.start()
1081 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1082 _tmp_client.stop()
1083
1084
Joshua Heskethd78b4482015-09-14 16:56:34 -06001085class MySQLSchemaFixture(fixtures.Fixture):
1086 def setUp(self):
1087 super(MySQLSchemaFixture, self).setUp()
1088
1089 random_bits = ''.join(random.choice(string.ascii_lowercase +
1090 string.ascii_uppercase)
1091 for x in range(8))
1092 self.name = '%s_%s' % (random_bits, os.getpid())
1093 self.passwd = uuid.uuid4().hex
1094 db = pymysql.connect(host="localhost",
1095 user="openstack_citest",
1096 passwd="openstack_citest",
1097 db="openstack_citest")
1098 cur = db.cursor()
1099 cur.execute("create database %s" % self.name)
1100 cur.execute(
1101 "grant all on %s.* to '%s'@'localhost' identified by '%s'" %
1102 (self.name, self.name, self.passwd))
1103 cur.execute("flush privileges")
1104
1105 self.dburi = 'mysql+pymysql://%s:%s@localhost/%s' % (self.name,
1106 self.passwd,
1107 self.name)
1108 self.addDetail('dburi', testtools.content.text_content(self.dburi))
1109 self.addCleanup(self.cleanup)
1110
1111 def cleanup(self):
1112 db = pymysql.connect(host="localhost",
1113 user="openstack_citest",
1114 passwd="openstack_citest",
1115 db="openstack_citest")
1116 cur = db.cursor()
1117 cur.execute("drop database %s" % self.name)
1118 cur.execute("drop user '%s'@'localhost'" % self.name)
1119 cur.execute("flush privileges")
1120
1121
Maru Newby3fe5f852015-01-13 04:22:14 +00001122class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001123 log = logging.getLogger("zuul.test")
Clint Byruma9626572017-02-22 14:04:00 -05001124 wait_timeout = 20
Clark Boylanb640e052014-04-03 16:41:46 -07001125
James E. Blair1c236df2017-02-01 14:07:24 -08001126 def attachLogs(self, *args):
1127 def reader():
1128 self._log_stream.seek(0)
1129 while True:
1130 x = self._log_stream.read(4096)
1131 if not x:
1132 break
1133 yield x.encode('utf8')
1134 content = testtools.content.content_from_reader(
1135 reader,
1136 testtools.content_type.UTF8_TEXT,
1137 False)
1138 self.addDetail('logging', content)
1139
Clark Boylanb640e052014-04-03 16:41:46 -07001140 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001141 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001142 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1143 try:
1144 test_timeout = int(test_timeout)
1145 except ValueError:
1146 # If timeout value is invalid do not set a timeout.
1147 test_timeout = 0
1148 if test_timeout > 0:
1149 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1150
1151 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1152 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1153 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1154 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1155 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1156 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1157 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1158 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1159 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1160 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001161 self._log_stream = StringIO()
1162 self.addOnException(self.attachLogs)
1163 else:
1164 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001165
James E. Blair1c236df2017-02-01 14:07:24 -08001166 handler = logging.StreamHandler(self._log_stream)
1167 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1168 '%(levelname)-8s %(message)s')
1169 handler.setFormatter(formatter)
1170
1171 logger = logging.getLogger()
1172 logger.setLevel(logging.DEBUG)
1173 logger.addHandler(handler)
1174
1175 # NOTE(notmorgan): Extract logging overrides for specific
1176 # libraries from the OS_LOG_DEFAULTS env and create loggers
1177 # for each. This is used to limit the output during test runs
1178 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001179 log_defaults_from_env = os.environ.get(
1180 'OS_LOG_DEFAULTS',
James E. Blair1c236df2017-02-01 14:07:24 -08001181 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001182
James E. Blairdce6cea2016-12-20 16:45:32 -08001183 if log_defaults_from_env:
1184 for default in log_defaults_from_env.split(','):
1185 try:
1186 name, level_str = default.split('=', 1)
1187 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001188 logger = logging.getLogger(name)
1189 logger.setLevel(level)
1190 logger.addHandler(handler)
1191 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001192 except ValueError:
1193 # NOTE(notmorgan): Invalid format of the log default,
1194 # skip and don't try and apply a logger for the
1195 # specified module
1196 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001197
Maru Newby3fe5f852015-01-13 04:22:14 +00001198
1199class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001200 """A test case with a functioning Zuul.
1201
1202 The following class variables are used during test setup and can
1203 be overidden by subclasses but are effectively read-only once a
1204 test method starts running:
1205
1206 :cvar str config_file: This points to the main zuul config file
1207 within the fixtures directory. Subclasses may override this
1208 to obtain a different behavior.
1209
1210 :cvar str tenant_config_file: This is the tenant config file
1211 (which specifies from what git repos the configuration should
1212 be loaded). It defaults to the value specified in
1213 `config_file` but can be overidden by subclasses to obtain a
1214 different tenant/project layout while using the standard main
1215 configuration.
1216
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001217 :cvar bool create_project_keys: Indicates whether Zuul should
1218 auto-generate keys for each project, or whether the test
1219 infrastructure should insert dummy keys to save time during
1220 startup. Defaults to False.
1221
James E. Blaire7b99a02016-08-05 14:27:34 -07001222 The following are instance variables that are useful within test
1223 methods:
1224
1225 :ivar FakeGerritConnection fake_<connection>:
1226 A :py:class:`~tests.base.FakeGerritConnection` will be
1227 instantiated for each connection present in the config file
1228 and stored here. For instance, `fake_gerrit` will hold the
1229 FakeGerritConnection object for a connection named `gerrit`.
1230
1231 :ivar FakeGearmanServer gearman_server: An instance of
1232 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1233 server that all of the Zuul components in this test use to
1234 communicate with each other.
1235
Paul Belanger174a8272017-03-14 13:20:10 -04001236 :ivar RecordingExecutorServer executor_server: An instance of
1237 :py:class:`~tests.base.RecordingExecutorServer` which is the
1238 Ansible execute server used to run jobs for this test.
James E. Blaire7b99a02016-08-05 14:27:34 -07001239
1240 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1241 representing currently running builds. They are appended to
Paul Belanger174a8272017-03-14 13:20:10 -04001242 the list in the order they are executed, and removed from this
James E. Blaire7b99a02016-08-05 14:27:34 -07001243 list upon completion.
1244
1245 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1246 objects representing completed builds. They are appended to
1247 the list in the order they complete.
1248
1249 """
1250
James E. Blair83005782015-12-11 14:46:03 -08001251 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001252 run_ansible = False
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001253 create_project_keys = False
James E. Blair3f876d52016-07-22 13:07:14 -07001254
1255 def _startMerger(self):
1256 self.merge_server = zuul.merger.server.MergeServer(self.config,
1257 self.connections)
1258 self.merge_server.start()
1259
Maru Newby3fe5f852015-01-13 04:22:14 +00001260 def setUp(self):
1261 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001262
1263 self.setupZK()
1264
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001265 if not KEEP_TEMPDIRS:
James E. Blair97d902e2014-08-21 13:25:56 -07001266 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001267 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1268 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001269 else:
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001270 tmp_root = tempfile.mkdtemp(
1271 dir=os.environ.get("ZUUL_TEST_ROOT", None))
Clark Boylanb640e052014-04-03 16:41:46 -07001272 self.test_root = os.path.join(tmp_root, "zuul-test")
1273 self.upstream_root = os.path.join(self.test_root, "upstream")
Monty Taylord642d852017-02-23 14:05:42 -05001274 self.merger_src_root = os.path.join(self.test_root, "merger-git")
Paul Belanger174a8272017-03-14 13:20:10 -04001275 self.executor_src_root = os.path.join(self.test_root, "executor-git")
James E. Blairce8a2132016-05-19 15:21:52 -07001276 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001277
1278 if os.path.exists(self.test_root):
1279 shutil.rmtree(self.test_root)
1280 os.makedirs(self.test_root)
1281 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001282 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001283
1284 # Make per test copy of Configuration.
1285 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001286 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001287 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001288 self.config.get('zuul', 'tenant_config')))
Monty Taylord642d852017-02-23 14:05:42 -05001289 self.config.set('merger', 'git_dir', self.merger_src_root)
Paul Belanger174a8272017-03-14 13:20:10 -04001290 self.config.set('executor', 'git_dir', self.executor_src_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001291 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001292
1293 # For each project in config:
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001294 # TODOv3(jeblair): remove these and replace with new git
1295 # filesystem fixtures
Clark Boylanb640e052014-04-03 16:41:46 -07001296 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -07001297 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -07001298 self.init_repo("org/project5")
1299 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -07001300 self.init_repo("org/one-job-project")
1301 self.init_repo("org/nonvoting-project")
1302 self.init_repo("org/templated-project")
1303 self.init_repo("org/layered-project")
1304 self.init_repo("org/node-project")
1305 self.init_repo("org/conflict-project")
1306 self.init_repo("org/noop-project")
1307 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00001308 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -07001309
1310 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001311 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1312 # see: https://github.com/jsocol/pystatsd/issues/61
1313 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001314 os.environ['STATSD_PORT'] = str(self.statsd.port)
1315 self.statsd.start()
1316 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001317 reload_module(statsd)
1318 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001319
1320 self.gearman_server = FakeGearmanServer()
1321
1322 self.config.set('gearman', 'port', str(self.gearman_server.port))
James E. Blaire47eb772017-02-02 17:19:40 -08001323 self.log.info("Gearman server on port %s" %
1324 (self.gearman_server.port,))
Clark Boylanb640e052014-04-03 16:41:46 -07001325
James E. Blaire511d2f2016-12-08 15:22:26 -08001326 gerritsource.GerritSource.replication_timeout = 1.5
1327 gerritsource.GerritSource.replication_retry_interval = 0.5
1328 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001329
Joshua Hesketh352264b2015-08-11 23:42:08 +10001330 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001331
Jan Hruban6b71aff2015-10-22 16:58:08 +02001332 self.event_queues = [
1333 self.sched.result_event_queue,
James E. Blair646322f2017-01-27 15:50:34 -08001334 self.sched.trigger_event_queue,
1335 self.sched.management_event_queue
Jan Hruban6b71aff2015-10-22 16:58:08 +02001336 ]
1337
James E. Blairfef78942016-03-11 16:28:56 -08001338 self.configure_connections()
Joshua Hesketh352264b2015-08-11 23:42:08 +10001339 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001340
Clark Boylanb640e052014-04-03 16:41:46 -07001341 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001342 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001343 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001344 return FakeURLOpener(self.upstream_root, *args, **kw)
1345
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001346 old_urlopen = urllib.request.urlopen
1347 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001348
James E. Blair3f876d52016-07-22 13:07:14 -07001349 self._startMerger()
James E. Blair3f876d52016-07-22 13:07:14 -07001350
Paul Belanger174a8272017-03-14 13:20:10 -04001351 self.executor_server = RecordingExecutorServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001352 self.config, self.connections,
James E. Blair854f8892017-02-02 11:25:39 -08001353 jobdir_root=self.test_root,
James E. Blaira92cbc82017-01-23 14:56:49 -08001354 _run_ansible=self.run_ansible,
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001355 _test_root=self.test_root,
1356 keep_jobdir=KEEP_TEMPDIRS)
Paul Belanger174a8272017-03-14 13:20:10 -04001357 self.executor_server.start()
1358 self.history = self.executor_server.build_history
1359 self.builds = self.executor_server.running_builds
James E. Blaire1767bc2016-08-02 10:00:27 -07001360
Paul Belanger174a8272017-03-14 13:20:10 -04001361 self.executor_client = zuul.executor.client.ExecutorClient(
James E. Blair92e953a2017-03-07 13:08:47 -08001362 self.config, self.sched)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001363 self.merge_client = zuul.merger.client.MergeClient(
1364 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001365 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001366 self.zk = zuul.zk.ZooKeeper()
James E. Blair0d5a36e2017-02-21 10:53:44 -05001367 self.zk.connect(self.zk_config)
James E. Blairdce6cea2016-12-20 16:45:32 -08001368
James E. Blair0d5a36e2017-02-21 10:53:44 -05001369 self.fake_nodepool = FakeNodepool(
1370 self.zk_chroot_fixture.zookeeper_host,
1371 self.zk_chroot_fixture.zookeeper_port,
1372 self.zk_chroot_fixture.zookeeper_chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001373
Paul Belanger174a8272017-03-14 13:20:10 -04001374 self.sched.setExecutor(self.executor_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001375 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001376 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001377 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001378
Paul Belanger88ef0ea2015-12-23 11:57:02 -05001379 self.webapp = zuul.webapp.WebApp(
1380 self.sched, port=0, listen_address='127.0.0.1')
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001381 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001382
1383 self.sched.start()
Clark Boylanb640e052014-04-03 16:41:46 -07001384 self.webapp.start()
1385 self.rpc.start()
Paul Belanger174a8272017-03-14 13:20:10 -04001386 self.executor_client.gearman.waitForServer()
Clark Boylanb640e052014-04-03 16:41:46 -07001387 self.addCleanup(self.shutdown)
1388
James E. Blairb9c0d772017-03-03 14:34:49 -08001389 self.sched.reconfigure(self.config)
1390 self.sched.resume()
1391
James E. Blaire18d4602017-01-05 11:17:28 -08001392 def tearDown(self):
1393 super(ZuulTestCase, self).tearDown()
1394 self.assertFinalState()
1395
James E. Blairfef78942016-03-11 16:28:56 -08001396 def configure_connections(self):
James E. Blaire511d2f2016-12-08 15:22:26 -08001397 # Set up gerrit related fakes
1398 # Set a changes database so multiple FakeGerrit's can report back to
1399 # a virtual canonical database given by the configured hostname
1400 self.gerrit_changes_dbs = {}
1401
1402 def getGerritConnection(driver, name, config):
1403 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1404 con = FakeGerritConnection(driver, name, config,
1405 changes_db=db,
1406 upstream_root=self.upstream_root)
1407 self.event_queues.append(con.event_queue)
1408 setattr(self, 'fake_' + name, con)
1409 return con
1410
1411 self.useFixture(fixtures.MonkeyPatch(
1412 'zuul.driver.gerrit.GerritDriver.getConnection',
1413 getGerritConnection))
1414
1415 # Set up smtp related fakes
Joshua Heskethd78b4482015-09-14 16:56:34 -06001416 # TODO(jhesketh): This should come from lib.connections for better
1417 # coverage
Joshua Hesketh352264b2015-08-11 23:42:08 +10001418 # Register connections from the config
1419 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001420
Joshua Hesketh352264b2015-08-11 23:42:08 +10001421 def FakeSMTPFactory(*args, **kw):
1422 args = [self.smtp_messages] + list(args)
1423 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001424
Joshua Hesketh352264b2015-08-11 23:42:08 +10001425 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001426
James E. Blaire511d2f2016-12-08 15:22:26 -08001427 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001428 self.connections = zuul.lib.connections.ConnectionRegistry()
James E. Blaire511d2f2016-12-08 15:22:26 -08001429 self.connections.configure(self.config)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001430
James E. Blair83005782015-12-11 14:46:03 -08001431 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001432 # This creates the per-test configuration object. It can be
1433 # overriden by subclasses, but should not need to be since it
1434 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001435 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001436 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair2a629ec2015-12-22 15:32:02 -08001437 if hasattr(self, 'tenant_config_file'):
1438 self.config.set('zuul', 'tenant_config', self.tenant_config_file)
James E. Blair96c6bf82016-01-15 16:20:40 -08001439 git_path = os.path.join(
1440 os.path.dirname(
1441 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1442 'git')
1443 if os.path.exists(git_path):
1444 for reponame in os.listdir(git_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001445 project = reponame.replace('_', '/')
1446 self.copyDirToRepo(project,
James E. Blair96c6bf82016-01-15 16:20:40 -08001447 os.path.join(git_path, reponame))
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001448 self.setupAllProjectKeys()
1449
1450 def setupAllProjectKeys(self):
1451 if self.create_project_keys:
1452 return
1453
1454 path = self.config.get('zuul', 'tenant_config')
1455 with open(os.path.join(FIXTURE_DIR, path)) as f:
1456 tenant_config = yaml.safe_load(f.read())
1457 for tenant in tenant_config:
1458 sources = tenant['tenant']['source']
1459 for source, conf in sources.items():
1460 for project in conf.get('config-repos', []):
1461 self.setupProjectKeys(source, project)
1462 for project in conf.get('project-repos', []):
1463 self.setupProjectKeys(source, project)
1464
1465 def setupProjectKeys(self, source, project):
1466 # Make sure we set up an RSA key for the project so that we
1467 # don't spend time generating one:
1468
1469 key_root = os.path.join(self.state_root, 'keys')
1470 if not os.path.isdir(key_root):
1471 os.mkdir(key_root, 0o700)
1472 private_key_file = os.path.join(key_root, source, project + '.pem')
1473 private_key_dir = os.path.dirname(private_key_file)
1474 self.log.debug("Installing test keys for project %s at %s" % (
1475 project, private_key_file))
1476 if not os.path.isdir(private_key_dir):
1477 os.makedirs(private_key_dir)
1478 with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
1479 with open(private_key_file, 'w') as o:
1480 o.write(i.read())
James E. Blair96c6bf82016-01-15 16:20:40 -08001481
James E. Blair498059b2016-12-20 13:50:13 -08001482 def setupZK(self):
1483 self.zk_chroot_fixture = self.useFixture(ChrootedKazooFixture())
James E. Blair0d5a36e2017-02-21 10:53:44 -05001484 self.zk_config = '%s:%s%s' % (
James E. Blairdce6cea2016-12-20 16:45:32 -08001485 self.zk_chroot_fixture.zookeeper_host,
1486 self.zk_chroot_fixture.zookeeper_port,
1487 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08001488
James E. Blair96c6bf82016-01-15 16:20:40 -08001489 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001490 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08001491
1492 files = {}
1493 for (dirpath, dirnames, filenames) in os.walk(source_path):
1494 for filename in filenames:
1495 test_tree_filepath = os.path.join(dirpath, filename)
1496 common_path = os.path.commonprefix([test_tree_filepath,
1497 source_path])
1498 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1499 with open(test_tree_filepath, 'r') as f:
1500 content = f.read()
1501 files[relative_filepath] = content
1502 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001503 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08001504
James E. Blaire18d4602017-01-05 11:17:28 -08001505 def assertNodepoolState(self):
1506 # Make sure that there are no pending requests
1507
1508 requests = self.fake_nodepool.getNodeRequests()
1509 self.assertEqual(len(requests), 0)
1510
1511 nodes = self.fake_nodepool.getNodes()
1512 for node in nodes:
1513 self.assertFalse(node['_lock'], "Node %s is locked" %
1514 (node['_oid'],))
1515
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001516 def assertNoGeneratedKeys(self):
1517 # Make sure that Zuul did not generate any project keys
1518 # (unless it was supposed to).
1519
1520 if self.create_project_keys:
1521 return
1522
1523 with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
1524 test_key = i.read()
1525
1526 key_root = os.path.join(self.state_root, 'keys')
1527 for root, dirname, files in os.walk(key_root):
1528 for fn in files:
1529 with open(os.path.join(root, fn)) as f:
1530 self.assertEqual(test_key, f.read())
1531
Clark Boylanb640e052014-04-03 16:41:46 -07001532 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001533 # Make sure that git.Repo objects have been garbage collected.
1534 repos = []
1535 gc.collect()
1536 for obj in gc.get_objects():
1537 if isinstance(obj, git.Repo):
James E. Blairc43525f2017-03-03 11:10:26 -08001538 self.log.debug("Leaked git repo object: %s" % repr(obj))
Clark Boylanb640e052014-04-03 16:41:46 -07001539 repos.append(obj)
1540 self.assertEqual(len(repos), 0)
1541 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08001542 self.assertNodepoolState()
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001543 self.assertNoGeneratedKeys()
James E. Blair83005782015-12-11 14:46:03 -08001544 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001545 for tenant in self.sched.abide.tenants.values():
1546 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001547 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001548 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001549
1550 def shutdown(self):
1551 self.log.debug("Shutting down after tests")
Paul Belanger174a8272017-03-14 13:20:10 -04001552 self.executor_client.stop()
James E. Blair3f876d52016-07-22 13:07:14 -07001553 self.merge_server.stop()
1554 self.merge_server.join()
Clark Boylanb640e052014-04-03 16:41:46 -07001555 self.merge_client.stop()
Paul Belanger174a8272017-03-14 13:20:10 -04001556 self.executor_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001557 self.sched.stop()
1558 self.sched.join()
1559 self.statsd.stop()
1560 self.statsd.join()
1561 self.webapp.stop()
1562 self.webapp.join()
1563 self.rpc.stop()
1564 self.rpc.join()
1565 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08001566 self.fake_nodepool.stop()
1567 self.zk.disconnect()
Clark Boylanb640e052014-04-03 16:41:46 -07001568 threads = threading.enumerate()
1569 if len(threads) > 1:
1570 self.log.error("More than one thread is running: %s" % threads)
James E. Blair6ac368c2016-12-22 18:07:20 -08001571 self.printHistory()
Clark Boylanb640e052014-04-03 16:41:46 -07001572
1573 def init_repo(self, project):
1574 parts = project.split('/')
1575 path = os.path.join(self.upstream_root, *parts[:-1])
1576 if not os.path.exists(path):
1577 os.makedirs(path)
1578 path = os.path.join(self.upstream_root, project)
1579 repo = git.Repo.init(path)
1580
Morgan Fainberg78c301a2016-07-14 13:47:01 -07001581 with repo.config_writer() as config_writer:
1582 config_writer.set_value('user', 'email', 'user@example.com')
1583 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07001584
Clark Boylanb640e052014-04-03 16:41:46 -07001585 repo.index.commit('initial commit')
1586 master = repo.create_head('master')
Clark Boylanb640e052014-04-03 16:41:46 -07001587
James E. Blair97d902e2014-08-21 13:25:56 -07001588 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001589 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001590 repo.git.clean('-x', '-f', '-d')
1591
James E. Blair97d902e2014-08-21 13:25:56 -07001592 def create_branch(self, project, branch):
1593 path = os.path.join(self.upstream_root, project)
1594 repo = git.Repo.init(path)
1595 fn = os.path.join(path, 'README')
1596
1597 branch_head = repo.create_head(branch)
1598 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001599 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001600 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001601 f.close()
1602 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001603 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001604
James E. Blair97d902e2014-08-21 13:25:56 -07001605 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001606 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001607 repo.git.clean('-x', '-f', '-d')
1608
Sachi King9f16d522016-03-16 12:20:45 +11001609 def create_commit(self, project):
1610 path = os.path.join(self.upstream_root, project)
1611 repo = git.Repo(path)
1612 repo.head.reference = repo.heads['master']
1613 file_name = os.path.join(path, 'README')
1614 with open(file_name, 'a') as f:
1615 f.write('creating fake commit\n')
1616 repo.index.add([file_name])
1617 commit = repo.index.commit('Creating a fake commit')
1618 return commit.hexsha
1619
James E. Blairb8c16472015-05-05 14:55:26 -07001620 def orderedRelease(self):
1621 # Run one build at a time to ensure non-race order:
1622 while len(self.builds):
1623 self.release(self.builds[0])
1624 self.waitUntilSettled()
1625
Clark Boylanb640e052014-04-03 16:41:46 -07001626 def release(self, job):
1627 if isinstance(job, FakeBuild):
1628 job.release()
1629 else:
1630 job.waiting = False
1631 self.log.debug("Queued job %s released" % job.unique)
1632 self.gearman_server.wakeConnections()
1633
1634 def getParameter(self, job, name):
1635 if isinstance(job, FakeBuild):
1636 return job.parameters[name]
1637 else:
1638 parameters = json.loads(job.arguments)
1639 return parameters[name]
1640
Clark Boylanb640e052014-04-03 16:41:46 -07001641 def haveAllBuildsReported(self):
1642 # See if Zuul is waiting on a meta job to complete
Paul Belanger174a8272017-03-14 13:20:10 -04001643 if self.executor_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07001644 return False
1645 # Find out if every build that the worker has completed has been
1646 # reported back to Zuul. If it hasn't then that means a Gearman
1647 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07001648 for build in self.history:
Paul Belanger174a8272017-03-14 13:20:10 -04001649 zbuild = self.executor_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07001650 if not zbuild:
1651 # It has already been reported
1652 continue
1653 # It hasn't been reported yet.
1654 return False
1655 # Make sure that none of the worker connections are in GRAB_WAIT
Paul Belanger174a8272017-03-14 13:20:10 -04001656 for connection in self.executor_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001657 if connection.state == 'GRAB_WAIT':
1658 return False
1659 return True
1660
1661 def areAllBuildsWaiting(self):
Paul Belanger174a8272017-03-14 13:20:10 -04001662 builds = self.executor_client.builds.values()
Clark Boylanb640e052014-04-03 16:41:46 -07001663 for build in builds:
1664 client_job = None
Paul Belanger174a8272017-03-14 13:20:10 -04001665 for conn in self.executor_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001666 for j in conn.related_jobs.values():
1667 if j.unique == build.uuid:
1668 client_job = j
1669 break
1670 if not client_job:
1671 self.log.debug("%s is not known to the gearman client" %
1672 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001673 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001674 if not client_job.handle:
1675 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001676 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001677 server_job = self.gearman_server.jobs.get(client_job.handle)
1678 if not server_job:
1679 self.log.debug("%s is not known to the gearman server" %
1680 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001681 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001682 if not hasattr(server_job, 'waiting'):
1683 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001684 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001685 if server_job.waiting:
1686 continue
James E. Blair17302972016-08-10 16:11:42 -07001687 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08001688 self.log.debug("%s has not reported start" % build)
1689 return False
Paul Belanger174a8272017-03-14 13:20:10 -04001690 worker_build = self.executor_server.job_builds.get(
1691 server_job.unique)
James E. Blair962220f2016-08-03 11:22:38 -07001692 if worker_build:
1693 if worker_build.isWaiting():
1694 continue
1695 else:
1696 self.log.debug("%s is running" % worker_build)
1697 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001698 else:
James E. Blair962220f2016-08-03 11:22:38 -07001699 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001700 return False
1701 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001702
James E. Blairdce6cea2016-12-20 16:45:32 -08001703 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001704 if self.fake_nodepool.paused:
1705 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08001706 if self.sched.nodepool.requests:
1707 return False
1708 return True
1709
Jan Hruban6b71aff2015-10-22 16:58:08 +02001710 def eventQueuesEmpty(self):
1711 for queue in self.event_queues:
1712 yield queue.empty()
1713
1714 def eventQueuesJoin(self):
1715 for queue in self.event_queues:
1716 queue.join()
1717
Clark Boylanb640e052014-04-03 16:41:46 -07001718 def waitUntilSettled(self):
1719 self.log.debug("Waiting until settled...")
1720 start = time.time()
1721 while True:
Clint Byruma9626572017-02-22 14:04:00 -05001722 if time.time() - start > self.wait_timeout:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001723 self.log.error("Timeout waiting for Zuul to settle")
1724 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07001725 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001726 self.log.error(" %s: %s" % (queue, queue.empty()))
1727 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07001728 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001729 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07001730 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001731 self.log.error("All requests completed: %s" %
1732 (self.areAllNodeRequestsComplete(),))
1733 self.log.error("Merge client jobs: %s" %
1734 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07001735 raise Exception("Timeout waiting for Zuul to settle")
1736 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07001737
Paul Belanger174a8272017-03-14 13:20:10 -04001738 self.executor_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07001739 # have all build states propogated to zuul?
1740 if self.haveAllBuildsReported():
1741 # Join ensures that the queue is empty _and_ events have been
1742 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001743 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001744 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001745 if (not self.merge_client.jobs and
Clark Boylanb640e052014-04-03 16:41:46 -07001746 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08001747 self.areAllBuildsWaiting() and
James E. Blair36c611a2017-02-06 15:59:43 -08001748 self.areAllNodeRequestsComplete() and
1749 all(self.eventQueuesEmpty())):
1750 # The queue empty check is placed at the end to
1751 # ensure that if a component adds an event between
1752 # when locked the run handler and checked that the
1753 # components were stable, we don't erroneously
1754 # report that we are settled.
Clark Boylanb640e052014-04-03 16:41:46 -07001755 self.sched.run_handler_lock.release()
Paul Belanger174a8272017-03-14 13:20:10 -04001756 self.executor_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001757 self.log.debug("...settled.")
1758 return
1759 self.sched.run_handler_lock.release()
Paul Belanger174a8272017-03-14 13:20:10 -04001760 self.executor_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001761 self.sched.wake_event.wait(0.1)
1762
1763 def countJobResults(self, jobs, result):
1764 jobs = filter(lambda x: x.result == result, jobs)
1765 return len(jobs)
1766
James E. Blair96c6bf82016-01-15 16:20:40 -08001767 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07001768 for job in self.history:
1769 if (job.name == name and
1770 (project is None or
1771 job.parameters['ZUUL_PROJECT'] == project)):
1772 return job
Clark Boylanb640e052014-04-03 16:41:46 -07001773 raise Exception("Unable to find job %s in history" % name)
1774
1775 def assertEmptyQueues(self):
1776 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001777 for tenant in self.sched.abide.tenants.values():
1778 for pipeline in tenant.layout.pipelines.values():
1779 for queue in pipeline.queues:
1780 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10001781 print('pipeline %s queue %s contents %s' % (
1782 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08001783 self.assertEqual(len(queue.queue), 0,
1784 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001785
1786 def assertReportedStat(self, key, value=None, kind=None):
1787 start = time.time()
1788 while time.time() < (start + 5):
1789 for stat in self.statsd.stats:
Clark Boylanb640e052014-04-03 16:41:46 -07001790 k, v = stat.split(':')
1791 if key == k:
1792 if value is None and kind is None:
1793 return
1794 elif value:
1795 if value == v:
1796 return
1797 elif kind:
1798 if v.endswith('|' + kind):
1799 return
1800 time.sleep(0.1)
1801
Clark Boylanb640e052014-04-03 16:41:46 -07001802 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001803
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001804 def assertBuilds(self, builds):
1805 """Assert that the running builds are as described.
1806
1807 The list of running builds is examined and must match exactly
1808 the list of builds described by the input.
1809
1810 :arg list builds: A list of dictionaries. Each item in the
1811 list must match the corresponding build in the build
1812 history, and each element of the dictionary must match the
1813 corresponding attribute of the build.
1814
1815 """
James E. Blair3158e282016-08-19 09:34:11 -07001816 try:
1817 self.assertEqual(len(self.builds), len(builds))
1818 for i, d in enumerate(builds):
1819 for k, v in d.items():
1820 self.assertEqual(
1821 getattr(self.builds[i], k), v,
1822 "Element %i in builds does not match" % (i,))
1823 except Exception:
1824 for build in self.builds:
1825 self.log.error("Running build: %s" % build)
1826 else:
1827 self.log.error("No running builds")
1828 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001829
James E. Blairb536ecc2016-08-31 10:11:42 -07001830 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001831 """Assert that the completed builds are as described.
1832
1833 The list of completed builds is examined and must match
1834 exactly the list of builds described by the input.
1835
1836 :arg list history: A list of dictionaries. Each item in the
1837 list must match the corresponding build in the build
1838 history, and each element of the dictionary must match the
1839 corresponding attribute of the build.
1840
James E. Blairb536ecc2016-08-31 10:11:42 -07001841 :arg bool ordered: If true, the history must match the order
1842 supplied, if false, the builds are permitted to have
1843 arrived in any order.
1844
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001845 """
James E. Blairb536ecc2016-08-31 10:11:42 -07001846 def matches(history_item, item):
1847 for k, v in item.items():
1848 if getattr(history_item, k) != v:
1849 return False
1850 return True
James E. Blair3158e282016-08-19 09:34:11 -07001851 try:
1852 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07001853 if ordered:
1854 for i, d in enumerate(history):
1855 if not matches(self.history[i], d):
1856 raise Exception(
K Jonathan Harkercc3a6f02017-02-22 19:08:06 -08001857 "Element %i in history does not match %s" %
1858 (i, self.history[i]))
James E. Blairb536ecc2016-08-31 10:11:42 -07001859 else:
1860 unseen = self.history[:]
1861 for i, d in enumerate(history):
1862 found = False
1863 for unseen_item in unseen:
1864 if matches(unseen_item, d):
1865 found = True
1866 unseen.remove(unseen_item)
1867 break
1868 if not found:
1869 raise Exception("No match found for element %i "
1870 "in history" % (i,))
1871 if unseen:
1872 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07001873 except Exception:
1874 for build in self.history:
1875 self.log.error("Completed build: %s" % build)
1876 else:
1877 self.log.error("No completed builds")
1878 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001879
James E. Blair6ac368c2016-12-22 18:07:20 -08001880 def printHistory(self):
1881 """Log the build history.
1882
1883 This can be useful during tests to summarize what jobs have
1884 completed.
1885
1886 """
1887 self.log.debug("Build history:")
1888 for build in self.history:
1889 self.log.debug(build)
1890
James E. Blair59fdbac2015-12-07 17:08:06 -08001891 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001892 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1893
1894 def updateConfigLayout(self, path):
1895 root = os.path.join(self.test_root, "config")
Clint Byrumead6c562017-02-01 16:34:04 -08001896 if not os.path.exists(root):
1897 os.makedirs(root)
James E. Blairf84026c2015-12-08 16:11:46 -08001898 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1899 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05001900- tenant:
1901 name: openstack
1902 source:
1903 gerrit:
1904 config-repos:
1905 - %s
1906 """ % path)
James E. Blairf84026c2015-12-08 16:11:46 -08001907 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05001908 self.config.set('zuul', 'tenant_config',
1909 os.path.join(FIXTURE_DIR, f.name))
Ricardo Carrillo Cruz22994f92016-12-02 11:41:58 +00001910 self.setupAllProjectKeys()
James E. Blair14abdf42015-12-09 16:11:53 -08001911
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001912 def addCommitToRepo(self, project, message, files,
1913 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08001914 path = os.path.join(self.upstream_root, project)
1915 repo = git.Repo(path)
1916 repo.head.reference = branch
1917 zuul.merger.merger.reset_repo_to_head(repo)
1918 for fn, content in files.items():
1919 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08001920 try:
1921 os.makedirs(os.path.dirname(fn))
1922 except OSError:
1923 pass
James E. Blair14abdf42015-12-09 16:11:53 -08001924 with open(fn, 'w') as f:
1925 f.write(content)
1926 repo.index.add([fn])
1927 commit = repo.index.commit(message)
Clint Byrum58264dc2017-02-07 21:21:22 -08001928 before = repo.heads[branch].commit
James E. Blair14abdf42015-12-09 16:11:53 -08001929 repo.heads[branch].commit = commit
1930 repo.head.reference = branch
1931 repo.git.clean('-x', '-f', '-d')
1932 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001933 if tag:
1934 repo.create_tag(tag)
Clint Byrum58264dc2017-02-07 21:21:22 -08001935 return before
1936
1937 def commitLayoutUpdate(self, orig_name, source_name):
1938 source_path = os.path.join(self.test_root, 'upstream',
Clint Byrum678e2c32017-03-16 16:27:21 -07001939 source_name)
1940 to_copy = ['zuul.yaml']
1941 for playbook in os.listdir(os.path.join(source_path, 'playbooks')):
1942 to_copy.append('playbooks/{}'.format(playbook))
1943 commit_data = {}
1944 for source_file in to_copy:
1945 source_file_path = os.path.join(source_path, source_file)
1946 with open(source_file_path, 'r') as nt:
1947 commit_data[source_file] = nt.read()
1948 before = self.addCommitToRepo(
1949 orig_name, 'Pulling content from %s' % source_name,
1950 commit_data)
Clint Byrum58264dc2017-02-07 21:21:22 -08001951 return before
James E. Blair3f876d52016-07-22 13:07:14 -07001952
James E. Blair7fc8daa2016-08-08 15:37:15 -07001953 def addEvent(self, connection, event):
1954 """Inject a Fake (Gerrit) event.
1955
1956 This method accepts a JSON-encoded event and simulates Zuul
1957 having received it from Gerrit. It could (and should)
1958 eventually apply to any connection type, but is currently only
1959 used with Gerrit connections. The name of the connection is
1960 used to look up the corresponding server, and the event is
1961 simulated as having been received by all Zuul connections
1962 attached to that server. So if two Gerrit connections in Zuul
1963 are connected to the same Gerrit server, and you invoke this
1964 method specifying the name of one of them, the event will be
1965 received by both.
1966
1967 .. note::
1968
1969 "self.fake_gerrit.addEvent" calls should be migrated to
1970 this method.
1971
1972 :arg str connection: The name of the connection corresponding
1973 to the gerrit server.
1974 :arg str event: The JSON-encoded event.
1975
1976 """
1977 specified_conn = self.connections.connections[connection]
1978 for conn in self.connections.connections.values():
1979 if (isinstance(conn, specified_conn.__class__) and
1980 specified_conn.server == conn.server):
1981 conn.addEvent(event)
1982
James E. Blair3f876d52016-07-22 13:07:14 -07001983
1984class AnsibleZuulTestCase(ZuulTestCase):
Paul Belanger174a8272017-03-14 13:20:10 -04001985 """ZuulTestCase but with an actual ansible executor running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07001986 run_ansible = True
Joshua Hesketh25695cb2017-03-06 12:50:04 +11001987
Joshua Heskethd78b4482015-09-14 16:56:34 -06001988
1989class ZuulDBTestCase(ZuulTestCase):
James E. Blair82844892017-03-06 10:55:26 -08001990 def setup_config(self):
1991 super(ZuulDBTestCase, self).setup_config()
Joshua Heskethd78b4482015-09-14 16:56:34 -06001992 for section_name in self.config.sections():
1993 con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
1994 section_name, re.I)
1995 if not con_match:
1996 continue
1997
1998 if self.config.get(section_name, 'driver') == 'sql':
1999 f = MySQLSchemaFixture()
2000 self.useFixture(f)
2001 if (self.config.get(section_name, 'dburi') ==
2002 '$MYSQL_FIXTURE_DBURI$'):
2003 self.config.set(section_name, 'dburi', f.dburi)