blob: 59cc9aeff8dabfd1b26103336f5fee8e1cd6904d [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070019import gc
20import hashlib
21import json
22import logging
23import os
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070025from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070026import random
27import re
28import select
29import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030030from six.moves import reload_module
James E. Blair1c236df2017-02-01 14:07:24 -080031from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070032import socket
33import string
34import subprocess
35import swiftclient
James E. Blair1c236df2017-02-01 14:07:24 -080036import sys
James E. Blairf84026c2015-12-08 16:11:46 -080037import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070038import threading
39import time
Clark Boylanb640e052014-04-03 16:41:46 -070040
41import git
42import gear
43import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080044import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080045import kazoo.exceptions
Clark Boylanb640e052014-04-03 16:41:46 -070046import statsd
47import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080048import testtools.content
49import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080050from git.exc import NoSuchPathError
Clark Boylanb640e052014-04-03 16:41:46 -070051
James E. Blaire511d2f2016-12-08 15:22:26 -080052import zuul.driver.gerrit.gerritsource as gerritsource
53import zuul.driver.gerrit.gerritconnection as gerritconnection
Clark Boylanb640e052014-04-03 16:41:46 -070054import zuul.scheduler
55import zuul.webapp
56import zuul.rpclistener
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +100057import zuul.launcher.server
58import zuul.launcher.client
Clark Boylanb640e052014-04-03 16:41:46 -070059import zuul.lib.swift
James E. Blair83005782015-12-11 14:46:03 -080060import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070061import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070062import zuul.merger.merger
63import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070064import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080065import zuul.zk
Clark Boylanb640e052014-04-03 16:41:46 -070066
67FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
68 'fixtures')
James E. Blair97d902e2014-08-21 13:25:56 -070069USE_TEMPDIR = True
Clark Boylanb640e052014-04-03 16:41:46 -070070
Clark Boylanb640e052014-04-03 16:41:46 -070071
72def repack_repo(path):
73 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
74 output = subprocess.Popen(cmd, close_fds=True,
75 stdout=subprocess.PIPE,
76 stderr=subprocess.PIPE)
77 out = output.communicate()
78 if output.returncode:
79 raise Exception("git repack returned %d" % output.returncode)
80 return out
81
82
83def random_sha1():
84 return hashlib.sha1(str(random.random())).hexdigest()
85
86
James E. Blaira190f3b2015-01-05 14:56:54 -080087def iterate_timeout(max_seconds, purpose):
88 start = time.time()
89 count = 0
90 while (time.time() < start + max_seconds):
91 count += 1
92 yield count
93 time.sleep(0)
94 raise Exception("Timeout waiting for %s" % purpose)
95
96
Clark Boylanb640e052014-04-03 16:41:46 -070097class ChangeReference(git.Reference):
98 _common_path_default = "refs/changes"
99 _points_to_commits_only = True
100
101
102class FakeChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700103 categories = {'approved': ('Approved', -1, 1),
104 'code-review': ('Code-Review', -2, 2),
105 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700106
107 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700108 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700109 self.gerrit = gerrit
110 self.reported = 0
111 self.queried = 0
112 self.patchsets = []
113 self.number = number
114 self.project = project
115 self.branch = branch
116 self.subject = subject
117 self.latest_patchset = 0
118 self.depends_on_change = None
119 self.needed_by_changes = []
120 self.fail_merge = False
121 self.messages = []
122 self.data = {
123 'branch': branch,
124 'comments': [],
125 'commitMessage': subject,
126 'createdOn': time.time(),
127 'id': 'I' + random_sha1(),
128 'lastUpdated': time.time(),
129 'number': str(number),
130 'open': status == 'NEW',
131 'owner': {'email': 'user@example.com',
132 'name': 'User Name',
133 'username': 'username'},
134 'patchSets': self.patchsets,
135 'project': project,
136 'status': status,
137 'subject': subject,
138 'submitRecords': [],
139 'url': 'https://hostname/%s' % number}
140
141 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700142 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700143 self.data['submitRecords'] = self.getSubmitRecords()
144 self.open = status == 'NEW'
145
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700146 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700147 path = os.path.join(self.upstream_root, self.project)
148 repo = git.Repo(path)
149 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
150 self.latest_patchset),
151 'refs/tags/init')
152 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700153 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700154 repo.git.clean('-x', '-f', '-d')
155
156 path = os.path.join(self.upstream_root, self.project)
157 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700158 for fn, content in files.items():
159 fn = os.path.join(path, fn)
160 with open(fn, 'w') as f:
161 f.write(content)
162 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700163 else:
164 for fni in range(100):
165 fn = os.path.join(path, str(fni))
166 f = open(fn, 'w')
167 for ci in range(4096):
168 f.write(random.choice(string.printable))
169 f.close()
170 repo.index.add([fn])
171
172 r = repo.index.commit(msg)
173 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700174 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700175 repo.git.clean('-x', '-f', '-d')
176 repo.heads['master'].checkout()
177 return r
178
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700179 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700180 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700181 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700182 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700183 data = ("test %s %s %s\n" %
184 (self.branch, self.number, self.latest_patchset))
185 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700186 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700187 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700188 ps_files = [{'file': '/COMMIT_MSG',
189 'type': 'ADDED'},
190 {'file': 'README',
191 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700192 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700193 ps_files.append({'file': f, 'type': 'ADDED'})
194 d = {'approvals': [],
195 'createdOn': time.time(),
196 'files': ps_files,
197 'number': str(self.latest_patchset),
198 'ref': 'refs/changes/1/%s/%s' % (self.number,
199 self.latest_patchset),
200 'revision': c.hexsha,
201 'uploader': {'email': 'user@example.com',
202 'name': 'User name',
203 'username': 'user'}}
204 self.data['currentPatchSet'] = d
205 self.patchsets.append(d)
206 self.data['submitRecords'] = self.getSubmitRecords()
207
208 def getPatchsetCreatedEvent(self, patchset):
209 event = {"type": "patchset-created",
210 "change": {"project": self.project,
211 "branch": self.branch,
212 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
213 "number": str(self.number),
214 "subject": self.subject,
215 "owner": {"name": "User Name"},
216 "url": "https://hostname/3"},
217 "patchSet": self.patchsets[patchset - 1],
218 "uploader": {"name": "User Name"}}
219 return event
220
221 def getChangeRestoredEvent(self):
222 event = {"type": "change-restored",
223 "change": {"project": self.project,
224 "branch": self.branch,
225 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
226 "number": str(self.number),
227 "subject": self.subject,
228 "owner": {"name": "User Name"},
229 "url": "https://hostname/3"},
230 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100231 "patchSet": self.patchsets[-1],
232 "reason": ""}
233 return event
234
235 def getChangeAbandonedEvent(self):
236 event = {"type": "change-abandoned",
237 "change": {"project": self.project,
238 "branch": self.branch,
239 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
240 "number": str(self.number),
241 "subject": self.subject,
242 "owner": {"name": "User Name"},
243 "url": "https://hostname/3"},
244 "abandoner": {"name": "User Name"},
245 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700246 "reason": ""}
247 return event
248
249 def getChangeCommentEvent(self, patchset):
250 event = {"type": "comment-added",
251 "change": {"project": self.project,
252 "branch": self.branch,
253 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
254 "number": str(self.number),
255 "subject": self.subject,
256 "owner": {"name": "User Name"},
257 "url": "https://hostname/3"},
258 "patchSet": self.patchsets[patchset - 1],
259 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700260 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700261 "description": "Code-Review",
262 "value": "0"}],
263 "comment": "This is a comment"}
264 return event
265
Joshua Hesketh642824b2014-07-01 17:54:59 +1000266 def addApproval(self, category, value, username='reviewer_john',
267 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700268 if not granted_on:
269 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000270 approval = {
271 'description': self.categories[category][0],
272 'type': category,
273 'value': str(value),
274 'by': {
275 'username': username,
276 'email': username + '@example.com',
277 },
278 'grantedOn': int(granted_on)
279 }
Clark Boylanb640e052014-04-03 16:41:46 -0700280 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
281 if x['by']['username'] == username and x['type'] == category:
282 del self.patchsets[-1]['approvals'][i]
283 self.patchsets[-1]['approvals'].append(approval)
284 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000285 'author': {'email': 'author@example.com',
286 'name': 'Patchset Author',
287 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700288 'change': {'branch': self.branch,
289 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
290 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000291 'owner': {'email': 'owner@example.com',
292 'name': 'Change Owner',
293 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700294 'project': self.project,
295 'subject': self.subject,
296 'topic': 'master',
297 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000298 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700299 'patchSet': self.patchsets[-1],
300 'type': 'comment-added'}
301 self.data['submitRecords'] = self.getSubmitRecords()
302 return json.loads(json.dumps(event))
303
304 def getSubmitRecords(self):
305 status = {}
306 for cat in self.categories.keys():
307 status[cat] = 0
308
309 for a in self.patchsets[-1]['approvals']:
310 cur = status[a['type']]
311 cat_min, cat_max = self.categories[a['type']][1:]
312 new = int(a['value'])
313 if new == cat_min:
314 cur = new
315 elif abs(new) > abs(cur):
316 cur = new
317 status[a['type']] = cur
318
319 labels = []
320 ok = True
321 for typ, cat in self.categories.items():
322 cur = status[typ]
323 cat_min, cat_max = cat[1:]
324 if cur == cat_min:
325 value = 'REJECT'
326 ok = False
327 elif cur == cat_max:
328 value = 'OK'
329 else:
330 value = 'NEED'
331 ok = False
332 labels.append({'label': cat[0], 'status': value})
333 if ok:
334 return [{'status': 'OK'}]
335 return [{'status': 'NOT_READY',
336 'labels': labels}]
337
338 def setDependsOn(self, other, patchset):
339 self.depends_on_change = other
340 d = {'id': other.data['id'],
341 'number': other.data['number'],
342 'ref': other.patchsets[patchset - 1]['ref']
343 }
344 self.data['dependsOn'] = [d]
345
346 other.needed_by_changes.append(self)
347 needed = other.data.get('neededBy', [])
348 d = {'id': self.data['id'],
349 'number': self.data['number'],
350 'ref': self.patchsets[patchset - 1]['ref'],
351 'revision': self.patchsets[patchset - 1]['revision']
352 }
353 needed.append(d)
354 other.data['neededBy'] = needed
355
356 def query(self):
357 self.queried += 1
358 d = self.data.get('dependsOn')
359 if d:
360 d = d[0]
361 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
362 d['isCurrentPatchSet'] = True
363 else:
364 d['isCurrentPatchSet'] = False
365 return json.loads(json.dumps(self.data))
366
367 def setMerged(self):
368 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000369 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700370 return
371 if self.fail_merge:
372 return
373 self.data['status'] = 'MERGED'
374 self.open = False
375
376 path = os.path.join(self.upstream_root, self.project)
377 repo = git.Repo(path)
378 repo.heads[self.branch].commit = \
379 repo.commit(self.patchsets[-1]['revision'])
380
381 def setReported(self):
382 self.reported += 1
383
384
James E. Blaire511d2f2016-12-08 15:22:26 -0800385class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700386 """A Fake Gerrit connection for use in tests.
387
388 This subclasses
389 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
390 ability for tests to add changes to the fake Gerrit it represents.
391 """
392
Joshua Hesketh352264b2015-08-11 23:42:08 +1000393 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700394
James E. Blaire511d2f2016-12-08 15:22:26 -0800395 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700396 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800397 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000398 connection_config)
399
James E. Blair7fc8daa2016-08-08 15:37:15 -0700400 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700401 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
402 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000403 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700404 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200405 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700406
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700407 def addFakeChange(self, project, branch, subject, status='NEW',
408 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700409 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700410 self.change_number += 1
411 c = FakeChange(self, self.change_number, project, branch, subject,
412 upstream_root=self.upstream_root,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700413 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700414 self.changes[self.change_number] = c
415 return c
416
Clark Boylanb640e052014-04-03 16:41:46 -0700417 def review(self, project, changeid, message, action):
418 number, ps = changeid.split(',')
419 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000420
421 # Add the approval back onto the change (ie simulate what gerrit would
422 # do).
423 # Usually when zuul leaves a review it'll create a feedback loop where
424 # zuul's review enters another gerrit event (which is then picked up by
425 # zuul). However, we can't mimic this behaviour (by adding this
426 # approval event into the queue) as it stops jobs from checking what
427 # happens before this event is triggered. If a job needs to see what
428 # happens they can add their own verified event into the queue.
429 # Nevertheless, we can update change with the new review in gerrit.
430
James E. Blair8b5408c2016-08-08 15:37:46 -0700431 for cat in action.keys():
432 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000433 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000434
James E. Blair8b5408c2016-08-08 15:37:46 -0700435 # TODOv3(jeblair): can this be removed?
Joshua Hesketh642824b2014-07-01 17:54:59 +1000436 if 'label' in action:
437 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000438 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000439
Clark Boylanb640e052014-04-03 16:41:46 -0700440 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000441
Clark Boylanb640e052014-04-03 16:41:46 -0700442 if 'submit' in action:
443 change.setMerged()
444 if message:
445 change.setReported()
446
447 def query(self, number):
448 change = self.changes.get(int(number))
449 if change:
450 return change.query()
451 return {}
452
James E. Blairc494d542014-08-06 09:23:52 -0700453 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700454 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700455 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800456 if query.startswith('change:'):
457 # Query a specific changeid
458 changeid = query[len('change:'):]
459 l = [change.query() for change in self.changes.values()
460 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700461 elif query.startswith('message:'):
462 # Query the content of a commit message
463 msg = query[len('message:'):].strip()
464 l = [change.query() for change in self.changes.values()
465 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800466 else:
467 # Query all open changes
468 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700469 return l
James E. Blairc494d542014-08-06 09:23:52 -0700470
Joshua Hesketh352264b2015-08-11 23:42:08 +1000471 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700472 pass
473
Joshua Hesketh352264b2015-08-11 23:42:08 +1000474 def getGitUrl(self, project):
475 return os.path.join(self.upstream_root, project.name)
476
Clark Boylanb640e052014-04-03 16:41:46 -0700477
478class BuildHistory(object):
479 def __init__(self, **kw):
480 self.__dict__.update(kw)
481
482 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -0700483 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
484 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -0700485
486
487class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200488 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700489 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700490 self.url = url
491
492 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -0700493 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -0700494 path = res.path
495 project = '/'.join(path.split('/')[2:-2])
496 ret = '001e# service=git-upload-pack\n'
497 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
498 'multi_ack thin-pack side-band side-band-64k ofs-delta '
499 'shallow no-progress include-tag multi_ack_detailed no-done\n')
500 path = os.path.join(self.upstream_root, project)
501 repo = git.Repo(path)
502 for ref in repo.refs:
503 r = ref.object.hexsha + ' ' + ref.path + '\n'
504 ret += '%04x%s' % (len(r) + 4, r)
505 ret += '0000'
506 return ret
507
508
Clark Boylanb640e052014-04-03 16:41:46 -0700509class FakeStatsd(threading.Thread):
510 def __init__(self):
511 threading.Thread.__init__(self)
512 self.daemon = True
513 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
514 self.sock.bind(('', 0))
515 self.port = self.sock.getsockname()[1]
516 self.wake_read, self.wake_write = os.pipe()
517 self.stats = []
518
519 def run(self):
520 while True:
521 poll = select.poll()
522 poll.register(self.sock, select.POLLIN)
523 poll.register(self.wake_read, select.POLLIN)
524 ret = poll.poll()
525 for (fd, event) in ret:
526 if fd == self.sock.fileno():
527 data = self.sock.recvfrom(1024)
528 if not data:
529 return
530 self.stats.append(data[0])
531 if fd == self.wake_read:
532 return
533
534 def stop(self):
535 os.write(self.wake_write, '1\n')
536
537
James E. Blaire1767bc2016-08-02 10:00:27 -0700538class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -0700539 log = logging.getLogger("zuul.test")
540
James E. Blair34776ee2016-08-25 13:53:54 -0700541 def __init__(self, launch_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -0700542 self.daemon = True
James E. Blaire1767bc2016-08-02 10:00:27 -0700543 self.launch_server = launch_server
Clark Boylanb640e052014-04-03 16:41:46 -0700544 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -0700545 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -0700546 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -0700547 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -0700548 # TODOv3(jeblair): self.node is really "the image of the node
549 # assigned". We should rename it (self.node_image?) if we
550 # keep using it like this, or we may end up exposing more of
551 # the complexity around multi-node jobs here
552 # (self.nodes[0].image?)
553 self.node = None
554 if len(self.parameters.get('nodes')) == 1:
555 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -0700556 self.unique = self.parameters['ZUUL_UUID']
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100557 self.pipeline = self.parameters['ZUUL_PIPELINE']
558 self.project = self.parameters['ZUUL_PROJECT']
James E. Blair3f876d52016-07-22 13:07:14 -0700559 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -0700560 self.wait_condition = threading.Condition()
561 self.waiting = False
562 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -0500563 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -0700564 self.created = time.time()
James E. Blaire1767bc2016-08-02 10:00:27 -0700565 self.changes = None
566 if 'ZUUL_CHANGE_IDS' in self.parameters:
567 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -0700568
James E. Blair3158e282016-08-19 09:34:11 -0700569 def __repr__(self):
570 waiting = ''
571 if self.waiting:
572 waiting = ' [waiting]'
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100573 return '<FakeBuild %s:%s %s%s>' % (self.pipeline, self.name,
574 self.changes, waiting)
James E. Blair3158e282016-08-19 09:34:11 -0700575
Clark Boylanb640e052014-04-03 16:41:46 -0700576 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700577 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -0700578 self.wait_condition.acquire()
579 self.wait_condition.notify()
580 self.waiting = False
581 self.log.debug("Build %s released" % self.unique)
582 self.wait_condition.release()
583
584 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700585 """Return whether this build is being held.
586
587 :returns: Whether the build is being held.
588 :rtype: bool
589 """
590
Clark Boylanb640e052014-04-03 16:41:46 -0700591 self.wait_condition.acquire()
592 if self.waiting:
593 ret = True
594 else:
595 ret = False
596 self.wait_condition.release()
597 return ret
598
599 def _wait(self):
600 self.wait_condition.acquire()
601 self.waiting = True
602 self.log.debug("Build %s waiting" % self.unique)
603 self.wait_condition.wait()
604 self.wait_condition.release()
605
606 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -0700607 self.log.debug('Running build %s' % self.unique)
608
James E. Blaire1767bc2016-08-02 10:00:27 -0700609 if self.launch_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -0700610 self.log.debug('Holding build %s' % self.unique)
611 self._wait()
612 self.log.debug("Build %s continuing" % self.unique)
613
James E. Blair412fba82017-01-26 15:00:50 -0800614 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
James E. Blaira5dba232016-08-08 15:53:24 -0700615 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
James E. Blair412fba82017-01-26 15:00:50 -0800616 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
Clark Boylanb640e052014-04-03 16:41:46 -0700617 if self.aborted:
James E. Blair412fba82017-01-26 15:00:50 -0800618 result = (RecordingAnsibleJob.RESULT_ABORTED, None)
Paul Belanger71d98172016-11-08 10:56:31 -0500619 if self.requeue:
James E. Blair412fba82017-01-26 15:00:50 -0800620 result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
Clark Boylanb640e052014-04-03 16:41:46 -0700621
James E. Blaire1767bc2016-08-02 10:00:27 -0700622 return result
Clark Boylanb640e052014-04-03 16:41:46 -0700623
James E. Blaira5dba232016-08-08 15:53:24 -0700624 def shouldFail(self):
625 changes = self.launch_server.fail_tests.get(self.name, [])
626 for change in changes:
627 if self.hasChanges(change):
628 return True
629 return False
630
James E. Blaire7b99a02016-08-05 14:27:34 -0700631 def hasChanges(self, *changes):
632 """Return whether this build has certain changes in its git repos.
633
634 :arg FakeChange changes: One or more changes (varargs) that
635 are expected to be present (in order) in the git repository of
636 the active project.
637
638 :returns: Whether the build has the indicated changes.
639 :rtype: bool
640
641 """
Clint Byrum3343e3e2016-11-15 16:05:03 -0800642 for change in changes:
643 path = os.path.join(self.jobdir.git_root, change.project)
644 try:
645 repo = git.Repo(path)
646 except NoSuchPathError as e:
647 self.log.debug('%s' % e)
648 return False
649 ref = self.parameters['ZUUL_REF']
650 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
651 commit_message = '%s-1' % change.subject
652 self.log.debug("Checking if build %s has changes; commit_message "
653 "%s; repo_messages %s" % (self, commit_message,
654 repo_messages))
655 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -0700656 self.log.debug(" messages do not match")
657 return False
658 self.log.debug(" OK")
659 return True
660
Clark Boylanb640e052014-04-03 16:41:46 -0700661
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +1000662class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
James E. Blaire7b99a02016-08-05 14:27:34 -0700663 """An Ansible launcher to be used in tests.
664
665 :ivar bool hold_jobs_in_build: If true, when jobs are launched
666 they will report that they have started but then pause until
667 released before reporting completion. This attribute may be
668 changed at any time and will take effect for subsequently
669 launched builds, but previously held builds will still need to
670 be explicitly released.
671
672 """
James E. Blairf5dbd002015-12-23 15:26:17 -0800673 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -0700674 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -0800675 self._test_root = kw.pop('_test_root', False)
James E. Blairf5dbd002015-12-23 15:26:17 -0800676 super(RecordingLaunchServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -0700677 self.hold_jobs_in_build = False
678 self.lock = threading.Lock()
679 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -0700680 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -0700681 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -0700682 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -0800683
James E. Blaira5dba232016-08-08 15:53:24 -0700684 def failJob(self, name, change):
James E. Blaire7b99a02016-08-05 14:27:34 -0700685 """Instruct the launcher to report matching builds as failures.
686
687 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -0700688 :arg Change change: The :py:class:`~tests.base.FakeChange`
689 instance which should cause the job to fail. This job
690 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -0700691
692 """
James E. Blaire1767bc2016-08-02 10:00:27 -0700693 l = self.fail_tests.get(name, [])
694 l.append(change)
695 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -0800696
James E. Blair962220f2016-08-03 11:22:38 -0700697 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700698 """Release a held build.
699
700 :arg str regex: A regular expression which, if supplied, will
701 cause only builds with matching names to be released. If
702 not supplied, all builds will be released.
703
704 """
James E. Blair962220f2016-08-03 11:22:38 -0700705 builds = self.running_builds[:]
706 self.log.debug("Releasing build %s (%s)" % (regex,
707 len(self.running_builds)))
708 for build in builds:
709 if not regex or re.match(regex, build.name):
710 self.log.debug("Releasing build %s" %
711 (build.parameters['ZUUL_UUID']))
712 build.release()
713 else:
714 self.log.debug("Not releasing build %s" %
715 (build.parameters['ZUUL_UUID']))
716 self.log.debug("Done releasing builds %s (%s)" %
717 (regex, len(self.running_builds)))
718
James E. Blair17302972016-08-10 16:11:42 -0700719 def launchJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -0700720 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -0700721 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -0700722 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -0700723 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -0800724 args = json.loads(job.arguments)
725 args['zuul']['_test'] = dict(test_root=self._test_root)
726 job.arguments = json.dumps(args)
Joshua Hesketh50c21782016-10-13 21:34:14 +1100727 self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
728 self.job_workers[job.unique].run()
James E. Blair17302972016-08-10 16:11:42 -0700729
730 def stopJob(self, job):
731 self.log.debug("handle stop")
732 parameters = json.loads(job.arguments)
733 uuid = parameters['uuid']
734 for build in self.running_builds:
735 if build.unique == uuid:
736 build.aborted = True
737 build.release()
738 super(RecordingLaunchServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -0700739
Joshua Hesketh50c21782016-10-13 21:34:14 +1100740
741class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob):
James E. Blair412fba82017-01-26 15:00:50 -0800742 def runPlaybooks(self):
Joshua Hesketh50c21782016-10-13 21:34:14 +1100743 build = self.launcher_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -0800744 build.jobdir = self.jobdir
James E. Blaire1767bc2016-08-02 10:00:27 -0700745
James E. Blair412fba82017-01-26 15:00:50 -0800746 result = super(RecordingAnsibleJob, self).runPlaybooks()
747
Joshua Hesketh50c21782016-10-13 21:34:14 +1100748 self.launcher_server.lock.acquire()
749 self.launcher_server.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -0700750 BuildHistory(name=build.name, result=result, changes=build.changes,
751 node=build.node, uuid=build.unique,
752 parameters=build.parameters,
James E. Blaire1767bc2016-08-02 10:00:27 -0700753 pipeline=build.parameters['ZUUL_PIPELINE'])
754 )
Joshua Hesketh50c21782016-10-13 21:34:14 +1100755 self.launcher_server.running_builds.remove(build)
756 del self.launcher_server.job_builds[self.job.unique]
757 self.launcher_server.lock.release()
James E. Blair412fba82017-01-26 15:00:50 -0800758 return result
759
Monty Taylorc231d932017-02-03 09:57:15 -0600760 def runAnsible(self, cmd, timeout, secure=False):
James E. Blair412fba82017-01-26 15:00:50 -0800761 build = self.launcher_server.job_builds[self.job.unique]
762
763 if self.launcher_server._run_ansible:
Monty Taylorc231d932017-02-03 09:57:15 -0600764 result = super(RecordingAnsibleJob, self).runAnsible(
765 cmd, timeout, secure=secure)
James E. Blair412fba82017-01-26 15:00:50 -0800766 else:
767 result = build.run()
James E. Blaire1767bc2016-08-02 10:00:27 -0700768 return result
James E. Blairf5dbd002015-12-23 15:26:17 -0800769
770
Clark Boylanb640e052014-04-03 16:41:46 -0700771class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -0700772 """A Gearman server for use in tests.
773
774 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
775 added to the queue but will not be distributed to workers
776 until released. This attribute may be changed at any time and
777 will take effect for subsequently enqueued jobs, but
778 previously held jobs will still need to be explicitly
779 released.
780
781 """
782
Clark Boylanb640e052014-04-03 16:41:46 -0700783 def __init__(self):
784 self.hold_jobs_in_queue = False
785 super(FakeGearmanServer, self).__init__(0)
786
787 def getJobForConnection(self, connection, peek=False):
788 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
789 for job in queue:
790 if not hasattr(job, 'waiting'):
Paul Belanger6ab6af72016-11-06 11:32:59 -0500791 if job.name.startswith('launcher:launch'):
Clark Boylanb640e052014-04-03 16:41:46 -0700792 job.waiting = self.hold_jobs_in_queue
793 else:
794 job.waiting = False
795 if job.waiting:
796 continue
797 if job.name in connection.functions:
798 if not peek:
799 queue.remove(job)
800 connection.related_jobs[job.handle] = job
801 job.worker_connection = connection
802 job.running = True
803 return job
804 return None
805
806 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700807 """Release a held job.
808
809 :arg str regex: A regular expression which, if supplied, will
810 cause only jobs with matching names to be released. If
811 not supplied, all jobs will be released.
812 """
Clark Boylanb640e052014-04-03 16:41:46 -0700813 released = False
814 qlen = (len(self.high_queue) + len(self.normal_queue) +
815 len(self.low_queue))
816 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
817 for job in self.getQueue():
Paul Belanger6ab6af72016-11-06 11:32:59 -0500818 if job.name != 'launcher:launch':
Clark Boylanb640e052014-04-03 16:41:46 -0700819 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -0500820 parameters = json.loads(job.arguments)
821 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -0700822 self.log.debug("releasing queued job %s" %
823 job.unique)
824 job.waiting = False
825 released = True
826 else:
827 self.log.debug("not releasing queued job %s" %
828 job.unique)
829 if released:
830 self.wakeConnections()
831 qlen = (len(self.high_queue) + len(self.normal_queue) +
832 len(self.low_queue))
833 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
834
835
836class FakeSMTP(object):
837 log = logging.getLogger('zuul.FakeSMTP')
838
839 def __init__(self, messages, server, port):
840 self.server = server
841 self.port = port
842 self.messages = messages
843
844 def sendmail(self, from_email, to_email, msg):
845 self.log.info("Sending email from %s, to %s, with msg %s" % (
846 from_email, to_email, msg))
847
848 headers = msg.split('\n\n', 1)[0]
849 body = msg.split('\n\n', 1)[1]
850
851 self.messages.append(dict(
852 from_email=from_email,
853 to_email=to_email,
854 msg=msg,
855 headers=headers,
856 body=body,
857 ))
858
859 return True
860
861 def quit(self):
862 return True
863
864
865class FakeSwiftClientConnection(swiftclient.client.Connection):
866 def post_account(self, headers):
867 # Do nothing
868 pass
869
870 def get_auth(self):
871 # Returns endpoint and (unused) auth token
872 endpoint = os.path.join('https://storage.example.org', 'V1',
873 'AUTH_account')
874 return endpoint, ''
875
876
James E. Blairdce6cea2016-12-20 16:45:32 -0800877class FakeNodepool(object):
878 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -0800879 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -0800880
881 log = logging.getLogger("zuul.test.FakeNodepool")
882
883 def __init__(self, host, port, chroot):
884 self.client = kazoo.client.KazooClient(
885 hosts='%s:%s%s' % (host, port, chroot))
886 self.client.start()
887 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -0800888 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -0800889 self.thread = threading.Thread(target=self.run)
890 self.thread.daemon = True
891 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -0800892 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -0800893
894 def stop(self):
895 self._running = False
896 self.thread.join()
897 self.client.stop()
898 self.client.close()
899
900 def run(self):
901 while self._running:
902 self._run()
903 time.sleep(0.1)
904
905 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -0800906 if self.paused:
907 return
James E. Blairdce6cea2016-12-20 16:45:32 -0800908 for req in self.getNodeRequests():
909 self.fulfillRequest(req)
910
911 def getNodeRequests(self):
912 try:
913 reqids = self.client.get_children(self.REQUEST_ROOT)
914 except kazoo.exceptions.NoNodeError:
915 return []
916 reqs = []
917 for oid in sorted(reqids):
918 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -0800919 try:
920 data, stat = self.client.get(path)
921 data = json.loads(data)
922 data['_oid'] = oid
923 reqs.append(data)
924 except kazoo.exceptions.NoNodeError:
925 pass
James E. Blairdce6cea2016-12-20 16:45:32 -0800926 return reqs
927
James E. Blaire18d4602017-01-05 11:17:28 -0800928 def getNodes(self):
929 try:
930 nodeids = self.client.get_children(self.NODE_ROOT)
931 except kazoo.exceptions.NoNodeError:
932 return []
933 nodes = []
934 for oid in sorted(nodeids):
935 path = self.NODE_ROOT + '/' + oid
936 data, stat = self.client.get(path)
937 data = json.loads(data)
938 data['_oid'] = oid
939 try:
940 lockfiles = self.client.get_children(path + '/lock')
941 except kazoo.exceptions.NoNodeError:
942 lockfiles = []
943 if lockfiles:
944 data['_lock'] = True
945 else:
946 data['_lock'] = False
947 nodes.append(data)
948 return nodes
949
James E. Blaira38c28e2017-01-04 10:33:20 -0800950 def makeNode(self, request_id, node_type):
951 now = time.time()
952 path = '/nodepool/nodes/'
953 data = dict(type=node_type,
954 provider='test-provider',
955 region='test-region',
956 az=None,
957 public_ipv4='127.0.0.1',
958 private_ipv4=None,
959 public_ipv6=None,
960 allocated_to=request_id,
961 state='ready',
962 state_time=now,
963 created_time=now,
964 updated_time=now,
965 image_id=None,
966 launcher='fake-nodepool')
967 data = json.dumps(data)
968 path = self.client.create(path, data,
969 makepath=True,
970 sequence=True)
971 nodeid = path.split("/")[-1]
972 return nodeid
973
James E. Blair6ab79e02017-01-06 10:10:17 -0800974 def addFailRequest(self, request):
975 self.fail_requests.add(request['_oid'])
976
James E. Blairdce6cea2016-12-20 16:45:32 -0800977 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -0800978 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -0800979 return
980 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -0800981 oid = request['_oid']
982 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -0800983
James E. Blair6ab79e02017-01-06 10:10:17 -0800984 if oid in self.fail_requests:
985 request['state'] = 'failed'
986 else:
987 request['state'] = 'fulfilled'
988 nodes = []
989 for node in request['node_types']:
990 nodeid = self.makeNode(oid, node)
991 nodes.append(nodeid)
992 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -0800993
James E. Blaira38c28e2017-01-04 10:33:20 -0800994 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -0800995 path = self.REQUEST_ROOT + '/' + oid
996 data = json.dumps(request)
997 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
998 self.client.set(path, data)
999
1000
James E. Blair498059b2016-12-20 13:50:13 -08001001class ChrootedKazooFixture(fixtures.Fixture):
1002 def __init__(self):
1003 super(ChrootedKazooFixture, self).__init__()
1004
1005 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1006 if ':' in zk_host:
1007 host, port = zk_host.split(':')
1008 else:
1009 host = zk_host
1010 port = None
1011
1012 self.zookeeper_host = host
1013
1014 if not port:
1015 self.zookeeper_port = 2181
1016 else:
1017 self.zookeeper_port = int(port)
1018
1019 def _setUp(self):
1020 # Make sure the test chroot paths do not conflict
1021 random_bits = ''.join(random.choice(string.ascii_lowercase +
1022 string.ascii_uppercase)
1023 for x in range(8))
1024
1025 rand_test_path = '%s_%s' % (random_bits, os.getpid())
1026 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1027
1028 # Ensure the chroot path exists and clean up any pre-existing znodes.
1029 _tmp_client = kazoo.client.KazooClient(
1030 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1031 _tmp_client.start()
1032
1033 if _tmp_client.exists(self.zookeeper_chroot):
1034 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1035
1036 _tmp_client.ensure_path(self.zookeeper_chroot)
1037 _tmp_client.stop()
1038 _tmp_client.close()
1039
1040 self.addCleanup(self._cleanup)
1041
1042 def _cleanup(self):
1043 '''Remove the chroot path.'''
1044 # Need a non-chroot'ed client to remove the chroot path
1045 _tmp_client = kazoo.client.KazooClient(
1046 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1047 _tmp_client.start()
1048 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1049 _tmp_client.stop()
1050
1051
Maru Newby3fe5f852015-01-13 04:22:14 +00001052class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001053 log = logging.getLogger("zuul.test")
1054
James E. Blair1c236df2017-02-01 14:07:24 -08001055 def attachLogs(self, *args):
1056 def reader():
1057 self._log_stream.seek(0)
1058 while True:
1059 x = self._log_stream.read(4096)
1060 if not x:
1061 break
1062 yield x.encode('utf8')
1063 content = testtools.content.content_from_reader(
1064 reader,
1065 testtools.content_type.UTF8_TEXT,
1066 False)
1067 self.addDetail('logging', content)
1068
Clark Boylanb640e052014-04-03 16:41:46 -07001069 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001070 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001071 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1072 try:
1073 test_timeout = int(test_timeout)
1074 except ValueError:
1075 # If timeout value is invalid do not set a timeout.
1076 test_timeout = 0
1077 if test_timeout > 0:
1078 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1079
1080 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1081 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1082 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1083 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1084 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1085 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1086 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1087 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1088 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1089 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001090 self._log_stream = StringIO()
1091 self.addOnException(self.attachLogs)
1092 else:
1093 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001094
James E. Blair1c236df2017-02-01 14:07:24 -08001095 handler = logging.StreamHandler(self._log_stream)
1096 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1097 '%(levelname)-8s %(message)s')
1098 handler.setFormatter(formatter)
1099
1100 logger = logging.getLogger()
1101 logger.setLevel(logging.DEBUG)
1102 logger.addHandler(handler)
1103
1104 # NOTE(notmorgan): Extract logging overrides for specific
1105 # libraries from the OS_LOG_DEFAULTS env and create loggers
1106 # for each. This is used to limit the output during test runs
1107 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001108 log_defaults_from_env = os.environ.get(
1109 'OS_LOG_DEFAULTS',
James E. Blair1c236df2017-02-01 14:07:24 -08001110 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001111
James E. Blairdce6cea2016-12-20 16:45:32 -08001112 if log_defaults_from_env:
1113 for default in log_defaults_from_env.split(','):
1114 try:
1115 name, level_str = default.split('=', 1)
1116 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001117 logger = logging.getLogger(name)
1118 logger.setLevel(level)
1119 logger.addHandler(handler)
1120 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001121 except ValueError:
1122 # NOTE(notmorgan): Invalid format of the log default,
1123 # skip and don't try and apply a logger for the
1124 # specified module
1125 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001126
Maru Newby3fe5f852015-01-13 04:22:14 +00001127
1128class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001129 """A test case with a functioning Zuul.
1130
1131 The following class variables are used during test setup and can
1132 be overidden by subclasses but are effectively read-only once a
1133 test method starts running:
1134
1135 :cvar str config_file: This points to the main zuul config file
1136 within the fixtures directory. Subclasses may override this
1137 to obtain a different behavior.
1138
1139 :cvar str tenant_config_file: This is the tenant config file
1140 (which specifies from what git repos the configuration should
1141 be loaded). It defaults to the value specified in
1142 `config_file` but can be overidden by subclasses to obtain a
1143 different tenant/project layout while using the standard main
1144 configuration.
1145
1146 The following are instance variables that are useful within test
1147 methods:
1148
1149 :ivar FakeGerritConnection fake_<connection>:
1150 A :py:class:`~tests.base.FakeGerritConnection` will be
1151 instantiated for each connection present in the config file
1152 and stored here. For instance, `fake_gerrit` will hold the
1153 FakeGerritConnection object for a connection named `gerrit`.
1154
1155 :ivar FakeGearmanServer gearman_server: An instance of
1156 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1157 server that all of the Zuul components in this test use to
1158 communicate with each other.
1159
1160 :ivar RecordingLaunchServer launch_server: An instance of
1161 :py:class:`~tests.base.RecordingLaunchServer` which is the
1162 Ansible launch server used to run jobs for this test.
1163
1164 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1165 representing currently running builds. They are appended to
1166 the list in the order they are launched, and removed from this
1167 list upon completion.
1168
1169 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1170 objects representing completed builds. They are appended to
1171 the list in the order they complete.
1172
1173 """
1174
James E. Blair83005782015-12-11 14:46:03 -08001175 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001176 run_ansible = False
James E. Blair3f876d52016-07-22 13:07:14 -07001177
1178 def _startMerger(self):
1179 self.merge_server = zuul.merger.server.MergeServer(self.config,
1180 self.connections)
1181 self.merge_server.start()
1182
Maru Newby3fe5f852015-01-13 04:22:14 +00001183 def setUp(self):
1184 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001185
1186 self.setupZK()
1187
James E. Blair97d902e2014-08-21 13:25:56 -07001188 if USE_TEMPDIR:
1189 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001190 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1191 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001192 else:
1193 tmp_root = os.environ.get("ZUUL_TEST_ROOT")
Clark Boylanb640e052014-04-03 16:41:46 -07001194 self.test_root = os.path.join(tmp_root, "zuul-test")
1195 self.upstream_root = os.path.join(self.test_root, "upstream")
James E. Blair8c1be532017-02-07 14:04:12 -08001196 self.merger_git_root = os.path.join(self.test_root, "merger-git")
1197 self.launcher_git_root = os.path.join(self.test_root, "launcher-git")
James E. Blairce8a2132016-05-19 15:21:52 -07001198 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001199
1200 if os.path.exists(self.test_root):
1201 shutil.rmtree(self.test_root)
1202 os.makedirs(self.test_root)
1203 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001204 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001205
1206 # Make per test copy of Configuration.
1207 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001208 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001209 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001210 self.config.get('zuul', 'tenant_config')))
James E. Blair8c1be532017-02-07 14:04:12 -08001211 self.config.set('merger', 'git_dir', self.merger_git_root)
1212 self.config.set('launcher', 'git_dir', self.launcher_git_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001213 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001214
1215 # For each project in config:
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001216 # TODOv3(jeblair): remove these and replace with new git
1217 # filesystem fixtures
Clark Boylanb640e052014-04-03 16:41:46 -07001218 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -07001219 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -07001220 self.init_repo("org/project5")
1221 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -07001222 self.init_repo("org/one-job-project")
1223 self.init_repo("org/nonvoting-project")
1224 self.init_repo("org/templated-project")
1225 self.init_repo("org/layered-project")
1226 self.init_repo("org/node-project")
1227 self.init_repo("org/conflict-project")
1228 self.init_repo("org/noop-project")
1229 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00001230 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -07001231
1232 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001233 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1234 # see: https://github.com/jsocol/pystatsd/issues/61
1235 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001236 os.environ['STATSD_PORT'] = str(self.statsd.port)
1237 self.statsd.start()
1238 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001239 reload_module(statsd)
1240 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001241
1242 self.gearman_server = FakeGearmanServer()
1243
1244 self.config.set('gearman', 'port', str(self.gearman_server.port))
James E. Blaire47eb772017-02-02 17:19:40 -08001245 self.log.info("Gearman server on port %s" %
1246 (self.gearman_server.port,))
Clark Boylanb640e052014-04-03 16:41:46 -07001247
James E. Blaire511d2f2016-12-08 15:22:26 -08001248 gerritsource.GerritSource.replication_timeout = 1.5
1249 gerritsource.GerritSource.replication_retry_interval = 0.5
1250 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001251
Joshua Hesketh352264b2015-08-11 23:42:08 +10001252 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001253
1254 self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
1255 FakeSwiftClientConnection))
James E. Blaire511d2f2016-12-08 15:22:26 -08001256
Clark Boylanb640e052014-04-03 16:41:46 -07001257 self.swift = zuul.lib.swift.Swift(self.config)
1258
Jan Hruban6b71aff2015-10-22 16:58:08 +02001259 self.event_queues = [
1260 self.sched.result_event_queue,
1261 self.sched.trigger_event_queue
1262 ]
1263
James E. Blairfef78942016-03-11 16:28:56 -08001264 self.configure_connections()
Joshua Hesketh352264b2015-08-11 23:42:08 +10001265 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001266
Clark Boylanb640e052014-04-03 16:41:46 -07001267 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001268 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001269 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001270 return FakeURLOpener(self.upstream_root, *args, **kw)
1271
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001272 old_urlopen = urllib.request.urlopen
1273 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001274
James E. Blair3f876d52016-07-22 13:07:14 -07001275 self._startMerger()
James E. Blair3f876d52016-07-22 13:07:14 -07001276
James E. Blaire1767bc2016-08-02 10:00:27 -07001277 self.launch_server = RecordingLaunchServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001278 self.config, self.connections,
James E. Blair854f8892017-02-02 11:25:39 -08001279 jobdir_root=self.test_root,
James E. Blaira92cbc82017-01-23 14:56:49 -08001280 _run_ansible=self.run_ansible,
1281 _test_root=self.test_root)
James E. Blaire1767bc2016-08-02 10:00:27 -07001282 self.launch_server.start()
1283 self.history = self.launch_server.build_history
1284 self.builds = self.launch_server.running_builds
1285
1286 self.launch_client = zuul.launcher.client.LaunchClient(
James E. Blair82938472016-01-11 14:38:13 -08001287 self.config, self.sched, self.swift)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001288 self.merge_client = zuul.merger.client.MergeClient(
1289 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001290 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001291 self.zk = zuul.zk.ZooKeeper()
1292 self.zk.connect([self.zk_config])
1293
1294 self.fake_nodepool = FakeNodepool(self.zk_config.host,
1295 self.zk_config.port,
1296 self.zk_config.chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001297
James E. Blaire1767bc2016-08-02 10:00:27 -07001298 self.sched.setLauncher(self.launch_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001299 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001300 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001301 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001302
Paul Belanger88ef0ea2015-12-23 11:57:02 -05001303 self.webapp = zuul.webapp.WebApp(
1304 self.sched, port=0, listen_address='127.0.0.1')
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001305 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001306
1307 self.sched.start()
1308 self.sched.reconfigure(self.config)
1309 self.sched.resume()
1310 self.webapp.start()
1311 self.rpc.start()
James E. Blaire1767bc2016-08-02 10:00:27 -07001312 self.launch_client.gearman.waitForServer()
Clark Boylanb640e052014-04-03 16:41:46 -07001313
Clark Boylanb640e052014-04-03 16:41:46 -07001314 self.addCleanup(self.shutdown)
1315
James E. Blaire18d4602017-01-05 11:17:28 -08001316 def tearDown(self):
1317 super(ZuulTestCase, self).tearDown()
1318 self.assertFinalState()
1319
James E. Blairfef78942016-03-11 16:28:56 -08001320 def configure_connections(self):
James E. Blaire511d2f2016-12-08 15:22:26 -08001321 # Set up gerrit related fakes
1322 # Set a changes database so multiple FakeGerrit's can report back to
1323 # a virtual canonical database given by the configured hostname
1324 self.gerrit_changes_dbs = {}
1325
1326 def getGerritConnection(driver, name, config):
1327 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1328 con = FakeGerritConnection(driver, name, config,
1329 changes_db=db,
1330 upstream_root=self.upstream_root)
1331 self.event_queues.append(con.event_queue)
1332 setattr(self, 'fake_' + name, con)
1333 return con
1334
1335 self.useFixture(fixtures.MonkeyPatch(
1336 'zuul.driver.gerrit.GerritDriver.getConnection',
1337 getGerritConnection))
1338
1339 # Set up smtp related fakes
Joshua Hesketh352264b2015-08-11 23:42:08 +10001340 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001341
Joshua Hesketh352264b2015-08-11 23:42:08 +10001342 def FakeSMTPFactory(*args, **kw):
1343 args = [self.smtp_messages] + list(args)
1344 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001345
Joshua Hesketh352264b2015-08-11 23:42:08 +10001346 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001347
James E. Blaire511d2f2016-12-08 15:22:26 -08001348 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001349 self.connections = zuul.lib.connections.ConnectionRegistry()
James E. Blaire511d2f2016-12-08 15:22:26 -08001350 self.connections.configure(self.config)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001351
James E. Blair83005782015-12-11 14:46:03 -08001352 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001353 # This creates the per-test configuration object. It can be
1354 # overriden by subclasses, but should not need to be since it
1355 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001356 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001357 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair2a629ec2015-12-22 15:32:02 -08001358 if hasattr(self, 'tenant_config_file'):
1359 self.config.set('zuul', 'tenant_config', self.tenant_config_file)
James E. Blair96c6bf82016-01-15 16:20:40 -08001360 git_path = os.path.join(
1361 os.path.dirname(
1362 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1363 'git')
1364 if os.path.exists(git_path):
1365 for reponame in os.listdir(git_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001366 project = reponame.replace('_', '/')
1367 self.copyDirToRepo(project,
James E. Blair96c6bf82016-01-15 16:20:40 -08001368 os.path.join(git_path, reponame))
1369
James E. Blair498059b2016-12-20 13:50:13 -08001370 def setupZK(self):
1371 self.zk_chroot_fixture = self.useFixture(ChrootedKazooFixture())
James E. Blairdce6cea2016-12-20 16:45:32 -08001372 self.zk_config = zuul.zk.ZooKeeperConnectionConfig(
1373 self.zk_chroot_fixture.zookeeper_host,
1374 self.zk_chroot_fixture.zookeeper_port,
1375 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08001376
James E. Blair96c6bf82016-01-15 16:20:40 -08001377 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001378 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08001379
1380 files = {}
1381 for (dirpath, dirnames, filenames) in os.walk(source_path):
1382 for filename in filenames:
1383 test_tree_filepath = os.path.join(dirpath, filename)
1384 common_path = os.path.commonprefix([test_tree_filepath,
1385 source_path])
1386 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1387 with open(test_tree_filepath, 'r') as f:
1388 content = f.read()
1389 files[relative_filepath] = content
1390 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001391 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08001392
James E. Blaire18d4602017-01-05 11:17:28 -08001393 def assertNodepoolState(self):
1394 # Make sure that there are no pending requests
1395
1396 requests = self.fake_nodepool.getNodeRequests()
1397 self.assertEqual(len(requests), 0)
1398
1399 nodes = self.fake_nodepool.getNodes()
1400 for node in nodes:
1401 self.assertFalse(node['_lock'], "Node %s is locked" %
1402 (node['_oid'],))
1403
Clark Boylanb640e052014-04-03 16:41:46 -07001404 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001405 # Make sure that git.Repo objects have been garbage collected.
1406 repos = []
1407 gc.collect()
1408 for obj in gc.get_objects():
1409 if isinstance(obj, git.Repo):
1410 repos.append(obj)
1411 self.assertEqual(len(repos), 0)
1412 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08001413 self.assertNodepoolState()
James E. Blair83005782015-12-11 14:46:03 -08001414 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001415 for tenant in self.sched.abide.tenants.values():
1416 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001417 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001418 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001419
1420 def shutdown(self):
1421 self.log.debug("Shutting down after tests")
James E. Blaire1767bc2016-08-02 10:00:27 -07001422 self.launch_client.stop()
James E. Blair3f876d52016-07-22 13:07:14 -07001423 self.merge_server.stop()
1424 self.merge_server.join()
Clark Boylanb640e052014-04-03 16:41:46 -07001425 self.merge_client.stop()
James E. Blaire1767bc2016-08-02 10:00:27 -07001426 self.launch_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001427 self.sched.stop()
1428 self.sched.join()
1429 self.statsd.stop()
1430 self.statsd.join()
1431 self.webapp.stop()
1432 self.webapp.join()
1433 self.rpc.stop()
1434 self.rpc.join()
1435 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08001436 self.fake_nodepool.stop()
1437 self.zk.disconnect()
Clark Boylanb640e052014-04-03 16:41:46 -07001438 threads = threading.enumerate()
1439 if len(threads) > 1:
1440 self.log.error("More than one thread is running: %s" % threads)
James E. Blair6ac368c2016-12-22 18:07:20 -08001441 self.printHistory()
Clark Boylanb640e052014-04-03 16:41:46 -07001442
1443 def init_repo(self, project):
1444 parts = project.split('/')
1445 path = os.path.join(self.upstream_root, *parts[:-1])
1446 if not os.path.exists(path):
1447 os.makedirs(path)
1448 path = os.path.join(self.upstream_root, project)
1449 repo = git.Repo.init(path)
1450
Morgan Fainberg78c301a2016-07-14 13:47:01 -07001451 with repo.config_writer() as config_writer:
1452 config_writer.set_value('user', 'email', 'user@example.com')
1453 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07001454
Clark Boylanb640e052014-04-03 16:41:46 -07001455 repo.index.commit('initial commit')
1456 master = repo.create_head('master')
Clark Boylanb640e052014-04-03 16:41:46 -07001457
James E. Blair97d902e2014-08-21 13:25:56 -07001458 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001459 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001460 repo.git.clean('-x', '-f', '-d')
1461
James E. Blair97d902e2014-08-21 13:25:56 -07001462 def create_branch(self, project, branch):
1463 path = os.path.join(self.upstream_root, project)
1464 repo = git.Repo.init(path)
1465 fn = os.path.join(path, 'README')
1466
1467 branch_head = repo.create_head(branch)
1468 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001469 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001470 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001471 f.close()
1472 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001473 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001474
James E. Blair97d902e2014-08-21 13:25:56 -07001475 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001476 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001477 repo.git.clean('-x', '-f', '-d')
1478
Sachi King9f16d522016-03-16 12:20:45 +11001479 def create_commit(self, project):
1480 path = os.path.join(self.upstream_root, project)
1481 repo = git.Repo(path)
1482 repo.head.reference = repo.heads['master']
1483 file_name = os.path.join(path, 'README')
1484 with open(file_name, 'a') as f:
1485 f.write('creating fake commit\n')
1486 repo.index.add([file_name])
1487 commit = repo.index.commit('Creating a fake commit')
1488 return commit.hexsha
1489
James E. Blairb8c16472015-05-05 14:55:26 -07001490 def orderedRelease(self):
1491 # Run one build at a time to ensure non-race order:
1492 while len(self.builds):
1493 self.release(self.builds[0])
1494 self.waitUntilSettled()
1495
Clark Boylanb640e052014-04-03 16:41:46 -07001496 def release(self, job):
1497 if isinstance(job, FakeBuild):
1498 job.release()
1499 else:
1500 job.waiting = False
1501 self.log.debug("Queued job %s released" % job.unique)
1502 self.gearman_server.wakeConnections()
1503
1504 def getParameter(self, job, name):
1505 if isinstance(job, FakeBuild):
1506 return job.parameters[name]
1507 else:
1508 parameters = json.loads(job.arguments)
1509 return parameters[name]
1510
Clark Boylanb640e052014-04-03 16:41:46 -07001511 def haveAllBuildsReported(self):
1512 # See if Zuul is waiting on a meta job to complete
James E. Blaire1767bc2016-08-02 10:00:27 -07001513 if self.launch_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07001514 return False
1515 # Find out if every build that the worker has completed has been
1516 # reported back to Zuul. If it hasn't then that means a Gearman
1517 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07001518 for build in self.history:
James E. Blaire1767bc2016-08-02 10:00:27 -07001519 zbuild = self.launch_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07001520 if not zbuild:
1521 # It has already been reported
1522 continue
1523 # It hasn't been reported yet.
1524 return False
1525 # Make sure that none of the worker connections are in GRAB_WAIT
James E. Blaire1767bc2016-08-02 10:00:27 -07001526 for connection in self.launch_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001527 if connection.state == 'GRAB_WAIT':
1528 return False
1529 return True
1530
1531 def areAllBuildsWaiting(self):
James E. Blaire1767bc2016-08-02 10:00:27 -07001532 builds = self.launch_client.builds.values()
Clark Boylanb640e052014-04-03 16:41:46 -07001533 for build in builds:
1534 client_job = None
James E. Blaire1767bc2016-08-02 10:00:27 -07001535 for conn in self.launch_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001536 for j in conn.related_jobs.values():
1537 if j.unique == build.uuid:
1538 client_job = j
1539 break
1540 if not client_job:
1541 self.log.debug("%s is not known to the gearman client" %
1542 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001543 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001544 if not client_job.handle:
1545 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001546 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001547 server_job = self.gearman_server.jobs.get(client_job.handle)
1548 if not server_job:
1549 self.log.debug("%s is not known to the gearman server" %
1550 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001551 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001552 if not hasattr(server_job, 'waiting'):
1553 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001554 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001555 if server_job.waiting:
1556 continue
James E. Blair17302972016-08-10 16:11:42 -07001557 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08001558 self.log.debug("%s has not reported start" % build)
1559 return False
James E. Blairab7132b2016-08-05 12:36:22 -07001560 worker_build = self.launch_server.job_builds.get(server_job.unique)
James E. Blair962220f2016-08-03 11:22:38 -07001561 if worker_build:
1562 if worker_build.isWaiting():
1563 continue
1564 else:
1565 self.log.debug("%s is running" % worker_build)
1566 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001567 else:
James E. Blair962220f2016-08-03 11:22:38 -07001568 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001569 return False
1570 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001571
James E. Blairdce6cea2016-12-20 16:45:32 -08001572 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001573 if self.fake_nodepool.paused:
1574 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08001575 if self.sched.nodepool.requests:
1576 return False
1577 return True
1578
Jan Hruban6b71aff2015-10-22 16:58:08 +02001579 def eventQueuesEmpty(self):
1580 for queue in self.event_queues:
1581 yield queue.empty()
1582
1583 def eventQueuesJoin(self):
1584 for queue in self.event_queues:
1585 queue.join()
1586
Clark Boylanb640e052014-04-03 16:41:46 -07001587 def waitUntilSettled(self):
1588 self.log.debug("Waiting until settled...")
1589 start = time.time()
1590 while True:
James E. Blair71932482017-02-02 11:29:07 -08001591 if time.time() - start > 20:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001592 self.log.error("Timeout waiting for Zuul to settle")
1593 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07001594 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001595 self.log.error(" %s: %s" % (queue, queue.empty()))
1596 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07001597 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001598 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07001599 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001600 self.log.error("All requests completed: %s" %
1601 (self.areAllNodeRequestsComplete(),))
1602 self.log.error("Merge client jobs: %s" %
1603 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07001604 raise Exception("Timeout waiting for Zuul to settle")
1605 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07001606
James E. Blaire1767bc2016-08-02 10:00:27 -07001607 self.launch_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07001608 # have all build states propogated to zuul?
1609 if self.haveAllBuildsReported():
1610 # Join ensures that the queue is empty _and_ events have been
1611 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001612 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001613 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001614 if (not self.merge_client.jobs and
Clark Boylanb640e052014-04-03 16:41:46 -07001615 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08001616 self.areAllBuildsWaiting() and
James E. Blair36c611a2017-02-06 15:59:43 -08001617 self.areAllNodeRequestsComplete() and
1618 all(self.eventQueuesEmpty())):
1619 # The queue empty check is placed at the end to
1620 # ensure that if a component adds an event between
1621 # when locked the run handler and checked that the
1622 # components were stable, we don't erroneously
1623 # report that we are settled.
Clark Boylanb640e052014-04-03 16:41:46 -07001624 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001625 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001626 self.log.debug("...settled.")
1627 return
1628 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001629 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001630 self.sched.wake_event.wait(0.1)
1631
1632 def countJobResults(self, jobs, result):
1633 jobs = filter(lambda x: x.result == result, jobs)
1634 return len(jobs)
1635
James E. Blair96c6bf82016-01-15 16:20:40 -08001636 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07001637 for job in self.history:
1638 if (job.name == name and
1639 (project is None or
1640 job.parameters['ZUUL_PROJECT'] == project)):
1641 return job
Clark Boylanb640e052014-04-03 16:41:46 -07001642 raise Exception("Unable to find job %s in history" % name)
1643
1644 def assertEmptyQueues(self):
1645 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001646 for tenant in self.sched.abide.tenants.values():
1647 for pipeline in tenant.layout.pipelines.values():
1648 for queue in pipeline.queues:
1649 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10001650 print('pipeline %s queue %s contents %s' % (
1651 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08001652 self.assertEqual(len(queue.queue), 0,
1653 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001654
1655 def assertReportedStat(self, key, value=None, kind=None):
1656 start = time.time()
1657 while time.time() < (start + 5):
1658 for stat in self.statsd.stats:
Clark Boylanb640e052014-04-03 16:41:46 -07001659 k, v = stat.split(':')
1660 if key == k:
1661 if value is None and kind is None:
1662 return
1663 elif value:
1664 if value == v:
1665 return
1666 elif kind:
1667 if v.endswith('|' + kind):
1668 return
1669 time.sleep(0.1)
1670
Clark Boylanb640e052014-04-03 16:41:46 -07001671 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001672
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001673 def assertBuilds(self, builds):
1674 """Assert that the running builds are as described.
1675
1676 The list of running builds is examined and must match exactly
1677 the list of builds described by the input.
1678
1679 :arg list builds: A list of dictionaries. Each item in the
1680 list must match the corresponding build in the build
1681 history, and each element of the dictionary must match the
1682 corresponding attribute of the build.
1683
1684 """
James E. Blair3158e282016-08-19 09:34:11 -07001685 try:
1686 self.assertEqual(len(self.builds), len(builds))
1687 for i, d in enumerate(builds):
1688 for k, v in d.items():
1689 self.assertEqual(
1690 getattr(self.builds[i], k), v,
1691 "Element %i in builds does not match" % (i,))
1692 except Exception:
1693 for build in self.builds:
1694 self.log.error("Running build: %s" % build)
1695 else:
1696 self.log.error("No running builds")
1697 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001698
James E. Blairb536ecc2016-08-31 10:11:42 -07001699 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001700 """Assert that the completed builds are as described.
1701
1702 The list of completed builds is examined and must match
1703 exactly the list of builds described by the input.
1704
1705 :arg list history: A list of dictionaries. Each item in the
1706 list must match the corresponding build in the build
1707 history, and each element of the dictionary must match the
1708 corresponding attribute of the build.
1709
James E. Blairb536ecc2016-08-31 10:11:42 -07001710 :arg bool ordered: If true, the history must match the order
1711 supplied, if false, the builds are permitted to have
1712 arrived in any order.
1713
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001714 """
James E. Blairb536ecc2016-08-31 10:11:42 -07001715 def matches(history_item, item):
1716 for k, v in item.items():
1717 if getattr(history_item, k) != v:
1718 return False
1719 return True
James E. Blair3158e282016-08-19 09:34:11 -07001720 try:
1721 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07001722 if ordered:
1723 for i, d in enumerate(history):
1724 if not matches(self.history[i], d):
1725 raise Exception(
1726 "Element %i in history does not match" % (i,))
1727 else:
1728 unseen = self.history[:]
1729 for i, d in enumerate(history):
1730 found = False
1731 for unseen_item in unseen:
1732 if matches(unseen_item, d):
1733 found = True
1734 unseen.remove(unseen_item)
1735 break
1736 if not found:
1737 raise Exception("No match found for element %i "
1738 "in history" % (i,))
1739 if unseen:
1740 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07001741 except Exception:
1742 for build in self.history:
1743 self.log.error("Completed build: %s" % build)
1744 else:
1745 self.log.error("No completed builds")
1746 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001747
James E. Blair6ac368c2016-12-22 18:07:20 -08001748 def printHistory(self):
1749 """Log the build history.
1750
1751 This can be useful during tests to summarize what jobs have
1752 completed.
1753
1754 """
1755 self.log.debug("Build history:")
1756 for build in self.history:
1757 self.log.debug(build)
1758
James E. Blair59fdbac2015-12-07 17:08:06 -08001759 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001760 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1761
1762 def updateConfigLayout(self, path):
1763 root = os.path.join(self.test_root, "config")
Clint Byrumead6c562017-02-01 16:34:04 -08001764 if not os.path.exists(root):
1765 os.makedirs(root)
James E. Blairf84026c2015-12-08 16:11:46 -08001766 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1767 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05001768- tenant:
1769 name: openstack
1770 source:
1771 gerrit:
1772 config-repos:
1773 - %s
1774 """ % path)
James E. Blairf84026c2015-12-08 16:11:46 -08001775 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05001776 self.config.set('zuul', 'tenant_config',
1777 os.path.join(FIXTURE_DIR, f.name))
James E. Blair14abdf42015-12-09 16:11:53 -08001778
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001779 def addCommitToRepo(self, project, message, files,
1780 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08001781 path = os.path.join(self.upstream_root, project)
1782 repo = git.Repo(path)
1783 repo.head.reference = branch
1784 zuul.merger.merger.reset_repo_to_head(repo)
1785 for fn, content in files.items():
1786 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08001787 try:
1788 os.makedirs(os.path.dirname(fn))
1789 except OSError:
1790 pass
James E. Blair14abdf42015-12-09 16:11:53 -08001791 with open(fn, 'w') as f:
1792 f.write(content)
1793 repo.index.add([fn])
1794 commit = repo.index.commit(message)
1795 repo.heads[branch].commit = commit
1796 repo.head.reference = branch
1797 repo.git.clean('-x', '-f', '-d')
1798 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001799 if tag:
1800 repo.create_tag(tag)
James E. Blair3f876d52016-07-22 13:07:14 -07001801
James E. Blair7fc8daa2016-08-08 15:37:15 -07001802 def addEvent(self, connection, event):
1803 """Inject a Fake (Gerrit) event.
1804
1805 This method accepts a JSON-encoded event and simulates Zuul
1806 having received it from Gerrit. It could (and should)
1807 eventually apply to any connection type, but is currently only
1808 used with Gerrit connections. The name of the connection is
1809 used to look up the corresponding server, and the event is
1810 simulated as having been received by all Zuul connections
1811 attached to that server. So if two Gerrit connections in Zuul
1812 are connected to the same Gerrit server, and you invoke this
1813 method specifying the name of one of them, the event will be
1814 received by both.
1815
1816 .. note::
1817
1818 "self.fake_gerrit.addEvent" calls should be migrated to
1819 this method.
1820
1821 :arg str connection: The name of the connection corresponding
1822 to the gerrit server.
1823 :arg str event: The JSON-encoded event.
1824
1825 """
1826 specified_conn = self.connections.connections[connection]
1827 for conn in self.connections.connections.values():
1828 if (isinstance(conn, specified_conn.__class__) and
1829 specified_conn.server == conn.server):
1830 conn.addEvent(event)
1831
James E. Blair3f876d52016-07-22 13:07:14 -07001832
1833class AnsibleZuulTestCase(ZuulTestCase):
1834 """ZuulTestCase but with an actual ansible launcher running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07001835 run_ansible = True