blob: d5fd7b8816ddba23ed7aad6b065d6d41d68452f0 [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070019import gc
20import hashlib
21import json
22import logging
23import os
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070025from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070026import random
27import re
28import select
29import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030030from six.moves import reload_module
James E. Blair1c236df2017-02-01 14:07:24 -080031from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070032import socket
33import string
34import subprocess
35import swiftclient
James E. Blair1c236df2017-02-01 14:07:24 -080036import sys
James E. Blairf84026c2015-12-08 16:11:46 -080037import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070038import threading
39import time
Clark Boylanb640e052014-04-03 16:41:46 -070040
41import git
42import gear
43import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080044import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080045import kazoo.exceptions
Clark Boylanb640e052014-04-03 16:41:46 -070046import statsd
47import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080048import testtools.content
49import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080050from git.exc import NoSuchPathError
Clark Boylanb640e052014-04-03 16:41:46 -070051
James E. Blaire511d2f2016-12-08 15:22:26 -080052import zuul.driver.gerrit.gerritsource as gerritsource
53import zuul.driver.gerrit.gerritconnection as gerritconnection
Clark Boylanb640e052014-04-03 16:41:46 -070054import zuul.scheduler
55import zuul.webapp
56import zuul.rpclistener
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +100057import zuul.launcher.server
58import zuul.launcher.client
Clark Boylanb640e052014-04-03 16:41:46 -070059import zuul.lib.swift
James E. Blair83005782015-12-11 14:46:03 -080060import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070061import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070062import zuul.merger.merger
63import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070064import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080065import zuul.zk
Clark Boylanb640e052014-04-03 16:41:46 -070066
67FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
68 'fixtures')
James E. Blair97d902e2014-08-21 13:25:56 -070069USE_TEMPDIR = True
Clark Boylanb640e052014-04-03 16:41:46 -070070
Clark Boylanb640e052014-04-03 16:41:46 -070071
72def repack_repo(path):
73 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
74 output = subprocess.Popen(cmd, close_fds=True,
75 stdout=subprocess.PIPE,
76 stderr=subprocess.PIPE)
77 out = output.communicate()
78 if output.returncode:
79 raise Exception("git repack returned %d" % output.returncode)
80 return out
81
82
83def random_sha1():
84 return hashlib.sha1(str(random.random())).hexdigest()
85
86
James E. Blaira190f3b2015-01-05 14:56:54 -080087def iterate_timeout(max_seconds, purpose):
88 start = time.time()
89 count = 0
90 while (time.time() < start + max_seconds):
91 count += 1
92 yield count
93 time.sleep(0)
94 raise Exception("Timeout waiting for %s" % purpose)
95
96
Clark Boylanb640e052014-04-03 16:41:46 -070097class ChangeReference(git.Reference):
98 _common_path_default = "refs/changes"
99 _points_to_commits_only = True
100
101
102class FakeChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700103 categories = {'approved': ('Approved', -1, 1),
104 'code-review': ('Code-Review', -2, 2),
105 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700106
107 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700108 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700109 self.gerrit = gerrit
110 self.reported = 0
111 self.queried = 0
112 self.patchsets = []
113 self.number = number
114 self.project = project
115 self.branch = branch
116 self.subject = subject
117 self.latest_patchset = 0
118 self.depends_on_change = None
119 self.needed_by_changes = []
120 self.fail_merge = False
121 self.messages = []
122 self.data = {
123 'branch': branch,
124 'comments': [],
125 'commitMessage': subject,
126 'createdOn': time.time(),
127 'id': 'I' + random_sha1(),
128 'lastUpdated': time.time(),
129 'number': str(number),
130 'open': status == 'NEW',
131 'owner': {'email': 'user@example.com',
132 'name': 'User Name',
133 'username': 'username'},
134 'patchSets': self.patchsets,
135 'project': project,
136 'status': status,
137 'subject': subject,
138 'submitRecords': [],
139 'url': 'https://hostname/%s' % number}
140
141 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700142 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700143 self.data['submitRecords'] = self.getSubmitRecords()
144 self.open = status == 'NEW'
145
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700146 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700147 path = os.path.join(self.upstream_root, self.project)
148 repo = git.Repo(path)
149 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
150 self.latest_patchset),
151 'refs/tags/init')
152 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700153 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700154 repo.git.clean('-x', '-f', '-d')
155
156 path = os.path.join(self.upstream_root, self.project)
157 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700158 for fn, content in files.items():
159 fn = os.path.join(path, fn)
160 with open(fn, 'w') as f:
161 f.write(content)
162 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700163 else:
164 for fni in range(100):
165 fn = os.path.join(path, str(fni))
166 f = open(fn, 'w')
167 for ci in range(4096):
168 f.write(random.choice(string.printable))
169 f.close()
170 repo.index.add([fn])
171
172 r = repo.index.commit(msg)
173 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700174 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700175 repo.git.clean('-x', '-f', '-d')
176 repo.heads['master'].checkout()
177 return r
178
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700179 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700180 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700181 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700182 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700183 data = ("test %s %s %s\n" %
184 (self.branch, self.number, self.latest_patchset))
185 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700186 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700187 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700188 ps_files = [{'file': '/COMMIT_MSG',
189 'type': 'ADDED'},
190 {'file': 'README',
191 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700192 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700193 ps_files.append({'file': f, 'type': 'ADDED'})
194 d = {'approvals': [],
195 'createdOn': time.time(),
196 'files': ps_files,
197 'number': str(self.latest_patchset),
198 'ref': 'refs/changes/1/%s/%s' % (self.number,
199 self.latest_patchset),
200 'revision': c.hexsha,
201 'uploader': {'email': 'user@example.com',
202 'name': 'User name',
203 'username': 'user'}}
204 self.data['currentPatchSet'] = d
205 self.patchsets.append(d)
206 self.data['submitRecords'] = self.getSubmitRecords()
207
208 def getPatchsetCreatedEvent(self, patchset):
209 event = {"type": "patchset-created",
210 "change": {"project": self.project,
211 "branch": self.branch,
212 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
213 "number": str(self.number),
214 "subject": self.subject,
215 "owner": {"name": "User Name"},
216 "url": "https://hostname/3"},
217 "patchSet": self.patchsets[patchset - 1],
218 "uploader": {"name": "User Name"}}
219 return event
220
221 def getChangeRestoredEvent(self):
222 event = {"type": "change-restored",
223 "change": {"project": self.project,
224 "branch": self.branch,
225 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
226 "number": str(self.number),
227 "subject": self.subject,
228 "owner": {"name": "User Name"},
229 "url": "https://hostname/3"},
230 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100231 "patchSet": self.patchsets[-1],
232 "reason": ""}
233 return event
234
235 def getChangeAbandonedEvent(self):
236 event = {"type": "change-abandoned",
237 "change": {"project": self.project,
238 "branch": self.branch,
239 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
240 "number": str(self.number),
241 "subject": self.subject,
242 "owner": {"name": "User Name"},
243 "url": "https://hostname/3"},
244 "abandoner": {"name": "User Name"},
245 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700246 "reason": ""}
247 return event
248
249 def getChangeCommentEvent(self, patchset):
250 event = {"type": "comment-added",
251 "change": {"project": self.project,
252 "branch": self.branch,
253 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
254 "number": str(self.number),
255 "subject": self.subject,
256 "owner": {"name": "User Name"},
257 "url": "https://hostname/3"},
258 "patchSet": self.patchsets[patchset - 1],
259 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700260 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700261 "description": "Code-Review",
262 "value": "0"}],
263 "comment": "This is a comment"}
264 return event
265
Joshua Hesketh642824b2014-07-01 17:54:59 +1000266 def addApproval(self, category, value, username='reviewer_john',
267 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700268 if not granted_on:
269 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000270 approval = {
271 'description': self.categories[category][0],
272 'type': category,
273 'value': str(value),
274 'by': {
275 'username': username,
276 'email': username + '@example.com',
277 },
278 'grantedOn': int(granted_on)
279 }
Clark Boylanb640e052014-04-03 16:41:46 -0700280 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
281 if x['by']['username'] == username and x['type'] == category:
282 del self.patchsets[-1]['approvals'][i]
283 self.patchsets[-1]['approvals'].append(approval)
284 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000285 'author': {'email': 'author@example.com',
286 'name': 'Patchset Author',
287 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700288 'change': {'branch': self.branch,
289 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
290 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000291 'owner': {'email': 'owner@example.com',
292 'name': 'Change Owner',
293 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700294 'project': self.project,
295 'subject': self.subject,
296 'topic': 'master',
297 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000298 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700299 'patchSet': self.patchsets[-1],
300 'type': 'comment-added'}
301 self.data['submitRecords'] = self.getSubmitRecords()
302 return json.loads(json.dumps(event))
303
304 def getSubmitRecords(self):
305 status = {}
306 for cat in self.categories.keys():
307 status[cat] = 0
308
309 for a in self.patchsets[-1]['approvals']:
310 cur = status[a['type']]
311 cat_min, cat_max = self.categories[a['type']][1:]
312 new = int(a['value'])
313 if new == cat_min:
314 cur = new
315 elif abs(new) > abs(cur):
316 cur = new
317 status[a['type']] = cur
318
319 labels = []
320 ok = True
321 for typ, cat in self.categories.items():
322 cur = status[typ]
323 cat_min, cat_max = cat[1:]
324 if cur == cat_min:
325 value = 'REJECT'
326 ok = False
327 elif cur == cat_max:
328 value = 'OK'
329 else:
330 value = 'NEED'
331 ok = False
332 labels.append({'label': cat[0], 'status': value})
333 if ok:
334 return [{'status': 'OK'}]
335 return [{'status': 'NOT_READY',
336 'labels': labels}]
337
338 def setDependsOn(self, other, patchset):
339 self.depends_on_change = other
340 d = {'id': other.data['id'],
341 'number': other.data['number'],
342 'ref': other.patchsets[patchset - 1]['ref']
343 }
344 self.data['dependsOn'] = [d]
345
346 other.needed_by_changes.append(self)
347 needed = other.data.get('neededBy', [])
348 d = {'id': self.data['id'],
349 'number': self.data['number'],
350 'ref': self.patchsets[patchset - 1]['ref'],
351 'revision': self.patchsets[patchset - 1]['revision']
352 }
353 needed.append(d)
354 other.data['neededBy'] = needed
355
356 def query(self):
357 self.queried += 1
358 d = self.data.get('dependsOn')
359 if d:
360 d = d[0]
361 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
362 d['isCurrentPatchSet'] = True
363 else:
364 d['isCurrentPatchSet'] = False
365 return json.loads(json.dumps(self.data))
366
367 def setMerged(self):
368 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000369 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700370 return
371 if self.fail_merge:
372 return
373 self.data['status'] = 'MERGED'
374 self.open = False
375
376 path = os.path.join(self.upstream_root, self.project)
377 repo = git.Repo(path)
378 repo.heads[self.branch].commit = \
379 repo.commit(self.patchsets[-1]['revision'])
380
381 def setReported(self):
382 self.reported += 1
383
384
James E. Blaire511d2f2016-12-08 15:22:26 -0800385class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700386 """A Fake Gerrit connection for use in tests.
387
388 This subclasses
389 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
390 ability for tests to add changes to the fake Gerrit it represents.
391 """
392
Joshua Hesketh352264b2015-08-11 23:42:08 +1000393 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700394
James E. Blaire511d2f2016-12-08 15:22:26 -0800395 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700396 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800397 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000398 connection_config)
399
James E. Blair7fc8daa2016-08-08 15:37:15 -0700400 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700401 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
402 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000403 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700404 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200405 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700406
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700407 def addFakeChange(self, project, branch, subject, status='NEW',
408 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700409 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700410 self.change_number += 1
411 c = FakeChange(self, self.change_number, project, branch, subject,
412 upstream_root=self.upstream_root,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700413 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700414 self.changes[self.change_number] = c
415 return c
416
Clark Boylanb640e052014-04-03 16:41:46 -0700417 def review(self, project, changeid, message, action):
418 number, ps = changeid.split(',')
419 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000420
421 # Add the approval back onto the change (ie simulate what gerrit would
422 # do).
423 # Usually when zuul leaves a review it'll create a feedback loop where
424 # zuul's review enters another gerrit event (which is then picked up by
425 # zuul). However, we can't mimic this behaviour (by adding this
426 # approval event into the queue) as it stops jobs from checking what
427 # happens before this event is triggered. If a job needs to see what
428 # happens they can add their own verified event into the queue.
429 # Nevertheless, we can update change with the new review in gerrit.
430
James E. Blair8b5408c2016-08-08 15:37:46 -0700431 for cat in action.keys():
432 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000433 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000434
James E. Blair8b5408c2016-08-08 15:37:46 -0700435 # TODOv3(jeblair): can this be removed?
Joshua Hesketh642824b2014-07-01 17:54:59 +1000436 if 'label' in action:
437 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000438 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000439
Clark Boylanb640e052014-04-03 16:41:46 -0700440 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000441
Clark Boylanb640e052014-04-03 16:41:46 -0700442 if 'submit' in action:
443 change.setMerged()
444 if message:
445 change.setReported()
446
447 def query(self, number):
448 change = self.changes.get(int(number))
449 if change:
450 return change.query()
451 return {}
452
James E. Blairc494d542014-08-06 09:23:52 -0700453 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700454 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700455 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800456 if query.startswith('change:'):
457 # Query a specific changeid
458 changeid = query[len('change:'):]
459 l = [change.query() for change in self.changes.values()
460 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700461 elif query.startswith('message:'):
462 # Query the content of a commit message
463 msg = query[len('message:'):].strip()
464 l = [change.query() for change in self.changes.values()
465 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800466 else:
467 # Query all open changes
468 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700469 return l
James E. Blairc494d542014-08-06 09:23:52 -0700470
Joshua Hesketh352264b2015-08-11 23:42:08 +1000471 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700472 pass
473
Joshua Hesketh352264b2015-08-11 23:42:08 +1000474 def getGitUrl(self, project):
475 return os.path.join(self.upstream_root, project.name)
476
Adam Gandelmanc5e4f1d2016-11-29 14:27:17 -0800477 def _getGitwebUrl(self, project, sha=None):
478 return self.getGitwebUrl(project, sha)
479
Clark Boylanb640e052014-04-03 16:41:46 -0700480
481class BuildHistory(object):
482 def __init__(self, **kw):
483 self.__dict__.update(kw)
484
485 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -0700486 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
487 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -0700488
489
490class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200491 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700492 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700493 self.url = url
494
495 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -0700496 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -0700497 path = res.path
498 project = '/'.join(path.split('/')[2:-2])
499 ret = '001e# service=git-upload-pack\n'
500 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
501 'multi_ack thin-pack side-band side-band-64k ofs-delta '
502 'shallow no-progress include-tag multi_ack_detailed no-done\n')
503 path = os.path.join(self.upstream_root, project)
504 repo = git.Repo(path)
505 for ref in repo.refs:
506 r = ref.object.hexsha + ' ' + ref.path + '\n'
507 ret += '%04x%s' % (len(r) + 4, r)
508 ret += '0000'
509 return ret
510
511
Clark Boylanb640e052014-04-03 16:41:46 -0700512class FakeStatsd(threading.Thread):
513 def __init__(self):
514 threading.Thread.__init__(self)
515 self.daemon = True
516 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
517 self.sock.bind(('', 0))
518 self.port = self.sock.getsockname()[1]
519 self.wake_read, self.wake_write = os.pipe()
520 self.stats = []
521
522 def run(self):
523 while True:
524 poll = select.poll()
525 poll.register(self.sock, select.POLLIN)
526 poll.register(self.wake_read, select.POLLIN)
527 ret = poll.poll()
528 for (fd, event) in ret:
529 if fd == self.sock.fileno():
530 data = self.sock.recvfrom(1024)
531 if not data:
532 return
533 self.stats.append(data[0])
534 if fd == self.wake_read:
535 return
536
537 def stop(self):
538 os.write(self.wake_write, '1\n')
539
540
James E. Blaire1767bc2016-08-02 10:00:27 -0700541class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -0700542 log = logging.getLogger("zuul.test")
543
James E. Blair34776ee2016-08-25 13:53:54 -0700544 def __init__(self, launch_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -0700545 self.daemon = True
James E. Blaire1767bc2016-08-02 10:00:27 -0700546 self.launch_server = launch_server
Clark Boylanb640e052014-04-03 16:41:46 -0700547 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -0700548 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -0700549 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -0700550 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -0700551 # TODOv3(jeblair): self.node is really "the image of the node
552 # assigned". We should rename it (self.node_image?) if we
553 # keep using it like this, or we may end up exposing more of
554 # the complexity around multi-node jobs here
555 # (self.nodes[0].image?)
556 self.node = None
557 if len(self.parameters.get('nodes')) == 1:
558 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -0700559 self.unique = self.parameters['ZUUL_UUID']
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100560 self.pipeline = self.parameters['ZUUL_PIPELINE']
561 self.project = self.parameters['ZUUL_PROJECT']
James E. Blair3f876d52016-07-22 13:07:14 -0700562 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -0700563 self.wait_condition = threading.Condition()
564 self.waiting = False
565 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -0500566 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -0700567 self.created = time.time()
James E. Blaire1767bc2016-08-02 10:00:27 -0700568 self.changes = None
569 if 'ZUUL_CHANGE_IDS' in self.parameters:
570 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -0700571
James E. Blair3158e282016-08-19 09:34:11 -0700572 def __repr__(self):
573 waiting = ''
574 if self.waiting:
575 waiting = ' [waiting]'
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100576 return '<FakeBuild %s:%s %s%s>' % (self.pipeline, self.name,
577 self.changes, waiting)
James E. Blair3158e282016-08-19 09:34:11 -0700578
Clark Boylanb640e052014-04-03 16:41:46 -0700579 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700580 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -0700581 self.wait_condition.acquire()
582 self.wait_condition.notify()
583 self.waiting = False
584 self.log.debug("Build %s released" % self.unique)
585 self.wait_condition.release()
586
587 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700588 """Return whether this build is being held.
589
590 :returns: Whether the build is being held.
591 :rtype: bool
592 """
593
Clark Boylanb640e052014-04-03 16:41:46 -0700594 self.wait_condition.acquire()
595 if self.waiting:
596 ret = True
597 else:
598 ret = False
599 self.wait_condition.release()
600 return ret
601
602 def _wait(self):
603 self.wait_condition.acquire()
604 self.waiting = True
605 self.log.debug("Build %s waiting" % self.unique)
606 self.wait_condition.wait()
607 self.wait_condition.release()
608
609 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -0700610 self.log.debug('Running build %s' % self.unique)
611
James E. Blaire1767bc2016-08-02 10:00:27 -0700612 if self.launch_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -0700613 self.log.debug('Holding build %s' % self.unique)
614 self._wait()
615 self.log.debug("Build %s continuing" % self.unique)
616
James E. Blair412fba82017-01-26 15:00:50 -0800617 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
James E. Blaira5dba232016-08-08 15:53:24 -0700618 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
James E. Blair412fba82017-01-26 15:00:50 -0800619 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
Clark Boylanb640e052014-04-03 16:41:46 -0700620 if self.aborted:
James E. Blair412fba82017-01-26 15:00:50 -0800621 result = (RecordingAnsibleJob.RESULT_ABORTED, None)
Paul Belanger71d98172016-11-08 10:56:31 -0500622 if self.requeue:
James E. Blair412fba82017-01-26 15:00:50 -0800623 result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
Clark Boylanb640e052014-04-03 16:41:46 -0700624
James E. Blaire1767bc2016-08-02 10:00:27 -0700625 return result
Clark Boylanb640e052014-04-03 16:41:46 -0700626
James E. Blaira5dba232016-08-08 15:53:24 -0700627 def shouldFail(self):
628 changes = self.launch_server.fail_tests.get(self.name, [])
629 for change in changes:
630 if self.hasChanges(change):
631 return True
632 return False
633
James E. Blaire7b99a02016-08-05 14:27:34 -0700634 def hasChanges(self, *changes):
635 """Return whether this build has certain changes in its git repos.
636
637 :arg FakeChange changes: One or more changes (varargs) that
638 are expected to be present (in order) in the git repository of
639 the active project.
640
641 :returns: Whether the build has the indicated changes.
642 :rtype: bool
643
644 """
Clint Byrum3343e3e2016-11-15 16:05:03 -0800645 for change in changes:
646 path = os.path.join(self.jobdir.git_root, change.project)
647 try:
648 repo = git.Repo(path)
649 except NoSuchPathError as e:
650 self.log.debug('%s' % e)
651 return False
652 ref = self.parameters['ZUUL_REF']
653 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
654 commit_message = '%s-1' % change.subject
655 self.log.debug("Checking if build %s has changes; commit_message "
656 "%s; repo_messages %s" % (self, commit_message,
657 repo_messages))
658 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -0700659 self.log.debug(" messages do not match")
660 return False
661 self.log.debug(" OK")
662 return True
663
Clark Boylanb640e052014-04-03 16:41:46 -0700664
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +1000665class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
James E. Blaire7b99a02016-08-05 14:27:34 -0700666 """An Ansible launcher to be used in tests.
667
668 :ivar bool hold_jobs_in_build: If true, when jobs are launched
669 they will report that they have started but then pause until
670 released before reporting completion. This attribute may be
671 changed at any time and will take effect for subsequently
672 launched builds, but previously held builds will still need to
673 be explicitly released.
674
675 """
James E. Blairf5dbd002015-12-23 15:26:17 -0800676 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -0700677 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -0800678 self._test_root = kw.pop('_test_root', False)
James E. Blairf5dbd002015-12-23 15:26:17 -0800679 super(RecordingLaunchServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -0700680 self.hold_jobs_in_build = False
681 self.lock = threading.Lock()
682 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -0700683 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -0700684 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -0700685 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -0800686
James E. Blaira5dba232016-08-08 15:53:24 -0700687 def failJob(self, name, change):
James E. Blaire7b99a02016-08-05 14:27:34 -0700688 """Instruct the launcher to report matching builds as failures.
689
690 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -0700691 :arg Change change: The :py:class:`~tests.base.FakeChange`
692 instance which should cause the job to fail. This job
693 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -0700694
695 """
James E. Blaire1767bc2016-08-02 10:00:27 -0700696 l = self.fail_tests.get(name, [])
697 l.append(change)
698 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -0800699
James E. Blair962220f2016-08-03 11:22:38 -0700700 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700701 """Release a held build.
702
703 :arg str regex: A regular expression which, if supplied, will
704 cause only builds with matching names to be released. If
705 not supplied, all builds will be released.
706
707 """
James E. Blair962220f2016-08-03 11:22:38 -0700708 builds = self.running_builds[:]
709 self.log.debug("Releasing build %s (%s)" % (regex,
710 len(self.running_builds)))
711 for build in builds:
712 if not regex or re.match(regex, build.name):
713 self.log.debug("Releasing build %s" %
714 (build.parameters['ZUUL_UUID']))
715 build.release()
716 else:
717 self.log.debug("Not releasing build %s" %
718 (build.parameters['ZUUL_UUID']))
719 self.log.debug("Done releasing builds %s (%s)" %
720 (regex, len(self.running_builds)))
721
James E. Blair17302972016-08-10 16:11:42 -0700722 def launchJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -0700723 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -0700724 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -0700725 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -0700726 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -0800727 args = json.loads(job.arguments)
728 args['zuul']['_test'] = dict(test_root=self._test_root)
729 job.arguments = json.dumps(args)
Joshua Hesketh50c21782016-10-13 21:34:14 +1100730 self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
731 self.job_workers[job.unique].run()
James E. Blair17302972016-08-10 16:11:42 -0700732
733 def stopJob(self, job):
734 self.log.debug("handle stop")
735 parameters = json.loads(job.arguments)
736 uuid = parameters['uuid']
737 for build in self.running_builds:
738 if build.unique == uuid:
739 build.aborted = True
740 build.release()
741 super(RecordingLaunchServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -0700742
Joshua Hesketh50c21782016-10-13 21:34:14 +1100743
744class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob):
James E. Blair412fba82017-01-26 15:00:50 -0800745 def runPlaybooks(self):
Joshua Hesketh50c21782016-10-13 21:34:14 +1100746 build = self.launcher_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -0800747 build.jobdir = self.jobdir
James E. Blaire1767bc2016-08-02 10:00:27 -0700748
James E. Blair412fba82017-01-26 15:00:50 -0800749 result = super(RecordingAnsibleJob, self).runPlaybooks()
750
Joshua Hesketh50c21782016-10-13 21:34:14 +1100751 self.launcher_server.lock.acquire()
752 self.launcher_server.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -0700753 BuildHistory(name=build.name, result=result, changes=build.changes,
754 node=build.node, uuid=build.unique,
755 parameters=build.parameters,
James E. Blaire1767bc2016-08-02 10:00:27 -0700756 pipeline=build.parameters['ZUUL_PIPELINE'])
757 )
Joshua Hesketh50c21782016-10-13 21:34:14 +1100758 self.launcher_server.running_builds.remove(build)
759 del self.launcher_server.job_builds[self.job.unique]
760 self.launcher_server.lock.release()
James E. Blair412fba82017-01-26 15:00:50 -0800761 return result
762
763 def runAnsible(self, cmd, timeout):
764 build = self.launcher_server.job_builds[self.job.unique]
765
766 if self.launcher_server._run_ansible:
767 result = super(RecordingAnsibleJob, self).runAnsible(cmd, timeout)
768 else:
769 result = build.run()
James E. Blaire1767bc2016-08-02 10:00:27 -0700770 return result
James E. Blairf5dbd002015-12-23 15:26:17 -0800771
772
Clark Boylanb640e052014-04-03 16:41:46 -0700773class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -0700774 """A Gearman server for use in tests.
775
776 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
777 added to the queue but will not be distributed to workers
778 until released. This attribute may be changed at any time and
779 will take effect for subsequently enqueued jobs, but
780 previously held jobs will still need to be explicitly
781 released.
782
783 """
784
Clark Boylanb640e052014-04-03 16:41:46 -0700785 def __init__(self):
786 self.hold_jobs_in_queue = False
787 super(FakeGearmanServer, self).__init__(0)
788
789 def getJobForConnection(self, connection, peek=False):
790 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
791 for job in queue:
792 if not hasattr(job, 'waiting'):
Paul Belanger6ab6af72016-11-06 11:32:59 -0500793 if job.name.startswith('launcher:launch'):
Clark Boylanb640e052014-04-03 16:41:46 -0700794 job.waiting = self.hold_jobs_in_queue
795 else:
796 job.waiting = False
797 if job.waiting:
798 continue
799 if job.name in connection.functions:
800 if not peek:
801 queue.remove(job)
802 connection.related_jobs[job.handle] = job
803 job.worker_connection = connection
804 job.running = True
805 return job
806 return None
807
808 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700809 """Release a held job.
810
811 :arg str regex: A regular expression which, if supplied, will
812 cause only jobs with matching names to be released. If
813 not supplied, all jobs will be released.
814 """
Clark Boylanb640e052014-04-03 16:41:46 -0700815 released = False
816 qlen = (len(self.high_queue) + len(self.normal_queue) +
817 len(self.low_queue))
818 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
819 for job in self.getQueue():
Paul Belanger6ab6af72016-11-06 11:32:59 -0500820 if job.name != 'launcher:launch':
Clark Boylanb640e052014-04-03 16:41:46 -0700821 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -0500822 parameters = json.loads(job.arguments)
823 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -0700824 self.log.debug("releasing queued job %s" %
825 job.unique)
826 job.waiting = False
827 released = True
828 else:
829 self.log.debug("not releasing queued job %s" %
830 job.unique)
831 if released:
832 self.wakeConnections()
833 qlen = (len(self.high_queue) + len(self.normal_queue) +
834 len(self.low_queue))
835 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
836
837
838class FakeSMTP(object):
839 log = logging.getLogger('zuul.FakeSMTP')
840
841 def __init__(self, messages, server, port):
842 self.server = server
843 self.port = port
844 self.messages = messages
845
846 def sendmail(self, from_email, to_email, msg):
847 self.log.info("Sending email from %s, to %s, with msg %s" % (
848 from_email, to_email, msg))
849
850 headers = msg.split('\n\n', 1)[0]
851 body = msg.split('\n\n', 1)[1]
852
853 self.messages.append(dict(
854 from_email=from_email,
855 to_email=to_email,
856 msg=msg,
857 headers=headers,
858 body=body,
859 ))
860
861 return True
862
863 def quit(self):
864 return True
865
866
867class FakeSwiftClientConnection(swiftclient.client.Connection):
868 def post_account(self, headers):
869 # Do nothing
870 pass
871
872 def get_auth(self):
873 # Returns endpoint and (unused) auth token
874 endpoint = os.path.join('https://storage.example.org', 'V1',
875 'AUTH_account')
876 return endpoint, ''
877
878
James E. Blairdce6cea2016-12-20 16:45:32 -0800879class FakeNodepool(object):
880 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -0800881 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -0800882
883 log = logging.getLogger("zuul.test.FakeNodepool")
884
885 def __init__(self, host, port, chroot):
886 self.client = kazoo.client.KazooClient(
887 hosts='%s:%s%s' % (host, port, chroot))
888 self.client.start()
889 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -0800890 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -0800891 self.thread = threading.Thread(target=self.run)
892 self.thread.daemon = True
893 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -0800894 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -0800895
896 def stop(self):
897 self._running = False
898 self.thread.join()
899 self.client.stop()
900 self.client.close()
901
902 def run(self):
903 while self._running:
904 self._run()
905 time.sleep(0.1)
906
907 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -0800908 if self.paused:
909 return
James E. Blairdce6cea2016-12-20 16:45:32 -0800910 for req in self.getNodeRequests():
911 self.fulfillRequest(req)
912
913 def getNodeRequests(self):
914 try:
915 reqids = self.client.get_children(self.REQUEST_ROOT)
916 except kazoo.exceptions.NoNodeError:
917 return []
918 reqs = []
919 for oid in sorted(reqids):
920 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -0800921 try:
922 data, stat = self.client.get(path)
923 data = json.loads(data)
924 data['_oid'] = oid
925 reqs.append(data)
926 except kazoo.exceptions.NoNodeError:
927 pass
James E. Blairdce6cea2016-12-20 16:45:32 -0800928 return reqs
929
James E. Blaire18d4602017-01-05 11:17:28 -0800930 def getNodes(self):
931 try:
932 nodeids = self.client.get_children(self.NODE_ROOT)
933 except kazoo.exceptions.NoNodeError:
934 return []
935 nodes = []
936 for oid in sorted(nodeids):
937 path = self.NODE_ROOT + '/' + oid
938 data, stat = self.client.get(path)
939 data = json.loads(data)
940 data['_oid'] = oid
941 try:
942 lockfiles = self.client.get_children(path + '/lock')
943 except kazoo.exceptions.NoNodeError:
944 lockfiles = []
945 if lockfiles:
946 data['_lock'] = True
947 else:
948 data['_lock'] = False
949 nodes.append(data)
950 return nodes
951
James E. Blaira38c28e2017-01-04 10:33:20 -0800952 def makeNode(self, request_id, node_type):
953 now = time.time()
954 path = '/nodepool/nodes/'
955 data = dict(type=node_type,
956 provider='test-provider',
957 region='test-region',
958 az=None,
959 public_ipv4='127.0.0.1',
960 private_ipv4=None,
961 public_ipv6=None,
962 allocated_to=request_id,
963 state='ready',
964 state_time=now,
965 created_time=now,
966 updated_time=now,
967 image_id=None,
968 launcher='fake-nodepool')
969 data = json.dumps(data)
970 path = self.client.create(path, data,
971 makepath=True,
972 sequence=True)
973 nodeid = path.split("/")[-1]
974 return nodeid
975
James E. Blair6ab79e02017-01-06 10:10:17 -0800976 def addFailRequest(self, request):
977 self.fail_requests.add(request['_oid'])
978
James E. Blairdce6cea2016-12-20 16:45:32 -0800979 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -0800980 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -0800981 return
982 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -0800983 oid = request['_oid']
984 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -0800985
James E. Blair6ab79e02017-01-06 10:10:17 -0800986 if oid in self.fail_requests:
987 request['state'] = 'failed'
988 else:
989 request['state'] = 'fulfilled'
990 nodes = []
991 for node in request['node_types']:
992 nodeid = self.makeNode(oid, node)
993 nodes.append(nodeid)
994 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -0800995
James E. Blaira38c28e2017-01-04 10:33:20 -0800996 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -0800997 path = self.REQUEST_ROOT + '/' + oid
998 data = json.dumps(request)
999 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
1000 self.client.set(path, data)
1001
1002
James E. Blair498059b2016-12-20 13:50:13 -08001003class ChrootedKazooFixture(fixtures.Fixture):
1004 def __init__(self):
1005 super(ChrootedKazooFixture, self).__init__()
1006
1007 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1008 if ':' in zk_host:
1009 host, port = zk_host.split(':')
1010 else:
1011 host = zk_host
1012 port = None
1013
1014 self.zookeeper_host = host
1015
1016 if not port:
1017 self.zookeeper_port = 2181
1018 else:
1019 self.zookeeper_port = int(port)
1020
1021 def _setUp(self):
1022 # Make sure the test chroot paths do not conflict
1023 random_bits = ''.join(random.choice(string.ascii_lowercase +
1024 string.ascii_uppercase)
1025 for x in range(8))
1026
1027 rand_test_path = '%s_%s' % (random_bits, os.getpid())
1028 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1029
1030 # Ensure the chroot path exists and clean up any pre-existing znodes.
1031 _tmp_client = kazoo.client.KazooClient(
1032 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1033 _tmp_client.start()
1034
1035 if _tmp_client.exists(self.zookeeper_chroot):
1036 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1037
1038 _tmp_client.ensure_path(self.zookeeper_chroot)
1039 _tmp_client.stop()
1040 _tmp_client.close()
1041
1042 self.addCleanup(self._cleanup)
1043
1044 def _cleanup(self):
1045 '''Remove the chroot path.'''
1046 # Need a non-chroot'ed client to remove the chroot path
1047 _tmp_client = kazoo.client.KazooClient(
1048 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1049 _tmp_client.start()
1050 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1051 _tmp_client.stop()
1052
1053
Maru Newby3fe5f852015-01-13 04:22:14 +00001054class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001055 log = logging.getLogger("zuul.test")
1056
James E. Blair1c236df2017-02-01 14:07:24 -08001057 def attachLogs(self, *args):
1058 def reader():
1059 self._log_stream.seek(0)
1060 while True:
1061 x = self._log_stream.read(4096)
1062 if not x:
1063 break
1064 yield x.encode('utf8')
1065 content = testtools.content.content_from_reader(
1066 reader,
1067 testtools.content_type.UTF8_TEXT,
1068 False)
1069 self.addDetail('logging', content)
1070
Clark Boylanb640e052014-04-03 16:41:46 -07001071 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001072 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001073 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1074 try:
1075 test_timeout = int(test_timeout)
1076 except ValueError:
1077 # If timeout value is invalid do not set a timeout.
1078 test_timeout = 0
1079 if test_timeout > 0:
1080 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1081
1082 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1083 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1084 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1085 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1086 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1087 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1088 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1089 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1090 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1091 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001092 self._log_stream = StringIO()
1093 self.addOnException(self.attachLogs)
1094 else:
1095 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001096
James E. Blair1c236df2017-02-01 14:07:24 -08001097 handler = logging.StreamHandler(self._log_stream)
1098 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1099 '%(levelname)-8s %(message)s')
1100 handler.setFormatter(formatter)
1101
1102 logger = logging.getLogger()
1103 logger.setLevel(logging.DEBUG)
1104 logger.addHandler(handler)
1105
1106 # NOTE(notmorgan): Extract logging overrides for specific
1107 # libraries from the OS_LOG_DEFAULTS env and create loggers
1108 # for each. This is used to limit the output during test runs
1109 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001110 log_defaults_from_env = os.environ.get(
1111 'OS_LOG_DEFAULTS',
James E. Blair1c236df2017-02-01 14:07:24 -08001112 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001113
James E. Blairdce6cea2016-12-20 16:45:32 -08001114 if log_defaults_from_env:
1115 for default in log_defaults_from_env.split(','):
1116 try:
1117 name, level_str = default.split('=', 1)
1118 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001119 logger = logging.getLogger(name)
1120 logger.setLevel(level)
1121 logger.addHandler(handler)
1122 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001123 except ValueError:
1124 # NOTE(notmorgan): Invalid format of the log default,
1125 # skip and don't try and apply a logger for the
1126 # specified module
1127 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001128
Maru Newby3fe5f852015-01-13 04:22:14 +00001129
1130class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001131 """A test case with a functioning Zuul.
1132
1133 The following class variables are used during test setup and can
1134 be overidden by subclasses but are effectively read-only once a
1135 test method starts running:
1136
1137 :cvar str config_file: This points to the main zuul config file
1138 within the fixtures directory. Subclasses may override this
1139 to obtain a different behavior.
1140
1141 :cvar str tenant_config_file: This is the tenant config file
1142 (which specifies from what git repos the configuration should
1143 be loaded). It defaults to the value specified in
1144 `config_file` but can be overidden by subclasses to obtain a
1145 different tenant/project layout while using the standard main
1146 configuration.
1147
1148 The following are instance variables that are useful within test
1149 methods:
1150
1151 :ivar FakeGerritConnection fake_<connection>:
1152 A :py:class:`~tests.base.FakeGerritConnection` will be
1153 instantiated for each connection present in the config file
1154 and stored here. For instance, `fake_gerrit` will hold the
1155 FakeGerritConnection object for a connection named `gerrit`.
1156
1157 :ivar FakeGearmanServer gearman_server: An instance of
1158 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1159 server that all of the Zuul components in this test use to
1160 communicate with each other.
1161
1162 :ivar RecordingLaunchServer launch_server: An instance of
1163 :py:class:`~tests.base.RecordingLaunchServer` which is the
1164 Ansible launch server used to run jobs for this test.
1165
1166 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1167 representing currently running builds. They are appended to
1168 the list in the order they are launched, and removed from this
1169 list upon completion.
1170
1171 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1172 objects representing completed builds. They are appended to
1173 the list in the order they complete.
1174
1175 """
1176
James E. Blair83005782015-12-11 14:46:03 -08001177 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001178 run_ansible = False
James E. Blair3f876d52016-07-22 13:07:14 -07001179
1180 def _startMerger(self):
1181 self.merge_server = zuul.merger.server.MergeServer(self.config,
1182 self.connections)
1183 self.merge_server.start()
1184
Maru Newby3fe5f852015-01-13 04:22:14 +00001185 def setUp(self):
1186 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001187
1188 self.setupZK()
1189
James E. Blair97d902e2014-08-21 13:25:56 -07001190 if USE_TEMPDIR:
1191 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001192 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1193 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001194 else:
1195 tmp_root = os.environ.get("ZUUL_TEST_ROOT")
Clark Boylanb640e052014-04-03 16:41:46 -07001196 self.test_root = os.path.join(tmp_root, "zuul-test")
1197 self.upstream_root = os.path.join(self.test_root, "upstream")
James E. Blair8c1be532017-02-07 14:04:12 -08001198 self.merger_git_root = os.path.join(self.test_root, "merger-git")
1199 self.launcher_git_root = os.path.join(self.test_root, "launcher-git")
James E. Blairce8a2132016-05-19 15:21:52 -07001200 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001201
1202 if os.path.exists(self.test_root):
1203 shutil.rmtree(self.test_root)
1204 os.makedirs(self.test_root)
1205 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001206 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001207
1208 # Make per test copy of Configuration.
1209 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001210 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001211 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001212 self.config.get('zuul', 'tenant_config')))
James E. Blair8c1be532017-02-07 14:04:12 -08001213 self.config.set('merger', 'git_dir', self.merger_git_root)
1214 self.config.set('launcher', 'git_dir', self.launcher_git_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001215 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001216
1217 # For each project in config:
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001218 # TODOv3(jeblair): remove these and replace with new git
1219 # filesystem fixtures
Clark Boylanb640e052014-04-03 16:41:46 -07001220 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -07001221 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -07001222 self.init_repo("org/project5")
1223 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -07001224 self.init_repo("org/one-job-project")
1225 self.init_repo("org/nonvoting-project")
1226 self.init_repo("org/templated-project")
1227 self.init_repo("org/layered-project")
1228 self.init_repo("org/node-project")
1229 self.init_repo("org/conflict-project")
1230 self.init_repo("org/noop-project")
1231 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00001232 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -07001233
1234 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001235 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1236 # see: https://github.com/jsocol/pystatsd/issues/61
1237 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001238 os.environ['STATSD_PORT'] = str(self.statsd.port)
1239 self.statsd.start()
1240 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001241 reload_module(statsd)
1242 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001243
1244 self.gearman_server = FakeGearmanServer()
1245
1246 self.config.set('gearman', 'port', str(self.gearman_server.port))
James E. Blaire47eb772017-02-02 17:19:40 -08001247 self.log.info("Gearman server on port %s" %
1248 (self.gearman_server.port,))
Clark Boylanb640e052014-04-03 16:41:46 -07001249
James E. Blaire511d2f2016-12-08 15:22:26 -08001250 gerritsource.GerritSource.replication_timeout = 1.5
1251 gerritsource.GerritSource.replication_retry_interval = 0.5
1252 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001253
Joshua Hesketh352264b2015-08-11 23:42:08 +10001254 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001255
1256 self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
1257 FakeSwiftClientConnection))
James E. Blaire511d2f2016-12-08 15:22:26 -08001258
Clark Boylanb640e052014-04-03 16:41:46 -07001259 self.swift = zuul.lib.swift.Swift(self.config)
1260
Jan Hruban6b71aff2015-10-22 16:58:08 +02001261 self.event_queues = [
1262 self.sched.result_event_queue,
1263 self.sched.trigger_event_queue
1264 ]
1265
James E. Blairfef78942016-03-11 16:28:56 -08001266 self.configure_connections()
Joshua Hesketh352264b2015-08-11 23:42:08 +10001267 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001268
Clark Boylanb640e052014-04-03 16:41:46 -07001269 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001270 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001271 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001272 return FakeURLOpener(self.upstream_root, *args, **kw)
1273
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001274 old_urlopen = urllib.request.urlopen
1275 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001276
James E. Blair3f876d52016-07-22 13:07:14 -07001277 self._startMerger()
James E. Blair3f876d52016-07-22 13:07:14 -07001278
James E. Blaire1767bc2016-08-02 10:00:27 -07001279 self.launch_server = RecordingLaunchServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001280 self.config, self.connections,
James E. Blair854f8892017-02-02 11:25:39 -08001281 jobdir_root=self.test_root,
James E. Blaira92cbc82017-01-23 14:56:49 -08001282 _run_ansible=self.run_ansible,
1283 _test_root=self.test_root)
James E. Blaire1767bc2016-08-02 10:00:27 -07001284 self.launch_server.start()
1285 self.history = self.launch_server.build_history
1286 self.builds = self.launch_server.running_builds
1287
1288 self.launch_client = zuul.launcher.client.LaunchClient(
James E. Blair82938472016-01-11 14:38:13 -08001289 self.config, self.sched, self.swift)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001290 self.merge_client = zuul.merger.client.MergeClient(
1291 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001292 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001293 self.zk = zuul.zk.ZooKeeper()
1294 self.zk.connect([self.zk_config])
1295
1296 self.fake_nodepool = FakeNodepool(self.zk_config.host,
1297 self.zk_config.port,
1298 self.zk_config.chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001299
James E. Blaire1767bc2016-08-02 10:00:27 -07001300 self.sched.setLauncher(self.launch_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001301 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001302 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001303 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001304
Paul Belanger88ef0ea2015-12-23 11:57:02 -05001305 self.webapp = zuul.webapp.WebApp(
1306 self.sched, port=0, listen_address='127.0.0.1')
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001307 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001308
1309 self.sched.start()
1310 self.sched.reconfigure(self.config)
1311 self.sched.resume()
1312 self.webapp.start()
1313 self.rpc.start()
James E. Blaire1767bc2016-08-02 10:00:27 -07001314 self.launch_client.gearman.waitForServer()
Clark Boylanb640e052014-04-03 16:41:46 -07001315
Clark Boylanb640e052014-04-03 16:41:46 -07001316 self.addCleanup(self.shutdown)
1317
James E. Blaire18d4602017-01-05 11:17:28 -08001318 def tearDown(self):
1319 super(ZuulTestCase, self).tearDown()
1320 self.assertFinalState()
1321
James E. Blairfef78942016-03-11 16:28:56 -08001322 def configure_connections(self):
James E. Blaire511d2f2016-12-08 15:22:26 -08001323 # Set up gerrit related fakes
1324 # Set a changes database so multiple FakeGerrit's can report back to
1325 # a virtual canonical database given by the configured hostname
1326 self.gerrit_changes_dbs = {}
1327
1328 def getGerritConnection(driver, name, config):
1329 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1330 con = FakeGerritConnection(driver, name, config,
1331 changes_db=db,
1332 upstream_root=self.upstream_root)
1333 self.event_queues.append(con.event_queue)
1334 setattr(self, 'fake_' + name, con)
1335 return con
1336
1337 self.useFixture(fixtures.MonkeyPatch(
1338 'zuul.driver.gerrit.GerritDriver.getConnection',
1339 getGerritConnection))
1340
1341 # Set up smtp related fakes
Joshua Hesketh352264b2015-08-11 23:42:08 +10001342 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001343
Joshua Hesketh352264b2015-08-11 23:42:08 +10001344 def FakeSMTPFactory(*args, **kw):
1345 args = [self.smtp_messages] + list(args)
1346 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001347
Joshua Hesketh352264b2015-08-11 23:42:08 +10001348 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001349
James E. Blaire511d2f2016-12-08 15:22:26 -08001350 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001351 self.connections = zuul.lib.connections.ConnectionRegistry()
James E. Blaire511d2f2016-12-08 15:22:26 -08001352 self.connections.configure(self.config)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001353
James E. Blair83005782015-12-11 14:46:03 -08001354 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001355 # This creates the per-test configuration object. It can be
1356 # overriden by subclasses, but should not need to be since it
1357 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001358 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001359 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair2a629ec2015-12-22 15:32:02 -08001360 if hasattr(self, 'tenant_config_file'):
1361 self.config.set('zuul', 'tenant_config', self.tenant_config_file)
James E. Blair96c6bf82016-01-15 16:20:40 -08001362 git_path = os.path.join(
1363 os.path.dirname(
1364 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1365 'git')
1366 if os.path.exists(git_path):
1367 for reponame in os.listdir(git_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001368 project = reponame.replace('_', '/')
1369 self.copyDirToRepo(project,
James E. Blair96c6bf82016-01-15 16:20:40 -08001370 os.path.join(git_path, reponame))
1371
James E. Blair498059b2016-12-20 13:50:13 -08001372 def setupZK(self):
1373 self.zk_chroot_fixture = self.useFixture(ChrootedKazooFixture())
James E. Blairdce6cea2016-12-20 16:45:32 -08001374 self.zk_config = zuul.zk.ZooKeeperConnectionConfig(
1375 self.zk_chroot_fixture.zookeeper_host,
1376 self.zk_chroot_fixture.zookeeper_port,
1377 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08001378
James E. Blair96c6bf82016-01-15 16:20:40 -08001379 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001380 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08001381
1382 files = {}
1383 for (dirpath, dirnames, filenames) in os.walk(source_path):
1384 for filename in filenames:
1385 test_tree_filepath = os.path.join(dirpath, filename)
1386 common_path = os.path.commonprefix([test_tree_filepath,
1387 source_path])
1388 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1389 with open(test_tree_filepath, 'r') as f:
1390 content = f.read()
1391 files[relative_filepath] = content
1392 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001393 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08001394
James E. Blaire18d4602017-01-05 11:17:28 -08001395 def assertNodepoolState(self):
1396 # Make sure that there are no pending requests
1397
1398 requests = self.fake_nodepool.getNodeRequests()
1399 self.assertEqual(len(requests), 0)
1400
1401 nodes = self.fake_nodepool.getNodes()
1402 for node in nodes:
1403 self.assertFalse(node['_lock'], "Node %s is locked" %
1404 (node['_oid'],))
1405
Clark Boylanb640e052014-04-03 16:41:46 -07001406 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001407 # Make sure that git.Repo objects have been garbage collected.
1408 repos = []
1409 gc.collect()
1410 for obj in gc.get_objects():
1411 if isinstance(obj, git.Repo):
1412 repos.append(obj)
1413 self.assertEqual(len(repos), 0)
1414 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08001415 self.assertNodepoolState()
James E. Blair83005782015-12-11 14:46:03 -08001416 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001417 for tenant in self.sched.abide.tenants.values():
1418 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001419 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001420 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001421
1422 def shutdown(self):
1423 self.log.debug("Shutting down after tests")
James E. Blaire1767bc2016-08-02 10:00:27 -07001424 self.launch_client.stop()
James E. Blair3f876d52016-07-22 13:07:14 -07001425 self.merge_server.stop()
1426 self.merge_server.join()
Clark Boylanb640e052014-04-03 16:41:46 -07001427 self.merge_client.stop()
James E. Blaire1767bc2016-08-02 10:00:27 -07001428 self.launch_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001429 self.sched.stop()
1430 self.sched.join()
1431 self.statsd.stop()
1432 self.statsd.join()
1433 self.webapp.stop()
1434 self.webapp.join()
1435 self.rpc.stop()
1436 self.rpc.join()
1437 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08001438 self.fake_nodepool.stop()
1439 self.zk.disconnect()
Clark Boylanb640e052014-04-03 16:41:46 -07001440 threads = threading.enumerate()
1441 if len(threads) > 1:
1442 self.log.error("More than one thread is running: %s" % threads)
James E. Blair6ac368c2016-12-22 18:07:20 -08001443 self.printHistory()
Clark Boylanb640e052014-04-03 16:41:46 -07001444
1445 def init_repo(self, project):
1446 parts = project.split('/')
1447 path = os.path.join(self.upstream_root, *parts[:-1])
1448 if not os.path.exists(path):
1449 os.makedirs(path)
1450 path = os.path.join(self.upstream_root, project)
1451 repo = git.Repo.init(path)
1452
Morgan Fainberg78c301a2016-07-14 13:47:01 -07001453 with repo.config_writer() as config_writer:
1454 config_writer.set_value('user', 'email', 'user@example.com')
1455 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07001456
Clark Boylanb640e052014-04-03 16:41:46 -07001457 repo.index.commit('initial commit')
1458 master = repo.create_head('master')
Clark Boylanb640e052014-04-03 16:41:46 -07001459
James E. Blair97d902e2014-08-21 13:25:56 -07001460 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001461 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001462 repo.git.clean('-x', '-f', '-d')
1463
James E. Blair97d902e2014-08-21 13:25:56 -07001464 def create_branch(self, project, branch):
1465 path = os.path.join(self.upstream_root, project)
1466 repo = git.Repo.init(path)
1467 fn = os.path.join(path, 'README')
1468
1469 branch_head = repo.create_head(branch)
1470 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001471 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001472 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001473 f.close()
1474 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001475 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001476
James E. Blair97d902e2014-08-21 13:25:56 -07001477 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001478 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001479 repo.git.clean('-x', '-f', '-d')
1480
Sachi King9f16d522016-03-16 12:20:45 +11001481 def create_commit(self, project):
1482 path = os.path.join(self.upstream_root, project)
1483 repo = git.Repo(path)
1484 repo.head.reference = repo.heads['master']
1485 file_name = os.path.join(path, 'README')
1486 with open(file_name, 'a') as f:
1487 f.write('creating fake commit\n')
1488 repo.index.add([file_name])
1489 commit = repo.index.commit('Creating a fake commit')
1490 return commit.hexsha
1491
James E. Blairb8c16472015-05-05 14:55:26 -07001492 def orderedRelease(self):
1493 # Run one build at a time to ensure non-race order:
1494 while len(self.builds):
1495 self.release(self.builds[0])
1496 self.waitUntilSettled()
1497
Clark Boylanb640e052014-04-03 16:41:46 -07001498 def release(self, job):
1499 if isinstance(job, FakeBuild):
1500 job.release()
1501 else:
1502 job.waiting = False
1503 self.log.debug("Queued job %s released" % job.unique)
1504 self.gearman_server.wakeConnections()
1505
1506 def getParameter(self, job, name):
1507 if isinstance(job, FakeBuild):
1508 return job.parameters[name]
1509 else:
1510 parameters = json.loads(job.arguments)
1511 return parameters[name]
1512
Clark Boylanb640e052014-04-03 16:41:46 -07001513 def haveAllBuildsReported(self):
1514 # See if Zuul is waiting on a meta job to complete
James E. Blaire1767bc2016-08-02 10:00:27 -07001515 if self.launch_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07001516 return False
1517 # Find out if every build that the worker has completed has been
1518 # reported back to Zuul. If it hasn't then that means a Gearman
1519 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07001520 for build in self.history:
James E. Blaire1767bc2016-08-02 10:00:27 -07001521 zbuild = self.launch_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07001522 if not zbuild:
1523 # It has already been reported
1524 continue
1525 # It hasn't been reported yet.
1526 return False
1527 # Make sure that none of the worker connections are in GRAB_WAIT
James E. Blaire1767bc2016-08-02 10:00:27 -07001528 for connection in self.launch_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001529 if connection.state == 'GRAB_WAIT':
1530 return False
1531 return True
1532
1533 def areAllBuildsWaiting(self):
James E. Blaire1767bc2016-08-02 10:00:27 -07001534 builds = self.launch_client.builds.values()
Clark Boylanb640e052014-04-03 16:41:46 -07001535 for build in builds:
1536 client_job = None
James E. Blaire1767bc2016-08-02 10:00:27 -07001537 for conn in self.launch_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001538 for j in conn.related_jobs.values():
1539 if j.unique == build.uuid:
1540 client_job = j
1541 break
1542 if not client_job:
1543 self.log.debug("%s is not known to the gearman client" %
1544 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001545 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001546 if not client_job.handle:
1547 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001548 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001549 server_job = self.gearman_server.jobs.get(client_job.handle)
1550 if not server_job:
1551 self.log.debug("%s is not known to the gearman server" %
1552 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001553 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001554 if not hasattr(server_job, 'waiting'):
1555 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001556 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001557 if server_job.waiting:
1558 continue
James E. Blair17302972016-08-10 16:11:42 -07001559 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08001560 self.log.debug("%s has not reported start" % build)
1561 return False
James E. Blairab7132b2016-08-05 12:36:22 -07001562 worker_build = self.launch_server.job_builds.get(server_job.unique)
James E. Blair962220f2016-08-03 11:22:38 -07001563 if worker_build:
1564 if worker_build.isWaiting():
1565 continue
1566 else:
1567 self.log.debug("%s is running" % worker_build)
1568 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001569 else:
James E. Blair962220f2016-08-03 11:22:38 -07001570 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001571 return False
1572 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001573
James E. Blairdce6cea2016-12-20 16:45:32 -08001574 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001575 if self.fake_nodepool.paused:
1576 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08001577 if self.sched.nodepool.requests:
1578 return False
1579 return True
1580
Jan Hruban6b71aff2015-10-22 16:58:08 +02001581 def eventQueuesEmpty(self):
1582 for queue in self.event_queues:
1583 yield queue.empty()
1584
1585 def eventQueuesJoin(self):
1586 for queue in self.event_queues:
1587 queue.join()
1588
Clark Boylanb640e052014-04-03 16:41:46 -07001589 def waitUntilSettled(self):
1590 self.log.debug("Waiting until settled...")
1591 start = time.time()
1592 while True:
James E. Blair71932482017-02-02 11:29:07 -08001593 if time.time() - start > 20:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001594 self.log.error("Timeout waiting for Zuul to settle")
1595 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07001596 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001597 self.log.error(" %s: %s" % (queue, queue.empty()))
1598 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07001599 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001600 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07001601 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001602 self.log.error("All requests completed: %s" %
1603 (self.areAllNodeRequestsComplete(),))
1604 self.log.error("Merge client jobs: %s" %
1605 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07001606 raise Exception("Timeout waiting for Zuul to settle")
1607 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07001608
James E. Blaire1767bc2016-08-02 10:00:27 -07001609 self.launch_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07001610 # have all build states propogated to zuul?
1611 if self.haveAllBuildsReported():
1612 # Join ensures that the queue is empty _and_ events have been
1613 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001614 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001615 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001616 if (not self.merge_client.jobs and
Clark Boylanb640e052014-04-03 16:41:46 -07001617 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08001618 self.areAllBuildsWaiting() and
James E. Blair36c611a2017-02-06 15:59:43 -08001619 self.areAllNodeRequestsComplete() and
1620 all(self.eventQueuesEmpty())):
1621 # The queue empty check is placed at the end to
1622 # ensure that if a component adds an event between
1623 # when locked the run handler and checked that the
1624 # components were stable, we don't erroneously
1625 # report that we are settled.
Clark Boylanb640e052014-04-03 16:41:46 -07001626 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001627 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001628 self.log.debug("...settled.")
1629 return
1630 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001631 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001632 self.sched.wake_event.wait(0.1)
1633
1634 def countJobResults(self, jobs, result):
1635 jobs = filter(lambda x: x.result == result, jobs)
1636 return len(jobs)
1637
James E. Blair96c6bf82016-01-15 16:20:40 -08001638 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07001639 for job in self.history:
1640 if (job.name == name and
1641 (project is None or
1642 job.parameters['ZUUL_PROJECT'] == project)):
1643 return job
Clark Boylanb640e052014-04-03 16:41:46 -07001644 raise Exception("Unable to find job %s in history" % name)
1645
1646 def assertEmptyQueues(self):
1647 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001648 for tenant in self.sched.abide.tenants.values():
1649 for pipeline in tenant.layout.pipelines.values():
1650 for queue in pipeline.queues:
1651 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10001652 print('pipeline %s queue %s contents %s' % (
1653 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08001654 self.assertEqual(len(queue.queue), 0,
1655 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001656
1657 def assertReportedStat(self, key, value=None, kind=None):
1658 start = time.time()
1659 while time.time() < (start + 5):
1660 for stat in self.statsd.stats:
Clark Boylanb640e052014-04-03 16:41:46 -07001661 k, v = stat.split(':')
1662 if key == k:
1663 if value is None and kind is None:
1664 return
1665 elif value:
1666 if value == v:
1667 return
1668 elif kind:
1669 if v.endswith('|' + kind):
1670 return
1671 time.sleep(0.1)
1672
Clark Boylanb640e052014-04-03 16:41:46 -07001673 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001674
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001675 def assertBuilds(self, builds):
1676 """Assert that the running builds are as described.
1677
1678 The list of running builds is examined and must match exactly
1679 the list of builds described by the input.
1680
1681 :arg list builds: A list of dictionaries. Each item in the
1682 list must match the corresponding build in the build
1683 history, and each element of the dictionary must match the
1684 corresponding attribute of the build.
1685
1686 """
James E. Blair3158e282016-08-19 09:34:11 -07001687 try:
1688 self.assertEqual(len(self.builds), len(builds))
1689 for i, d in enumerate(builds):
1690 for k, v in d.items():
1691 self.assertEqual(
1692 getattr(self.builds[i], k), v,
1693 "Element %i in builds does not match" % (i,))
1694 except Exception:
1695 for build in self.builds:
1696 self.log.error("Running build: %s" % build)
1697 else:
1698 self.log.error("No running builds")
1699 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001700
James E. Blairb536ecc2016-08-31 10:11:42 -07001701 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001702 """Assert that the completed builds are as described.
1703
1704 The list of completed builds is examined and must match
1705 exactly the list of builds described by the input.
1706
1707 :arg list history: A list of dictionaries. Each item in the
1708 list must match the corresponding build in the build
1709 history, and each element of the dictionary must match the
1710 corresponding attribute of the build.
1711
James E. Blairb536ecc2016-08-31 10:11:42 -07001712 :arg bool ordered: If true, the history must match the order
1713 supplied, if false, the builds are permitted to have
1714 arrived in any order.
1715
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001716 """
James E. Blairb536ecc2016-08-31 10:11:42 -07001717 def matches(history_item, item):
1718 for k, v in item.items():
1719 if getattr(history_item, k) != v:
1720 return False
1721 return True
James E. Blair3158e282016-08-19 09:34:11 -07001722 try:
1723 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07001724 if ordered:
1725 for i, d in enumerate(history):
1726 if not matches(self.history[i], d):
1727 raise Exception(
1728 "Element %i in history does not match" % (i,))
1729 else:
1730 unseen = self.history[:]
1731 for i, d in enumerate(history):
1732 found = False
1733 for unseen_item in unseen:
1734 if matches(unseen_item, d):
1735 found = True
1736 unseen.remove(unseen_item)
1737 break
1738 if not found:
1739 raise Exception("No match found for element %i "
1740 "in history" % (i,))
1741 if unseen:
1742 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07001743 except Exception:
1744 for build in self.history:
1745 self.log.error("Completed build: %s" % build)
1746 else:
1747 self.log.error("No completed builds")
1748 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001749
James E. Blair6ac368c2016-12-22 18:07:20 -08001750 def printHistory(self):
1751 """Log the build history.
1752
1753 This can be useful during tests to summarize what jobs have
1754 completed.
1755
1756 """
1757 self.log.debug("Build history:")
1758 for build in self.history:
1759 self.log.debug(build)
1760
James E. Blair59fdbac2015-12-07 17:08:06 -08001761 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001762 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1763
1764 def updateConfigLayout(self, path):
1765 root = os.path.join(self.test_root, "config")
Clint Byrumead6c562017-02-01 16:34:04 -08001766 if not os.path.exists(root):
1767 os.makedirs(root)
James E. Blairf84026c2015-12-08 16:11:46 -08001768 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1769 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05001770- tenant:
1771 name: openstack
1772 source:
1773 gerrit:
1774 config-repos:
1775 - %s
1776 """ % path)
James E. Blairf84026c2015-12-08 16:11:46 -08001777 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05001778 self.config.set('zuul', 'tenant_config',
1779 os.path.join(FIXTURE_DIR, f.name))
James E. Blair14abdf42015-12-09 16:11:53 -08001780
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001781 def addCommitToRepo(self, project, message, files,
1782 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08001783 path = os.path.join(self.upstream_root, project)
1784 repo = git.Repo(path)
1785 repo.head.reference = branch
1786 zuul.merger.merger.reset_repo_to_head(repo)
1787 for fn, content in files.items():
1788 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08001789 try:
1790 os.makedirs(os.path.dirname(fn))
1791 except OSError:
1792 pass
James E. Blair14abdf42015-12-09 16:11:53 -08001793 with open(fn, 'w') as f:
1794 f.write(content)
1795 repo.index.add([fn])
1796 commit = repo.index.commit(message)
1797 repo.heads[branch].commit = commit
1798 repo.head.reference = branch
1799 repo.git.clean('-x', '-f', '-d')
1800 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001801 if tag:
1802 repo.create_tag(tag)
James E. Blair3f876d52016-07-22 13:07:14 -07001803
James E. Blair7fc8daa2016-08-08 15:37:15 -07001804 def addEvent(self, connection, event):
1805 """Inject a Fake (Gerrit) event.
1806
1807 This method accepts a JSON-encoded event and simulates Zuul
1808 having received it from Gerrit. It could (and should)
1809 eventually apply to any connection type, but is currently only
1810 used with Gerrit connections. The name of the connection is
1811 used to look up the corresponding server, and the event is
1812 simulated as having been received by all Zuul connections
1813 attached to that server. So if two Gerrit connections in Zuul
1814 are connected to the same Gerrit server, and you invoke this
1815 method specifying the name of one of them, the event will be
1816 received by both.
1817
1818 .. note::
1819
1820 "self.fake_gerrit.addEvent" calls should be migrated to
1821 this method.
1822
1823 :arg str connection: The name of the connection corresponding
1824 to the gerrit server.
1825 :arg str event: The JSON-encoded event.
1826
1827 """
1828 specified_conn = self.connections.connections[connection]
1829 for conn in self.connections.connections.values():
1830 if (isinstance(conn, specified_conn.__class__) and
1831 specified_conn.server == conn.server):
1832 conn.addEvent(event)
1833
James E. Blair3f876d52016-07-22 13:07:14 -07001834
1835class AnsibleZuulTestCase(ZuulTestCase):
1836 """ZuulTestCase but with an actual ansible launcher running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07001837 run_ansible = True