blob: 566ceae2cd96037b8d198fd11196129d0394d3a9 [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070019import gc
20import hashlib
21import json
22import logging
23import os
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070025from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070026import random
27import re
28import select
29import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030030from six.moves import reload_module
James E. Blair1c236df2017-02-01 14:07:24 -080031from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070032import socket
33import string
34import subprocess
35import swiftclient
James E. Blair1c236df2017-02-01 14:07:24 -080036import sys
James E. Blairf84026c2015-12-08 16:11:46 -080037import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070038import threading
39import time
Clark Boylanb640e052014-04-03 16:41:46 -070040
41import git
42import gear
43import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080044import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080045import kazoo.exceptions
Clark Boylanb640e052014-04-03 16:41:46 -070046import statsd
47import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080048import testtools.content
49import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080050from git.exc import NoSuchPathError
Clark Boylanb640e052014-04-03 16:41:46 -070051
James E. Blaire511d2f2016-12-08 15:22:26 -080052import zuul.driver.gerrit.gerritsource as gerritsource
53import zuul.driver.gerrit.gerritconnection as gerritconnection
Clark Boylanb640e052014-04-03 16:41:46 -070054import zuul.scheduler
55import zuul.webapp
56import zuul.rpclistener
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +100057import zuul.launcher.server
58import zuul.launcher.client
Clark Boylanb640e052014-04-03 16:41:46 -070059import zuul.lib.swift
James E. Blair83005782015-12-11 14:46:03 -080060import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070061import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070062import zuul.merger.merger
63import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070064import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080065import zuul.zk
Clark Boylanb640e052014-04-03 16:41:46 -070066
67FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
68 'fixtures')
James E. Blair97d902e2014-08-21 13:25:56 -070069USE_TEMPDIR = True
Clark Boylanb640e052014-04-03 16:41:46 -070070
Clark Boylanb640e052014-04-03 16:41:46 -070071
72def repack_repo(path):
73 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
74 output = subprocess.Popen(cmd, close_fds=True,
75 stdout=subprocess.PIPE,
76 stderr=subprocess.PIPE)
77 out = output.communicate()
78 if output.returncode:
79 raise Exception("git repack returned %d" % output.returncode)
80 return out
81
82
83def random_sha1():
84 return hashlib.sha1(str(random.random())).hexdigest()
85
86
James E. Blaira190f3b2015-01-05 14:56:54 -080087def iterate_timeout(max_seconds, purpose):
88 start = time.time()
89 count = 0
90 while (time.time() < start + max_seconds):
91 count += 1
92 yield count
93 time.sleep(0)
94 raise Exception("Timeout waiting for %s" % purpose)
95
96
Clark Boylanb640e052014-04-03 16:41:46 -070097class ChangeReference(git.Reference):
98 _common_path_default = "refs/changes"
99 _points_to_commits_only = True
100
101
102class FakeChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700103 categories = {'approved': ('Approved', -1, 1),
104 'code-review': ('Code-Review', -2, 2),
105 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700106
107 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700108 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700109 self.gerrit = gerrit
110 self.reported = 0
111 self.queried = 0
112 self.patchsets = []
113 self.number = number
114 self.project = project
115 self.branch = branch
116 self.subject = subject
117 self.latest_patchset = 0
118 self.depends_on_change = None
119 self.needed_by_changes = []
120 self.fail_merge = False
121 self.messages = []
122 self.data = {
123 'branch': branch,
124 'comments': [],
125 'commitMessage': subject,
126 'createdOn': time.time(),
127 'id': 'I' + random_sha1(),
128 'lastUpdated': time.time(),
129 'number': str(number),
130 'open': status == 'NEW',
131 'owner': {'email': 'user@example.com',
132 'name': 'User Name',
133 'username': 'username'},
134 'patchSets': self.patchsets,
135 'project': project,
136 'status': status,
137 'subject': subject,
138 'submitRecords': [],
139 'url': 'https://hostname/%s' % number}
140
141 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700142 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700143 self.data['submitRecords'] = self.getSubmitRecords()
144 self.open = status == 'NEW'
145
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700146 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700147 path = os.path.join(self.upstream_root, self.project)
148 repo = git.Repo(path)
149 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
150 self.latest_patchset),
151 'refs/tags/init')
152 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700153 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700154 repo.git.clean('-x', '-f', '-d')
155
156 path = os.path.join(self.upstream_root, self.project)
157 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700158 for fn, content in files.items():
159 fn = os.path.join(path, fn)
160 with open(fn, 'w') as f:
161 f.write(content)
162 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700163 else:
164 for fni in range(100):
165 fn = os.path.join(path, str(fni))
166 f = open(fn, 'w')
167 for ci in range(4096):
168 f.write(random.choice(string.printable))
169 f.close()
170 repo.index.add([fn])
171
172 r = repo.index.commit(msg)
173 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700174 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700175 repo.git.clean('-x', '-f', '-d')
176 repo.heads['master'].checkout()
177 return r
178
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700179 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700180 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700181 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700182 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700183 data = ("test %s %s %s\n" %
184 (self.branch, self.number, self.latest_patchset))
185 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700186 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700187 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700188 ps_files = [{'file': '/COMMIT_MSG',
189 'type': 'ADDED'},
190 {'file': 'README',
191 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700192 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700193 ps_files.append({'file': f, 'type': 'ADDED'})
194 d = {'approvals': [],
195 'createdOn': time.time(),
196 'files': ps_files,
197 'number': str(self.latest_patchset),
198 'ref': 'refs/changes/1/%s/%s' % (self.number,
199 self.latest_patchset),
200 'revision': c.hexsha,
201 'uploader': {'email': 'user@example.com',
202 'name': 'User name',
203 'username': 'user'}}
204 self.data['currentPatchSet'] = d
205 self.patchsets.append(d)
206 self.data['submitRecords'] = self.getSubmitRecords()
207
208 def getPatchsetCreatedEvent(self, patchset):
209 event = {"type": "patchset-created",
210 "change": {"project": self.project,
211 "branch": self.branch,
212 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
213 "number": str(self.number),
214 "subject": self.subject,
215 "owner": {"name": "User Name"},
216 "url": "https://hostname/3"},
217 "patchSet": self.patchsets[patchset - 1],
218 "uploader": {"name": "User Name"}}
219 return event
220
221 def getChangeRestoredEvent(self):
222 event = {"type": "change-restored",
223 "change": {"project": self.project,
224 "branch": self.branch,
225 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
226 "number": str(self.number),
227 "subject": self.subject,
228 "owner": {"name": "User Name"},
229 "url": "https://hostname/3"},
230 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100231 "patchSet": self.patchsets[-1],
232 "reason": ""}
233 return event
234
235 def getChangeAbandonedEvent(self):
236 event = {"type": "change-abandoned",
237 "change": {"project": self.project,
238 "branch": self.branch,
239 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
240 "number": str(self.number),
241 "subject": self.subject,
242 "owner": {"name": "User Name"},
243 "url": "https://hostname/3"},
244 "abandoner": {"name": "User Name"},
245 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700246 "reason": ""}
247 return event
248
249 def getChangeCommentEvent(self, patchset):
250 event = {"type": "comment-added",
251 "change": {"project": self.project,
252 "branch": self.branch,
253 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
254 "number": str(self.number),
255 "subject": self.subject,
256 "owner": {"name": "User Name"},
257 "url": "https://hostname/3"},
258 "patchSet": self.patchsets[patchset - 1],
259 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700260 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700261 "description": "Code-Review",
262 "value": "0"}],
263 "comment": "This is a comment"}
264 return event
265
Joshua Hesketh642824b2014-07-01 17:54:59 +1000266 def addApproval(self, category, value, username='reviewer_john',
267 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700268 if not granted_on:
269 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000270 approval = {
271 'description': self.categories[category][0],
272 'type': category,
273 'value': str(value),
274 'by': {
275 'username': username,
276 'email': username + '@example.com',
277 },
278 'grantedOn': int(granted_on)
279 }
Clark Boylanb640e052014-04-03 16:41:46 -0700280 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
281 if x['by']['username'] == username and x['type'] == category:
282 del self.patchsets[-1]['approvals'][i]
283 self.patchsets[-1]['approvals'].append(approval)
284 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000285 'author': {'email': 'author@example.com',
286 'name': 'Patchset Author',
287 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700288 'change': {'branch': self.branch,
289 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
290 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000291 'owner': {'email': 'owner@example.com',
292 'name': 'Change Owner',
293 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700294 'project': self.project,
295 'subject': self.subject,
296 'topic': 'master',
297 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000298 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700299 'patchSet': self.patchsets[-1],
300 'type': 'comment-added'}
301 self.data['submitRecords'] = self.getSubmitRecords()
302 return json.loads(json.dumps(event))
303
304 def getSubmitRecords(self):
305 status = {}
306 for cat in self.categories.keys():
307 status[cat] = 0
308
309 for a in self.patchsets[-1]['approvals']:
310 cur = status[a['type']]
311 cat_min, cat_max = self.categories[a['type']][1:]
312 new = int(a['value'])
313 if new == cat_min:
314 cur = new
315 elif abs(new) > abs(cur):
316 cur = new
317 status[a['type']] = cur
318
319 labels = []
320 ok = True
321 for typ, cat in self.categories.items():
322 cur = status[typ]
323 cat_min, cat_max = cat[1:]
324 if cur == cat_min:
325 value = 'REJECT'
326 ok = False
327 elif cur == cat_max:
328 value = 'OK'
329 else:
330 value = 'NEED'
331 ok = False
332 labels.append({'label': cat[0], 'status': value})
333 if ok:
334 return [{'status': 'OK'}]
335 return [{'status': 'NOT_READY',
336 'labels': labels}]
337
338 def setDependsOn(self, other, patchset):
339 self.depends_on_change = other
340 d = {'id': other.data['id'],
341 'number': other.data['number'],
342 'ref': other.patchsets[patchset - 1]['ref']
343 }
344 self.data['dependsOn'] = [d]
345
346 other.needed_by_changes.append(self)
347 needed = other.data.get('neededBy', [])
348 d = {'id': self.data['id'],
349 'number': self.data['number'],
350 'ref': self.patchsets[patchset - 1]['ref'],
351 'revision': self.patchsets[patchset - 1]['revision']
352 }
353 needed.append(d)
354 other.data['neededBy'] = needed
355
356 def query(self):
357 self.queried += 1
358 d = self.data.get('dependsOn')
359 if d:
360 d = d[0]
361 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
362 d['isCurrentPatchSet'] = True
363 else:
364 d['isCurrentPatchSet'] = False
365 return json.loads(json.dumps(self.data))
366
367 def setMerged(self):
368 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000369 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700370 return
371 if self.fail_merge:
372 return
373 self.data['status'] = 'MERGED'
374 self.open = False
375
376 path = os.path.join(self.upstream_root, self.project)
377 repo = git.Repo(path)
378 repo.heads[self.branch].commit = \
379 repo.commit(self.patchsets[-1]['revision'])
380
381 def setReported(self):
382 self.reported += 1
383
384
James E. Blaire511d2f2016-12-08 15:22:26 -0800385class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700386 """A Fake Gerrit connection for use in tests.
387
388 This subclasses
389 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
390 ability for tests to add changes to the fake Gerrit it represents.
391 """
392
Joshua Hesketh352264b2015-08-11 23:42:08 +1000393 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700394
James E. Blaire511d2f2016-12-08 15:22:26 -0800395 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700396 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800397 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000398 connection_config)
399
James E. Blair7fc8daa2016-08-08 15:37:15 -0700400 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700401 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
402 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000403 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700404 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200405 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700406
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700407 def addFakeChange(self, project, branch, subject, status='NEW',
408 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700409 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700410 self.change_number += 1
411 c = FakeChange(self, self.change_number, project, branch, subject,
412 upstream_root=self.upstream_root,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700413 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700414 self.changes[self.change_number] = c
415 return c
416
Clark Boylanb640e052014-04-03 16:41:46 -0700417 def review(self, project, changeid, message, action):
418 number, ps = changeid.split(',')
419 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000420
421 # Add the approval back onto the change (ie simulate what gerrit would
422 # do).
423 # Usually when zuul leaves a review it'll create a feedback loop where
424 # zuul's review enters another gerrit event (which is then picked up by
425 # zuul). However, we can't mimic this behaviour (by adding this
426 # approval event into the queue) as it stops jobs from checking what
427 # happens before this event is triggered. If a job needs to see what
428 # happens they can add their own verified event into the queue.
429 # Nevertheless, we can update change with the new review in gerrit.
430
James E. Blair8b5408c2016-08-08 15:37:46 -0700431 for cat in action.keys():
432 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000433 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000434
James E. Blair8b5408c2016-08-08 15:37:46 -0700435 # TODOv3(jeblair): can this be removed?
Joshua Hesketh642824b2014-07-01 17:54:59 +1000436 if 'label' in action:
437 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000438 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000439
Clark Boylanb640e052014-04-03 16:41:46 -0700440 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000441
Clark Boylanb640e052014-04-03 16:41:46 -0700442 if 'submit' in action:
443 change.setMerged()
444 if message:
445 change.setReported()
446
447 def query(self, number):
448 change = self.changes.get(int(number))
449 if change:
450 return change.query()
451 return {}
452
James E. Blairc494d542014-08-06 09:23:52 -0700453 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700454 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700455 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800456 if query.startswith('change:'):
457 # Query a specific changeid
458 changeid = query[len('change:'):]
459 l = [change.query() for change in self.changes.values()
460 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700461 elif query.startswith('message:'):
462 # Query the content of a commit message
463 msg = query[len('message:'):].strip()
464 l = [change.query() for change in self.changes.values()
465 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800466 else:
467 # Query all open changes
468 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700469 return l
James E. Blairc494d542014-08-06 09:23:52 -0700470
Joshua Hesketh352264b2015-08-11 23:42:08 +1000471 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700472 pass
473
Joshua Hesketh352264b2015-08-11 23:42:08 +1000474 def getGitUrl(self, project):
475 return os.path.join(self.upstream_root, project.name)
476
Adam Gandelmanc5e4f1d2016-11-29 14:27:17 -0800477 def _getGitwebUrl(self, project, sha=None):
478 return self.getGitwebUrl(project, sha)
479
Clark Boylanb640e052014-04-03 16:41:46 -0700480
481class BuildHistory(object):
482 def __init__(self, **kw):
483 self.__dict__.update(kw)
484
485 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -0700486 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
487 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -0700488
489
490class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200491 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700492 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700493 self.url = url
494
495 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -0700496 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -0700497 path = res.path
498 project = '/'.join(path.split('/')[2:-2])
499 ret = '001e# service=git-upload-pack\n'
500 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
501 'multi_ack thin-pack side-band side-band-64k ofs-delta '
502 'shallow no-progress include-tag multi_ack_detailed no-done\n')
503 path = os.path.join(self.upstream_root, project)
504 repo = git.Repo(path)
505 for ref in repo.refs:
506 r = ref.object.hexsha + ' ' + ref.path + '\n'
507 ret += '%04x%s' % (len(r) + 4, r)
508 ret += '0000'
509 return ret
510
511
Clark Boylanb640e052014-04-03 16:41:46 -0700512class FakeStatsd(threading.Thread):
513 def __init__(self):
514 threading.Thread.__init__(self)
515 self.daemon = True
516 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
517 self.sock.bind(('', 0))
518 self.port = self.sock.getsockname()[1]
519 self.wake_read, self.wake_write = os.pipe()
520 self.stats = []
521
522 def run(self):
523 while True:
524 poll = select.poll()
525 poll.register(self.sock, select.POLLIN)
526 poll.register(self.wake_read, select.POLLIN)
527 ret = poll.poll()
528 for (fd, event) in ret:
529 if fd == self.sock.fileno():
530 data = self.sock.recvfrom(1024)
531 if not data:
532 return
533 self.stats.append(data[0])
534 if fd == self.wake_read:
535 return
536
537 def stop(self):
538 os.write(self.wake_write, '1\n')
539
540
James E. Blaire1767bc2016-08-02 10:00:27 -0700541class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -0700542 log = logging.getLogger("zuul.test")
543
James E. Blair34776ee2016-08-25 13:53:54 -0700544 def __init__(self, launch_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -0700545 self.daemon = True
James E. Blaire1767bc2016-08-02 10:00:27 -0700546 self.launch_server = launch_server
Clark Boylanb640e052014-04-03 16:41:46 -0700547 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -0700548 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -0700549 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -0700550 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -0700551 # TODOv3(jeblair): self.node is really "the image of the node
552 # assigned". We should rename it (self.node_image?) if we
553 # keep using it like this, or we may end up exposing more of
554 # the complexity around multi-node jobs here
555 # (self.nodes[0].image?)
556 self.node = None
557 if len(self.parameters.get('nodes')) == 1:
558 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -0700559 self.unique = self.parameters['ZUUL_UUID']
James E. Blair3f876d52016-07-22 13:07:14 -0700560 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -0700561 self.wait_condition = threading.Condition()
562 self.waiting = False
563 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -0500564 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -0700565 self.created = time.time()
James E. Blaire1767bc2016-08-02 10:00:27 -0700566 self.changes = None
567 if 'ZUUL_CHANGE_IDS' in self.parameters:
568 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -0700569
James E. Blair3158e282016-08-19 09:34:11 -0700570 def __repr__(self):
571 waiting = ''
572 if self.waiting:
573 waiting = ' [waiting]'
574 return '<FakeBuild %s %s%s>' % (self.name, self.changes, waiting)
575
Clark Boylanb640e052014-04-03 16:41:46 -0700576 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700577 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -0700578 self.wait_condition.acquire()
579 self.wait_condition.notify()
580 self.waiting = False
581 self.log.debug("Build %s released" % self.unique)
582 self.wait_condition.release()
583
584 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700585 """Return whether this build is being held.
586
587 :returns: Whether the build is being held.
588 :rtype: bool
589 """
590
Clark Boylanb640e052014-04-03 16:41:46 -0700591 self.wait_condition.acquire()
592 if self.waiting:
593 ret = True
594 else:
595 ret = False
596 self.wait_condition.release()
597 return ret
598
599 def _wait(self):
600 self.wait_condition.acquire()
601 self.waiting = True
602 self.log.debug("Build %s waiting" % self.unique)
603 self.wait_condition.wait()
604 self.wait_condition.release()
605
606 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -0700607 self.log.debug('Running build %s' % self.unique)
608
James E. Blaire1767bc2016-08-02 10:00:27 -0700609 if self.launch_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -0700610 self.log.debug('Holding build %s' % self.unique)
611 self._wait()
612 self.log.debug("Build %s continuing" % self.unique)
613
James E. Blair412fba82017-01-26 15:00:50 -0800614 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
James E. Blaira5dba232016-08-08 15:53:24 -0700615 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
James E. Blair412fba82017-01-26 15:00:50 -0800616 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
Clark Boylanb640e052014-04-03 16:41:46 -0700617 if self.aborted:
James E. Blair412fba82017-01-26 15:00:50 -0800618 result = (RecordingAnsibleJob.RESULT_ABORTED, None)
Paul Belanger71d98172016-11-08 10:56:31 -0500619 if self.requeue:
James E. Blair412fba82017-01-26 15:00:50 -0800620 result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
Clark Boylanb640e052014-04-03 16:41:46 -0700621
James E. Blaire1767bc2016-08-02 10:00:27 -0700622 return result
Clark Boylanb640e052014-04-03 16:41:46 -0700623
James E. Blaira5dba232016-08-08 15:53:24 -0700624 def shouldFail(self):
625 changes = self.launch_server.fail_tests.get(self.name, [])
626 for change in changes:
627 if self.hasChanges(change):
628 return True
629 return False
630
James E. Blaire7b99a02016-08-05 14:27:34 -0700631 def hasChanges(self, *changes):
632 """Return whether this build has certain changes in its git repos.
633
634 :arg FakeChange changes: One or more changes (varargs) that
635 are expected to be present (in order) in the git repository of
636 the active project.
637
638 :returns: Whether the build has the indicated changes.
639 :rtype: bool
640
641 """
Clint Byrum3343e3e2016-11-15 16:05:03 -0800642 for change in changes:
643 path = os.path.join(self.jobdir.git_root, change.project)
644 try:
645 repo = git.Repo(path)
646 except NoSuchPathError as e:
647 self.log.debug('%s' % e)
648 return False
649 ref = self.parameters['ZUUL_REF']
650 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
651 commit_message = '%s-1' % change.subject
652 self.log.debug("Checking if build %s has changes; commit_message "
653 "%s; repo_messages %s" % (self, commit_message,
654 repo_messages))
655 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -0700656 self.log.debug(" messages do not match")
657 return False
658 self.log.debug(" OK")
659 return True
660
Clark Boylanb640e052014-04-03 16:41:46 -0700661
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +1000662class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
James E. Blaire7b99a02016-08-05 14:27:34 -0700663 """An Ansible launcher to be used in tests.
664
665 :ivar bool hold_jobs_in_build: If true, when jobs are launched
666 they will report that they have started but then pause until
667 released before reporting completion. This attribute may be
668 changed at any time and will take effect for subsequently
669 launched builds, but previously held builds will still need to
670 be explicitly released.
671
672 """
James E. Blairf5dbd002015-12-23 15:26:17 -0800673 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -0700674 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -0800675 self._test_root = kw.pop('_test_root', False)
James E. Blairf5dbd002015-12-23 15:26:17 -0800676 super(RecordingLaunchServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -0700677 self.hold_jobs_in_build = False
678 self.lock = threading.Lock()
679 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -0700680 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -0700681 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -0700682 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -0800683
James E. Blaira5dba232016-08-08 15:53:24 -0700684 def failJob(self, name, change):
James E. Blaire7b99a02016-08-05 14:27:34 -0700685 """Instruct the launcher to report matching builds as failures.
686
687 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -0700688 :arg Change change: The :py:class:`~tests.base.FakeChange`
689 instance which should cause the job to fail. This job
690 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -0700691
692 """
James E. Blaire1767bc2016-08-02 10:00:27 -0700693 l = self.fail_tests.get(name, [])
694 l.append(change)
695 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -0800696
James E. Blair962220f2016-08-03 11:22:38 -0700697 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700698 """Release a held build.
699
700 :arg str regex: A regular expression which, if supplied, will
701 cause only builds with matching names to be released. If
702 not supplied, all builds will be released.
703
704 """
James E. Blair962220f2016-08-03 11:22:38 -0700705 builds = self.running_builds[:]
706 self.log.debug("Releasing build %s (%s)" % (regex,
707 len(self.running_builds)))
708 for build in builds:
709 if not regex or re.match(regex, build.name):
710 self.log.debug("Releasing build %s" %
711 (build.parameters['ZUUL_UUID']))
712 build.release()
713 else:
714 self.log.debug("Not releasing build %s" %
715 (build.parameters['ZUUL_UUID']))
716 self.log.debug("Done releasing builds %s (%s)" %
717 (regex, len(self.running_builds)))
718
James E. Blair17302972016-08-10 16:11:42 -0700719 def launchJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -0700720 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -0700721 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -0700722 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -0700723 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -0800724 args = json.loads(job.arguments)
725 args['zuul']['_test'] = dict(test_root=self._test_root)
726 job.arguments = json.dumps(args)
Joshua Hesketh50c21782016-10-13 21:34:14 +1100727 self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
728 self.job_workers[job.unique].run()
James E. Blair17302972016-08-10 16:11:42 -0700729
730 def stopJob(self, job):
731 self.log.debug("handle stop")
732 parameters = json.loads(job.arguments)
733 uuid = parameters['uuid']
734 for build in self.running_builds:
735 if build.unique == uuid:
736 build.aborted = True
737 build.release()
738 super(RecordingLaunchServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -0700739
Joshua Hesketh50c21782016-10-13 21:34:14 +1100740
741class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob):
James E. Blair412fba82017-01-26 15:00:50 -0800742 def runPlaybooks(self):
Joshua Hesketh50c21782016-10-13 21:34:14 +1100743 build = self.launcher_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -0800744 build.jobdir = self.jobdir
James E. Blaire1767bc2016-08-02 10:00:27 -0700745
James E. Blair412fba82017-01-26 15:00:50 -0800746 result = super(RecordingAnsibleJob, self).runPlaybooks()
747
Joshua Hesketh50c21782016-10-13 21:34:14 +1100748 self.launcher_server.lock.acquire()
749 self.launcher_server.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -0700750 BuildHistory(name=build.name, result=result, changes=build.changes,
751 node=build.node, uuid=build.unique,
752 parameters=build.parameters,
James E. Blaire1767bc2016-08-02 10:00:27 -0700753 pipeline=build.parameters['ZUUL_PIPELINE'])
754 )
Joshua Hesketh50c21782016-10-13 21:34:14 +1100755 self.launcher_server.running_builds.remove(build)
756 del self.launcher_server.job_builds[self.job.unique]
757 self.launcher_server.lock.release()
James E. Blair412fba82017-01-26 15:00:50 -0800758 return result
759
760 def runAnsible(self, cmd, timeout):
761 build = self.launcher_server.job_builds[self.job.unique]
762
763 if self.launcher_server._run_ansible:
764 result = super(RecordingAnsibleJob, self).runAnsible(cmd, timeout)
765 else:
766 result = build.run()
James E. Blaire1767bc2016-08-02 10:00:27 -0700767 return result
James E. Blairf5dbd002015-12-23 15:26:17 -0800768
769
Clark Boylanb640e052014-04-03 16:41:46 -0700770class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -0700771 """A Gearman server for use in tests.
772
773 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
774 added to the queue but will not be distributed to workers
775 until released. This attribute may be changed at any time and
776 will take effect for subsequently enqueued jobs, but
777 previously held jobs will still need to be explicitly
778 released.
779
780 """
781
Clark Boylanb640e052014-04-03 16:41:46 -0700782 def __init__(self):
783 self.hold_jobs_in_queue = False
784 super(FakeGearmanServer, self).__init__(0)
785
786 def getJobForConnection(self, connection, peek=False):
787 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
788 for job in queue:
789 if not hasattr(job, 'waiting'):
Paul Belanger6ab6af72016-11-06 11:32:59 -0500790 if job.name.startswith('launcher:launch'):
Clark Boylanb640e052014-04-03 16:41:46 -0700791 job.waiting = self.hold_jobs_in_queue
792 else:
793 job.waiting = False
794 if job.waiting:
795 continue
796 if job.name in connection.functions:
797 if not peek:
798 queue.remove(job)
799 connection.related_jobs[job.handle] = job
800 job.worker_connection = connection
801 job.running = True
802 return job
803 return None
804
805 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700806 """Release a held job.
807
808 :arg str regex: A regular expression which, if supplied, will
809 cause only jobs with matching names to be released. If
810 not supplied, all jobs will be released.
811 """
Clark Boylanb640e052014-04-03 16:41:46 -0700812 released = False
813 qlen = (len(self.high_queue) + len(self.normal_queue) +
814 len(self.low_queue))
815 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
816 for job in self.getQueue():
Paul Belanger6ab6af72016-11-06 11:32:59 -0500817 if job.name != 'launcher:launch':
Clark Boylanb640e052014-04-03 16:41:46 -0700818 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -0500819 parameters = json.loads(job.arguments)
820 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -0700821 self.log.debug("releasing queued job %s" %
822 job.unique)
823 job.waiting = False
824 released = True
825 else:
826 self.log.debug("not releasing queued job %s" %
827 job.unique)
828 if released:
829 self.wakeConnections()
830 qlen = (len(self.high_queue) + len(self.normal_queue) +
831 len(self.low_queue))
832 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
833
834
835class FakeSMTP(object):
836 log = logging.getLogger('zuul.FakeSMTP')
837
838 def __init__(self, messages, server, port):
839 self.server = server
840 self.port = port
841 self.messages = messages
842
843 def sendmail(self, from_email, to_email, msg):
844 self.log.info("Sending email from %s, to %s, with msg %s" % (
845 from_email, to_email, msg))
846
847 headers = msg.split('\n\n', 1)[0]
848 body = msg.split('\n\n', 1)[1]
849
850 self.messages.append(dict(
851 from_email=from_email,
852 to_email=to_email,
853 msg=msg,
854 headers=headers,
855 body=body,
856 ))
857
858 return True
859
860 def quit(self):
861 return True
862
863
864class FakeSwiftClientConnection(swiftclient.client.Connection):
865 def post_account(self, headers):
866 # Do nothing
867 pass
868
869 def get_auth(self):
870 # Returns endpoint and (unused) auth token
871 endpoint = os.path.join('https://storage.example.org', 'V1',
872 'AUTH_account')
873 return endpoint, ''
874
875
James E. Blairdce6cea2016-12-20 16:45:32 -0800876class FakeNodepool(object):
877 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -0800878 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -0800879
880 log = logging.getLogger("zuul.test.FakeNodepool")
881
882 def __init__(self, host, port, chroot):
883 self.client = kazoo.client.KazooClient(
884 hosts='%s:%s%s' % (host, port, chroot))
885 self.client.start()
886 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -0800887 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -0800888 self.thread = threading.Thread(target=self.run)
889 self.thread.daemon = True
890 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -0800891 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -0800892
893 def stop(self):
894 self._running = False
895 self.thread.join()
896 self.client.stop()
897 self.client.close()
898
899 def run(self):
900 while self._running:
901 self._run()
902 time.sleep(0.1)
903
904 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -0800905 if self.paused:
906 return
James E. Blairdce6cea2016-12-20 16:45:32 -0800907 for req in self.getNodeRequests():
908 self.fulfillRequest(req)
909
910 def getNodeRequests(self):
911 try:
912 reqids = self.client.get_children(self.REQUEST_ROOT)
913 except kazoo.exceptions.NoNodeError:
914 return []
915 reqs = []
916 for oid in sorted(reqids):
917 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -0800918 try:
919 data, stat = self.client.get(path)
920 data = json.loads(data)
921 data['_oid'] = oid
922 reqs.append(data)
923 except kazoo.exceptions.NoNodeError:
924 pass
James E. Blairdce6cea2016-12-20 16:45:32 -0800925 return reqs
926
James E. Blaire18d4602017-01-05 11:17:28 -0800927 def getNodes(self):
928 try:
929 nodeids = self.client.get_children(self.NODE_ROOT)
930 except kazoo.exceptions.NoNodeError:
931 return []
932 nodes = []
933 for oid in sorted(nodeids):
934 path = self.NODE_ROOT + '/' + oid
935 data, stat = self.client.get(path)
936 data = json.loads(data)
937 data['_oid'] = oid
938 try:
939 lockfiles = self.client.get_children(path + '/lock')
940 except kazoo.exceptions.NoNodeError:
941 lockfiles = []
942 if lockfiles:
943 data['_lock'] = True
944 else:
945 data['_lock'] = False
946 nodes.append(data)
947 return nodes
948
James E. Blaira38c28e2017-01-04 10:33:20 -0800949 def makeNode(self, request_id, node_type):
950 now = time.time()
951 path = '/nodepool/nodes/'
952 data = dict(type=node_type,
953 provider='test-provider',
954 region='test-region',
955 az=None,
956 public_ipv4='127.0.0.1',
957 private_ipv4=None,
958 public_ipv6=None,
959 allocated_to=request_id,
960 state='ready',
961 state_time=now,
962 created_time=now,
963 updated_time=now,
964 image_id=None,
965 launcher='fake-nodepool')
966 data = json.dumps(data)
967 path = self.client.create(path, data,
968 makepath=True,
969 sequence=True)
970 nodeid = path.split("/")[-1]
971 return nodeid
972
James E. Blair6ab79e02017-01-06 10:10:17 -0800973 def addFailRequest(self, request):
974 self.fail_requests.add(request['_oid'])
975
James E. Blairdce6cea2016-12-20 16:45:32 -0800976 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -0800977 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -0800978 return
979 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -0800980 oid = request['_oid']
981 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -0800982
James E. Blair6ab79e02017-01-06 10:10:17 -0800983 if oid in self.fail_requests:
984 request['state'] = 'failed'
985 else:
986 request['state'] = 'fulfilled'
987 nodes = []
988 for node in request['node_types']:
989 nodeid = self.makeNode(oid, node)
990 nodes.append(nodeid)
991 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -0800992
James E. Blaira38c28e2017-01-04 10:33:20 -0800993 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -0800994 path = self.REQUEST_ROOT + '/' + oid
995 data = json.dumps(request)
996 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
997 self.client.set(path, data)
998
999
James E. Blair498059b2016-12-20 13:50:13 -08001000class ChrootedKazooFixture(fixtures.Fixture):
1001 def __init__(self):
1002 super(ChrootedKazooFixture, self).__init__()
1003
1004 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1005 if ':' in zk_host:
1006 host, port = zk_host.split(':')
1007 else:
1008 host = zk_host
1009 port = None
1010
1011 self.zookeeper_host = host
1012
1013 if not port:
1014 self.zookeeper_port = 2181
1015 else:
1016 self.zookeeper_port = int(port)
1017
1018 def _setUp(self):
1019 # Make sure the test chroot paths do not conflict
1020 random_bits = ''.join(random.choice(string.ascii_lowercase +
1021 string.ascii_uppercase)
1022 for x in range(8))
1023
1024 rand_test_path = '%s_%s' % (random_bits, os.getpid())
1025 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1026
1027 # Ensure the chroot path exists and clean up any pre-existing znodes.
1028 _tmp_client = kazoo.client.KazooClient(
1029 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1030 _tmp_client.start()
1031
1032 if _tmp_client.exists(self.zookeeper_chroot):
1033 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1034
1035 _tmp_client.ensure_path(self.zookeeper_chroot)
1036 _tmp_client.stop()
1037 _tmp_client.close()
1038
1039 self.addCleanup(self._cleanup)
1040
1041 def _cleanup(self):
1042 '''Remove the chroot path.'''
1043 # Need a non-chroot'ed client to remove the chroot path
1044 _tmp_client = kazoo.client.KazooClient(
1045 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1046 _tmp_client.start()
1047 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1048 _tmp_client.stop()
1049
1050
Maru Newby3fe5f852015-01-13 04:22:14 +00001051class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001052 log = logging.getLogger("zuul.test")
1053
James E. Blair1c236df2017-02-01 14:07:24 -08001054 def attachLogs(self, *args):
1055 def reader():
1056 self._log_stream.seek(0)
1057 while True:
1058 x = self._log_stream.read(4096)
1059 if not x:
1060 break
1061 yield x.encode('utf8')
1062 content = testtools.content.content_from_reader(
1063 reader,
1064 testtools.content_type.UTF8_TEXT,
1065 False)
1066 self.addDetail('logging', content)
1067
Clark Boylanb640e052014-04-03 16:41:46 -07001068 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001069 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001070 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1071 try:
1072 test_timeout = int(test_timeout)
1073 except ValueError:
1074 # If timeout value is invalid do not set a timeout.
1075 test_timeout = 0
1076 if test_timeout > 0:
1077 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1078
1079 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1080 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1081 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1082 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1083 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1084 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1085 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1086 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1087 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1088 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001089 self._log_stream = StringIO()
1090 self.addOnException(self.attachLogs)
1091 else:
1092 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001093
James E. Blair1c236df2017-02-01 14:07:24 -08001094 handler = logging.StreamHandler(self._log_stream)
1095 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1096 '%(levelname)-8s %(message)s')
1097 handler.setFormatter(formatter)
1098
1099 logger = logging.getLogger()
1100 logger.setLevel(logging.DEBUG)
1101 logger.addHandler(handler)
1102
1103 # NOTE(notmorgan): Extract logging overrides for specific
1104 # libraries from the OS_LOG_DEFAULTS env and create loggers
1105 # for each. This is used to limit the output during test runs
1106 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001107 log_defaults_from_env = os.environ.get(
1108 'OS_LOG_DEFAULTS',
James E. Blair1c236df2017-02-01 14:07:24 -08001109 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001110
James E. Blairdce6cea2016-12-20 16:45:32 -08001111 if log_defaults_from_env:
1112 for default in log_defaults_from_env.split(','):
1113 try:
1114 name, level_str = default.split('=', 1)
1115 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001116 logger = logging.getLogger(name)
1117 logger.setLevel(level)
1118 logger.addHandler(handler)
1119 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001120 except ValueError:
1121 # NOTE(notmorgan): Invalid format of the log default,
1122 # skip and don't try and apply a logger for the
1123 # specified module
1124 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001125
Maru Newby3fe5f852015-01-13 04:22:14 +00001126
1127class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001128 """A test case with a functioning Zuul.
1129
1130 The following class variables are used during test setup and can
1131 be overidden by subclasses but are effectively read-only once a
1132 test method starts running:
1133
1134 :cvar str config_file: This points to the main zuul config file
1135 within the fixtures directory. Subclasses may override this
1136 to obtain a different behavior.
1137
1138 :cvar str tenant_config_file: This is the tenant config file
1139 (which specifies from what git repos the configuration should
1140 be loaded). It defaults to the value specified in
1141 `config_file` but can be overidden by subclasses to obtain a
1142 different tenant/project layout while using the standard main
1143 configuration.
1144
1145 The following are instance variables that are useful within test
1146 methods:
1147
1148 :ivar FakeGerritConnection fake_<connection>:
1149 A :py:class:`~tests.base.FakeGerritConnection` will be
1150 instantiated for each connection present in the config file
1151 and stored here. For instance, `fake_gerrit` will hold the
1152 FakeGerritConnection object for a connection named `gerrit`.
1153
1154 :ivar FakeGearmanServer gearman_server: An instance of
1155 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1156 server that all of the Zuul components in this test use to
1157 communicate with each other.
1158
1159 :ivar RecordingLaunchServer launch_server: An instance of
1160 :py:class:`~tests.base.RecordingLaunchServer` which is the
1161 Ansible launch server used to run jobs for this test.
1162
1163 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1164 representing currently running builds. They are appended to
1165 the list in the order they are launched, and removed from this
1166 list upon completion.
1167
1168 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1169 objects representing completed builds. They are appended to
1170 the list in the order they complete.
1171
1172 """
1173
James E. Blair83005782015-12-11 14:46:03 -08001174 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001175 run_ansible = False
James E. Blair3f876d52016-07-22 13:07:14 -07001176
1177 def _startMerger(self):
1178 self.merge_server = zuul.merger.server.MergeServer(self.config,
1179 self.connections)
1180 self.merge_server.start()
1181
Maru Newby3fe5f852015-01-13 04:22:14 +00001182 def setUp(self):
1183 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001184
1185 self.setupZK()
1186
James E. Blair97d902e2014-08-21 13:25:56 -07001187 if USE_TEMPDIR:
1188 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001189 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1190 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001191 else:
1192 tmp_root = os.environ.get("ZUUL_TEST_ROOT")
Clark Boylanb640e052014-04-03 16:41:46 -07001193 self.test_root = os.path.join(tmp_root, "zuul-test")
1194 self.upstream_root = os.path.join(self.test_root, "upstream")
1195 self.git_root = os.path.join(self.test_root, "git")
James E. Blairce8a2132016-05-19 15:21:52 -07001196 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001197
1198 if os.path.exists(self.test_root):
1199 shutil.rmtree(self.test_root)
1200 os.makedirs(self.test_root)
1201 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001202 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001203
1204 # Make per test copy of Configuration.
1205 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001206 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001207 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001208 self.config.get('zuul', 'tenant_config')))
Clark Boylanb640e052014-04-03 16:41:46 -07001209 self.config.set('merger', 'git_dir', self.git_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001210 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001211
1212 # For each project in config:
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001213 # TODOv3(jeblair): remove these and replace with new git
1214 # filesystem fixtures
Clark Boylanb640e052014-04-03 16:41:46 -07001215 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -07001216 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -07001217 self.init_repo("org/project5")
1218 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -07001219 self.init_repo("org/one-job-project")
1220 self.init_repo("org/nonvoting-project")
1221 self.init_repo("org/templated-project")
1222 self.init_repo("org/layered-project")
1223 self.init_repo("org/node-project")
1224 self.init_repo("org/conflict-project")
1225 self.init_repo("org/noop-project")
1226 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00001227 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -07001228
1229 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001230 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1231 # see: https://github.com/jsocol/pystatsd/issues/61
1232 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001233 os.environ['STATSD_PORT'] = str(self.statsd.port)
1234 self.statsd.start()
1235 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001236 reload_module(statsd)
1237 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001238
1239 self.gearman_server = FakeGearmanServer()
1240
1241 self.config.set('gearman', 'port', str(self.gearman_server.port))
James E. Blaire47eb772017-02-02 17:19:40 -08001242 self.log.info("Gearman server on port %s" %
1243 (self.gearman_server.port,))
Clark Boylanb640e052014-04-03 16:41:46 -07001244
James E. Blaire511d2f2016-12-08 15:22:26 -08001245 gerritsource.GerritSource.replication_timeout = 1.5
1246 gerritsource.GerritSource.replication_retry_interval = 0.5
1247 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001248
Joshua Hesketh352264b2015-08-11 23:42:08 +10001249 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001250
1251 self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
1252 FakeSwiftClientConnection))
James E. Blaire511d2f2016-12-08 15:22:26 -08001253
Clark Boylanb640e052014-04-03 16:41:46 -07001254 self.swift = zuul.lib.swift.Swift(self.config)
1255
Jan Hruban6b71aff2015-10-22 16:58:08 +02001256 self.event_queues = [
1257 self.sched.result_event_queue,
1258 self.sched.trigger_event_queue
1259 ]
1260
James E. Blairfef78942016-03-11 16:28:56 -08001261 self.configure_connections()
Joshua Hesketh352264b2015-08-11 23:42:08 +10001262 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001263
Clark Boylanb640e052014-04-03 16:41:46 -07001264 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001265 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001266 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001267 return FakeURLOpener(self.upstream_root, *args, **kw)
1268
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001269 old_urlopen = urllib.request.urlopen
1270 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001271
James E. Blair3f876d52016-07-22 13:07:14 -07001272 self._startMerger()
James E. Blair3f876d52016-07-22 13:07:14 -07001273
James E. Blaire1767bc2016-08-02 10:00:27 -07001274 self.launch_server = RecordingLaunchServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001275 self.config, self.connections,
James E. Blair854f8892017-02-02 11:25:39 -08001276 jobdir_root=self.test_root,
James E. Blaira92cbc82017-01-23 14:56:49 -08001277 _run_ansible=self.run_ansible,
1278 _test_root=self.test_root)
James E. Blaire1767bc2016-08-02 10:00:27 -07001279 self.launch_server.start()
1280 self.history = self.launch_server.build_history
1281 self.builds = self.launch_server.running_builds
1282
1283 self.launch_client = zuul.launcher.client.LaunchClient(
James E. Blair82938472016-01-11 14:38:13 -08001284 self.config, self.sched, self.swift)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001285 self.merge_client = zuul.merger.client.MergeClient(
1286 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001287 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001288 self.zk = zuul.zk.ZooKeeper()
1289 self.zk.connect([self.zk_config])
1290
1291 self.fake_nodepool = FakeNodepool(self.zk_config.host,
1292 self.zk_config.port,
1293 self.zk_config.chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001294
James E. Blaire1767bc2016-08-02 10:00:27 -07001295 self.sched.setLauncher(self.launch_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001296 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001297 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001298 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001299
Paul Belanger88ef0ea2015-12-23 11:57:02 -05001300 self.webapp = zuul.webapp.WebApp(
1301 self.sched, port=0, listen_address='127.0.0.1')
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001302 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001303
1304 self.sched.start()
1305 self.sched.reconfigure(self.config)
1306 self.sched.resume()
1307 self.webapp.start()
1308 self.rpc.start()
James E. Blaire1767bc2016-08-02 10:00:27 -07001309 self.launch_client.gearman.waitForServer()
Clark Boylanb640e052014-04-03 16:41:46 -07001310
Clark Boylanb640e052014-04-03 16:41:46 -07001311 self.addCleanup(self.shutdown)
1312
James E. Blaire18d4602017-01-05 11:17:28 -08001313 def tearDown(self):
1314 super(ZuulTestCase, self).tearDown()
1315 self.assertFinalState()
1316
James E. Blairfef78942016-03-11 16:28:56 -08001317 def configure_connections(self):
James E. Blaire511d2f2016-12-08 15:22:26 -08001318 # Set up gerrit related fakes
1319 # Set a changes database so multiple FakeGerrit's can report back to
1320 # a virtual canonical database given by the configured hostname
1321 self.gerrit_changes_dbs = {}
1322
1323 def getGerritConnection(driver, name, config):
1324 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1325 con = FakeGerritConnection(driver, name, config,
1326 changes_db=db,
1327 upstream_root=self.upstream_root)
1328 self.event_queues.append(con.event_queue)
1329 setattr(self, 'fake_' + name, con)
1330 return con
1331
1332 self.useFixture(fixtures.MonkeyPatch(
1333 'zuul.driver.gerrit.GerritDriver.getConnection',
1334 getGerritConnection))
1335
1336 # Set up smtp related fakes
Joshua Hesketh352264b2015-08-11 23:42:08 +10001337 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001338
Joshua Hesketh352264b2015-08-11 23:42:08 +10001339 def FakeSMTPFactory(*args, **kw):
1340 args = [self.smtp_messages] + list(args)
1341 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001342
Joshua Hesketh352264b2015-08-11 23:42:08 +10001343 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001344
James E. Blaire511d2f2016-12-08 15:22:26 -08001345 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001346 self.connections = zuul.lib.connections.ConnectionRegistry()
James E. Blaire511d2f2016-12-08 15:22:26 -08001347 self.connections.configure(self.config)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001348
James E. Blair83005782015-12-11 14:46:03 -08001349 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001350 # This creates the per-test configuration object. It can be
1351 # overriden by subclasses, but should not need to be since it
1352 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001353 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001354 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair2a629ec2015-12-22 15:32:02 -08001355 if hasattr(self, 'tenant_config_file'):
1356 self.config.set('zuul', 'tenant_config', self.tenant_config_file)
James E. Blair96c6bf82016-01-15 16:20:40 -08001357 git_path = os.path.join(
1358 os.path.dirname(
1359 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1360 'git')
1361 if os.path.exists(git_path):
1362 for reponame in os.listdir(git_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001363 project = reponame.replace('_', '/')
1364 self.copyDirToRepo(project,
James E. Blair96c6bf82016-01-15 16:20:40 -08001365 os.path.join(git_path, reponame))
1366
James E. Blair498059b2016-12-20 13:50:13 -08001367 def setupZK(self):
1368 self.zk_chroot_fixture = self.useFixture(ChrootedKazooFixture())
James E. Blairdce6cea2016-12-20 16:45:32 -08001369 self.zk_config = zuul.zk.ZooKeeperConnectionConfig(
1370 self.zk_chroot_fixture.zookeeper_host,
1371 self.zk_chroot_fixture.zookeeper_port,
1372 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08001373
James E. Blair96c6bf82016-01-15 16:20:40 -08001374 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001375 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08001376
1377 files = {}
1378 for (dirpath, dirnames, filenames) in os.walk(source_path):
1379 for filename in filenames:
1380 test_tree_filepath = os.path.join(dirpath, filename)
1381 common_path = os.path.commonprefix([test_tree_filepath,
1382 source_path])
1383 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1384 with open(test_tree_filepath, 'r') as f:
1385 content = f.read()
1386 files[relative_filepath] = content
1387 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001388 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08001389
James E. Blaire18d4602017-01-05 11:17:28 -08001390 def assertNodepoolState(self):
1391 # Make sure that there are no pending requests
1392
1393 requests = self.fake_nodepool.getNodeRequests()
1394 self.assertEqual(len(requests), 0)
1395
1396 nodes = self.fake_nodepool.getNodes()
1397 for node in nodes:
1398 self.assertFalse(node['_lock'], "Node %s is locked" %
1399 (node['_oid'],))
1400
Clark Boylanb640e052014-04-03 16:41:46 -07001401 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001402 # Make sure that git.Repo objects have been garbage collected.
1403 repos = []
1404 gc.collect()
1405 for obj in gc.get_objects():
1406 if isinstance(obj, git.Repo):
1407 repos.append(obj)
1408 self.assertEqual(len(repos), 0)
1409 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08001410 self.assertNodepoolState()
James E. Blair83005782015-12-11 14:46:03 -08001411 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001412 for tenant in self.sched.abide.tenants.values():
1413 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001414 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001415 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001416
1417 def shutdown(self):
1418 self.log.debug("Shutting down after tests")
James E. Blaire1767bc2016-08-02 10:00:27 -07001419 self.launch_client.stop()
James E. Blair3f876d52016-07-22 13:07:14 -07001420 self.merge_server.stop()
1421 self.merge_server.join()
Clark Boylanb640e052014-04-03 16:41:46 -07001422 self.merge_client.stop()
James E. Blaire1767bc2016-08-02 10:00:27 -07001423 self.launch_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001424 self.sched.stop()
1425 self.sched.join()
1426 self.statsd.stop()
1427 self.statsd.join()
1428 self.webapp.stop()
1429 self.webapp.join()
1430 self.rpc.stop()
1431 self.rpc.join()
1432 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08001433 self.fake_nodepool.stop()
1434 self.zk.disconnect()
Clark Boylanb640e052014-04-03 16:41:46 -07001435 threads = threading.enumerate()
1436 if len(threads) > 1:
1437 self.log.error("More than one thread is running: %s" % threads)
James E. Blair6ac368c2016-12-22 18:07:20 -08001438 self.printHistory()
Clark Boylanb640e052014-04-03 16:41:46 -07001439
1440 def init_repo(self, project):
1441 parts = project.split('/')
1442 path = os.path.join(self.upstream_root, *parts[:-1])
1443 if not os.path.exists(path):
1444 os.makedirs(path)
1445 path = os.path.join(self.upstream_root, project)
1446 repo = git.Repo.init(path)
1447
Morgan Fainberg78c301a2016-07-14 13:47:01 -07001448 with repo.config_writer() as config_writer:
1449 config_writer.set_value('user', 'email', 'user@example.com')
1450 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07001451
Clark Boylanb640e052014-04-03 16:41:46 -07001452 repo.index.commit('initial commit')
1453 master = repo.create_head('master')
Clark Boylanb640e052014-04-03 16:41:46 -07001454
James E. Blair97d902e2014-08-21 13:25:56 -07001455 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001456 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001457 repo.git.clean('-x', '-f', '-d')
1458
James E. Blair97d902e2014-08-21 13:25:56 -07001459 def create_branch(self, project, branch):
1460 path = os.path.join(self.upstream_root, project)
1461 repo = git.Repo.init(path)
1462 fn = os.path.join(path, 'README')
1463
1464 branch_head = repo.create_head(branch)
1465 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001466 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001467 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001468 f.close()
1469 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001470 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001471
James E. Blair97d902e2014-08-21 13:25:56 -07001472 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001473 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001474 repo.git.clean('-x', '-f', '-d')
1475
Sachi King9f16d522016-03-16 12:20:45 +11001476 def create_commit(self, project):
1477 path = os.path.join(self.upstream_root, project)
1478 repo = git.Repo(path)
1479 repo.head.reference = repo.heads['master']
1480 file_name = os.path.join(path, 'README')
1481 with open(file_name, 'a') as f:
1482 f.write('creating fake commit\n')
1483 repo.index.add([file_name])
1484 commit = repo.index.commit('Creating a fake commit')
1485 return commit.hexsha
1486
James E. Blairb8c16472015-05-05 14:55:26 -07001487 def orderedRelease(self):
1488 # Run one build at a time to ensure non-race order:
1489 while len(self.builds):
1490 self.release(self.builds[0])
1491 self.waitUntilSettled()
1492
Clark Boylanb640e052014-04-03 16:41:46 -07001493 def release(self, job):
1494 if isinstance(job, FakeBuild):
1495 job.release()
1496 else:
1497 job.waiting = False
1498 self.log.debug("Queued job %s released" % job.unique)
1499 self.gearman_server.wakeConnections()
1500
1501 def getParameter(self, job, name):
1502 if isinstance(job, FakeBuild):
1503 return job.parameters[name]
1504 else:
1505 parameters = json.loads(job.arguments)
1506 return parameters[name]
1507
Clark Boylanb640e052014-04-03 16:41:46 -07001508 def haveAllBuildsReported(self):
1509 # See if Zuul is waiting on a meta job to complete
James E. Blaire1767bc2016-08-02 10:00:27 -07001510 if self.launch_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07001511 return False
1512 # Find out if every build that the worker has completed has been
1513 # reported back to Zuul. If it hasn't then that means a Gearman
1514 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07001515 for build in self.history:
James E. Blaire1767bc2016-08-02 10:00:27 -07001516 zbuild = self.launch_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07001517 if not zbuild:
1518 # It has already been reported
1519 continue
1520 # It hasn't been reported yet.
1521 return False
1522 # Make sure that none of the worker connections are in GRAB_WAIT
James E. Blaire1767bc2016-08-02 10:00:27 -07001523 for connection in self.launch_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001524 if connection.state == 'GRAB_WAIT':
1525 return False
1526 return True
1527
1528 def areAllBuildsWaiting(self):
James E. Blaire1767bc2016-08-02 10:00:27 -07001529 builds = self.launch_client.builds.values()
Clark Boylanb640e052014-04-03 16:41:46 -07001530 for build in builds:
1531 client_job = None
James E. Blaire1767bc2016-08-02 10:00:27 -07001532 for conn in self.launch_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001533 for j in conn.related_jobs.values():
1534 if j.unique == build.uuid:
1535 client_job = j
1536 break
1537 if not client_job:
1538 self.log.debug("%s is not known to the gearman client" %
1539 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001540 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001541 if not client_job.handle:
1542 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001543 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001544 server_job = self.gearman_server.jobs.get(client_job.handle)
1545 if not server_job:
1546 self.log.debug("%s is not known to the gearman server" %
1547 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001548 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001549 if not hasattr(server_job, 'waiting'):
1550 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001551 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001552 if server_job.waiting:
1553 continue
James E. Blair17302972016-08-10 16:11:42 -07001554 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08001555 self.log.debug("%s has not reported start" % build)
1556 return False
James E. Blairab7132b2016-08-05 12:36:22 -07001557 worker_build = self.launch_server.job_builds.get(server_job.unique)
James E. Blair962220f2016-08-03 11:22:38 -07001558 if worker_build:
1559 if worker_build.isWaiting():
1560 continue
1561 else:
1562 self.log.debug("%s is running" % worker_build)
1563 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001564 else:
James E. Blair962220f2016-08-03 11:22:38 -07001565 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001566 return False
1567 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001568
James E. Blairdce6cea2016-12-20 16:45:32 -08001569 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001570 if self.fake_nodepool.paused:
1571 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08001572 if self.sched.nodepool.requests:
1573 return False
1574 return True
1575
Jan Hruban6b71aff2015-10-22 16:58:08 +02001576 def eventQueuesEmpty(self):
1577 for queue in self.event_queues:
1578 yield queue.empty()
1579
1580 def eventQueuesJoin(self):
1581 for queue in self.event_queues:
1582 queue.join()
1583
Clark Boylanb640e052014-04-03 16:41:46 -07001584 def waitUntilSettled(self):
1585 self.log.debug("Waiting until settled...")
1586 start = time.time()
1587 while True:
James E. Blair71932482017-02-02 11:29:07 -08001588 if time.time() - start > 20:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001589 self.log.error("Timeout waiting for Zuul to settle")
1590 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07001591 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001592 self.log.error(" %s: %s" % (queue, queue.empty()))
1593 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07001594 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001595 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07001596 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001597 self.log.error("All requests completed: %s" %
1598 (self.areAllNodeRequestsComplete(),))
1599 self.log.error("Merge client jobs: %s" %
1600 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07001601 raise Exception("Timeout waiting for Zuul to settle")
1602 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07001603
James E. Blaire1767bc2016-08-02 10:00:27 -07001604 self.launch_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07001605 # have all build states propogated to zuul?
1606 if self.haveAllBuildsReported():
1607 # Join ensures that the queue is empty _and_ events have been
1608 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001609 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001610 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001611 if (not self.merge_client.jobs and
Clark Boylanb640e052014-04-03 16:41:46 -07001612 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08001613 self.areAllBuildsWaiting() and
James E. Blair36c611a2017-02-06 15:59:43 -08001614 self.areAllNodeRequestsComplete() and
1615 all(self.eventQueuesEmpty())):
1616 # The queue empty check is placed at the end to
1617 # ensure that if a component adds an event between
1618 # when locked the run handler and checked that the
1619 # components were stable, we don't erroneously
1620 # report that we are settled.
Clark Boylanb640e052014-04-03 16:41:46 -07001621 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001622 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001623 self.log.debug("...settled.")
1624 return
1625 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001626 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001627 self.sched.wake_event.wait(0.1)
1628
1629 def countJobResults(self, jobs, result):
1630 jobs = filter(lambda x: x.result == result, jobs)
1631 return len(jobs)
1632
James E. Blair96c6bf82016-01-15 16:20:40 -08001633 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07001634 for job in self.history:
1635 if (job.name == name and
1636 (project is None or
1637 job.parameters['ZUUL_PROJECT'] == project)):
1638 return job
Clark Boylanb640e052014-04-03 16:41:46 -07001639 raise Exception("Unable to find job %s in history" % name)
1640
1641 def assertEmptyQueues(self):
1642 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001643 for tenant in self.sched.abide.tenants.values():
1644 for pipeline in tenant.layout.pipelines.values():
1645 for queue in pipeline.queues:
1646 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10001647 print('pipeline %s queue %s contents %s' % (
1648 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08001649 self.assertEqual(len(queue.queue), 0,
1650 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001651
1652 def assertReportedStat(self, key, value=None, kind=None):
1653 start = time.time()
1654 while time.time() < (start + 5):
1655 for stat in self.statsd.stats:
Clark Boylanb640e052014-04-03 16:41:46 -07001656 k, v = stat.split(':')
1657 if key == k:
1658 if value is None and kind is None:
1659 return
1660 elif value:
1661 if value == v:
1662 return
1663 elif kind:
1664 if v.endswith('|' + kind):
1665 return
1666 time.sleep(0.1)
1667
Clark Boylanb640e052014-04-03 16:41:46 -07001668 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001669
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001670 def assertBuilds(self, builds):
1671 """Assert that the running builds are as described.
1672
1673 The list of running builds is examined and must match exactly
1674 the list of builds described by the input.
1675
1676 :arg list builds: A list of dictionaries. Each item in the
1677 list must match the corresponding build in the build
1678 history, and each element of the dictionary must match the
1679 corresponding attribute of the build.
1680
1681 """
James E. Blair3158e282016-08-19 09:34:11 -07001682 try:
1683 self.assertEqual(len(self.builds), len(builds))
1684 for i, d in enumerate(builds):
1685 for k, v in d.items():
1686 self.assertEqual(
1687 getattr(self.builds[i], k), v,
1688 "Element %i in builds does not match" % (i,))
1689 except Exception:
1690 for build in self.builds:
1691 self.log.error("Running build: %s" % build)
1692 else:
1693 self.log.error("No running builds")
1694 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001695
James E. Blairb536ecc2016-08-31 10:11:42 -07001696 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001697 """Assert that the completed builds are as described.
1698
1699 The list of completed builds is examined and must match
1700 exactly the list of builds described by the input.
1701
1702 :arg list history: A list of dictionaries. Each item in the
1703 list must match the corresponding build in the build
1704 history, and each element of the dictionary must match the
1705 corresponding attribute of the build.
1706
James E. Blairb536ecc2016-08-31 10:11:42 -07001707 :arg bool ordered: If true, the history must match the order
1708 supplied, if false, the builds are permitted to have
1709 arrived in any order.
1710
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001711 """
James E. Blairb536ecc2016-08-31 10:11:42 -07001712 def matches(history_item, item):
1713 for k, v in item.items():
1714 if getattr(history_item, k) != v:
1715 return False
1716 return True
James E. Blair3158e282016-08-19 09:34:11 -07001717 try:
1718 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07001719 if ordered:
1720 for i, d in enumerate(history):
1721 if not matches(self.history[i], d):
1722 raise Exception(
1723 "Element %i in history does not match" % (i,))
1724 else:
1725 unseen = self.history[:]
1726 for i, d in enumerate(history):
1727 found = False
1728 for unseen_item in unseen:
1729 if matches(unseen_item, d):
1730 found = True
1731 unseen.remove(unseen_item)
1732 break
1733 if not found:
1734 raise Exception("No match found for element %i "
1735 "in history" % (i,))
1736 if unseen:
1737 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07001738 except Exception:
1739 for build in self.history:
1740 self.log.error("Completed build: %s" % build)
1741 else:
1742 self.log.error("No completed builds")
1743 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001744
James E. Blair6ac368c2016-12-22 18:07:20 -08001745 def printHistory(self):
1746 """Log the build history.
1747
1748 This can be useful during tests to summarize what jobs have
1749 completed.
1750
1751 """
1752 self.log.debug("Build history:")
1753 for build in self.history:
1754 self.log.debug(build)
1755
James E. Blair59fdbac2015-12-07 17:08:06 -08001756 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001757 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1758
1759 def updateConfigLayout(self, path):
1760 root = os.path.join(self.test_root, "config")
1761 os.makedirs(root)
1762 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1763 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05001764- tenant:
1765 name: openstack
1766 source:
1767 gerrit:
1768 config-repos:
1769 - %s
1770 """ % path)
James E. Blairf84026c2015-12-08 16:11:46 -08001771 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05001772 self.config.set('zuul', 'tenant_config',
1773 os.path.join(FIXTURE_DIR, f.name))
James E. Blair14abdf42015-12-09 16:11:53 -08001774
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001775 def addCommitToRepo(self, project, message, files,
1776 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08001777 path = os.path.join(self.upstream_root, project)
1778 repo = git.Repo(path)
1779 repo.head.reference = branch
1780 zuul.merger.merger.reset_repo_to_head(repo)
1781 for fn, content in files.items():
1782 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08001783 try:
1784 os.makedirs(os.path.dirname(fn))
1785 except OSError:
1786 pass
James E. Blair14abdf42015-12-09 16:11:53 -08001787 with open(fn, 'w') as f:
1788 f.write(content)
1789 repo.index.add([fn])
1790 commit = repo.index.commit(message)
1791 repo.heads[branch].commit = commit
1792 repo.head.reference = branch
1793 repo.git.clean('-x', '-f', '-d')
1794 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001795 if tag:
1796 repo.create_tag(tag)
James E. Blair3f876d52016-07-22 13:07:14 -07001797
James E. Blair7fc8daa2016-08-08 15:37:15 -07001798 def addEvent(self, connection, event):
1799 """Inject a Fake (Gerrit) event.
1800
1801 This method accepts a JSON-encoded event and simulates Zuul
1802 having received it from Gerrit. It could (and should)
1803 eventually apply to any connection type, but is currently only
1804 used with Gerrit connections. The name of the connection is
1805 used to look up the corresponding server, and the event is
1806 simulated as having been received by all Zuul connections
1807 attached to that server. So if two Gerrit connections in Zuul
1808 are connected to the same Gerrit server, and you invoke this
1809 method specifying the name of one of them, the event will be
1810 received by both.
1811
1812 .. note::
1813
1814 "self.fake_gerrit.addEvent" calls should be migrated to
1815 this method.
1816
1817 :arg str connection: The name of the connection corresponding
1818 to the gerrit server.
1819 :arg str event: The JSON-encoded event.
1820
1821 """
1822 specified_conn = self.connections.connections[connection]
1823 for conn in self.connections.connections.values():
1824 if (isinstance(conn, specified_conn.__class__) and
1825 specified_conn.server == conn.server):
1826 conn.addEvent(event)
1827
James E. Blair3f876d52016-07-22 13:07:14 -07001828
1829class AnsibleZuulTestCase(ZuulTestCase):
1830 """ZuulTestCase but with an actual ansible launcher running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07001831 run_ansible = True