blob: 8802cfd986a471e5562f50dd791a3ce3188b571e [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070019import gc
20import hashlib
21import json
22import logging
23import os
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070025from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070026import random
27import re
28import select
29import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030030from six.moves import reload_module
James E. Blair1c236df2017-02-01 14:07:24 -080031from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070032import socket
33import string
34import subprocess
35import swiftclient
James E. Blair1c236df2017-02-01 14:07:24 -080036import sys
James E. Blairf84026c2015-12-08 16:11:46 -080037import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070038import threading
39import time
Clark Boylanb640e052014-04-03 16:41:46 -070040
41import git
42import gear
43import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080044import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080045import kazoo.exceptions
Clark Boylanb640e052014-04-03 16:41:46 -070046import statsd
47import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080048import testtools.content
49import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080050from git.exc import NoSuchPathError
Clark Boylanb640e052014-04-03 16:41:46 -070051
James E. Blaire511d2f2016-12-08 15:22:26 -080052import zuul.driver.gerrit.gerritsource as gerritsource
53import zuul.driver.gerrit.gerritconnection as gerritconnection
Clark Boylanb640e052014-04-03 16:41:46 -070054import zuul.scheduler
55import zuul.webapp
56import zuul.rpclistener
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +100057import zuul.launcher.server
58import zuul.launcher.client
Clark Boylanb640e052014-04-03 16:41:46 -070059import zuul.lib.swift
James E. Blair83005782015-12-11 14:46:03 -080060import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070061import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070062import zuul.merger.merger
63import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070064import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080065import zuul.zk
Clark Boylanb640e052014-04-03 16:41:46 -070066
67FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
68 'fixtures')
James E. Blair97d902e2014-08-21 13:25:56 -070069USE_TEMPDIR = True
Clark Boylanb640e052014-04-03 16:41:46 -070070
Clark Boylanb640e052014-04-03 16:41:46 -070071
72def repack_repo(path):
73 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
74 output = subprocess.Popen(cmd, close_fds=True,
75 stdout=subprocess.PIPE,
76 stderr=subprocess.PIPE)
77 out = output.communicate()
78 if output.returncode:
79 raise Exception("git repack returned %d" % output.returncode)
80 return out
81
82
83def random_sha1():
84 return hashlib.sha1(str(random.random())).hexdigest()
85
86
James E. Blaira190f3b2015-01-05 14:56:54 -080087def iterate_timeout(max_seconds, purpose):
88 start = time.time()
89 count = 0
90 while (time.time() < start + max_seconds):
91 count += 1
92 yield count
93 time.sleep(0)
94 raise Exception("Timeout waiting for %s" % purpose)
95
96
Clark Boylanb640e052014-04-03 16:41:46 -070097class ChangeReference(git.Reference):
98 _common_path_default = "refs/changes"
99 _points_to_commits_only = True
100
101
102class FakeChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700103 categories = {'approved': ('Approved', -1, 1),
104 'code-review': ('Code-Review', -2, 2),
105 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700106
107 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700108 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700109 self.gerrit = gerrit
110 self.reported = 0
111 self.queried = 0
112 self.patchsets = []
113 self.number = number
114 self.project = project
115 self.branch = branch
116 self.subject = subject
117 self.latest_patchset = 0
118 self.depends_on_change = None
119 self.needed_by_changes = []
120 self.fail_merge = False
121 self.messages = []
122 self.data = {
123 'branch': branch,
124 'comments': [],
125 'commitMessage': subject,
126 'createdOn': time.time(),
127 'id': 'I' + random_sha1(),
128 'lastUpdated': time.time(),
129 'number': str(number),
130 'open': status == 'NEW',
131 'owner': {'email': 'user@example.com',
132 'name': 'User Name',
133 'username': 'username'},
134 'patchSets': self.patchsets,
135 'project': project,
136 'status': status,
137 'subject': subject,
138 'submitRecords': [],
139 'url': 'https://hostname/%s' % number}
140
141 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700142 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700143 self.data['submitRecords'] = self.getSubmitRecords()
144 self.open = status == 'NEW'
145
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700146 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700147 path = os.path.join(self.upstream_root, self.project)
148 repo = git.Repo(path)
149 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
150 self.latest_patchset),
151 'refs/tags/init')
152 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700153 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700154 repo.git.clean('-x', '-f', '-d')
155
156 path = os.path.join(self.upstream_root, self.project)
157 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700158 for fn, content in files.items():
159 fn = os.path.join(path, fn)
160 with open(fn, 'w') as f:
161 f.write(content)
162 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700163 else:
164 for fni in range(100):
165 fn = os.path.join(path, str(fni))
166 f = open(fn, 'w')
167 for ci in range(4096):
168 f.write(random.choice(string.printable))
169 f.close()
170 repo.index.add([fn])
171
172 r = repo.index.commit(msg)
173 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700174 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700175 repo.git.clean('-x', '-f', '-d')
176 repo.heads['master'].checkout()
177 return r
178
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700179 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700180 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700181 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700182 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700183 data = ("test %s %s %s\n" %
184 (self.branch, self.number, self.latest_patchset))
185 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700186 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700187 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700188 ps_files = [{'file': '/COMMIT_MSG',
189 'type': 'ADDED'},
190 {'file': 'README',
191 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700192 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700193 ps_files.append({'file': f, 'type': 'ADDED'})
194 d = {'approvals': [],
195 'createdOn': time.time(),
196 'files': ps_files,
197 'number': str(self.latest_patchset),
198 'ref': 'refs/changes/1/%s/%s' % (self.number,
199 self.latest_patchset),
200 'revision': c.hexsha,
201 'uploader': {'email': 'user@example.com',
202 'name': 'User name',
203 'username': 'user'}}
204 self.data['currentPatchSet'] = d
205 self.patchsets.append(d)
206 self.data['submitRecords'] = self.getSubmitRecords()
207
208 def getPatchsetCreatedEvent(self, patchset):
209 event = {"type": "patchset-created",
210 "change": {"project": self.project,
211 "branch": self.branch,
212 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
213 "number": str(self.number),
214 "subject": self.subject,
215 "owner": {"name": "User Name"},
216 "url": "https://hostname/3"},
217 "patchSet": self.patchsets[patchset - 1],
218 "uploader": {"name": "User Name"}}
219 return event
220
221 def getChangeRestoredEvent(self):
222 event = {"type": "change-restored",
223 "change": {"project": self.project,
224 "branch": self.branch,
225 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
226 "number": str(self.number),
227 "subject": self.subject,
228 "owner": {"name": "User Name"},
229 "url": "https://hostname/3"},
230 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100231 "patchSet": self.patchsets[-1],
232 "reason": ""}
233 return event
234
235 def getChangeAbandonedEvent(self):
236 event = {"type": "change-abandoned",
237 "change": {"project": self.project,
238 "branch": self.branch,
239 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
240 "number": str(self.number),
241 "subject": self.subject,
242 "owner": {"name": "User Name"},
243 "url": "https://hostname/3"},
244 "abandoner": {"name": "User Name"},
245 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700246 "reason": ""}
247 return event
248
249 def getChangeCommentEvent(self, patchset):
250 event = {"type": "comment-added",
251 "change": {"project": self.project,
252 "branch": self.branch,
253 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
254 "number": str(self.number),
255 "subject": self.subject,
256 "owner": {"name": "User Name"},
257 "url": "https://hostname/3"},
258 "patchSet": self.patchsets[patchset - 1],
259 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700260 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700261 "description": "Code-Review",
262 "value": "0"}],
263 "comment": "This is a comment"}
264 return event
265
James E. Blairc2a5ed72017-02-20 14:12:01 -0500266 def getChangeMergedEvent(self):
267 event = {"submitter": {"name": "Jenkins",
268 "username": "jenkins"},
269 "newRev": "29ed3b5f8f750a225c5be70235230e3a6ccb04d9",
270 "patchSet": self.patchsets[-1],
271 "change": self.data,
272 "type": "change-merged",
273 "eventCreatedOn": 1487613810}
274 return event
275
Joshua Hesketh642824b2014-07-01 17:54:59 +1000276 def addApproval(self, category, value, username='reviewer_john',
277 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700278 if not granted_on:
279 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000280 approval = {
281 'description': self.categories[category][0],
282 'type': category,
283 'value': str(value),
284 'by': {
285 'username': username,
286 'email': username + '@example.com',
287 },
288 'grantedOn': int(granted_on)
289 }
Clark Boylanb640e052014-04-03 16:41:46 -0700290 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
291 if x['by']['username'] == username and x['type'] == category:
292 del self.patchsets[-1]['approvals'][i]
293 self.patchsets[-1]['approvals'].append(approval)
294 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000295 'author': {'email': 'author@example.com',
296 'name': 'Patchset Author',
297 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700298 'change': {'branch': self.branch,
299 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
300 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000301 'owner': {'email': 'owner@example.com',
302 'name': 'Change Owner',
303 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700304 'project': self.project,
305 'subject': self.subject,
306 'topic': 'master',
307 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000308 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700309 'patchSet': self.patchsets[-1],
310 'type': 'comment-added'}
311 self.data['submitRecords'] = self.getSubmitRecords()
312 return json.loads(json.dumps(event))
313
314 def getSubmitRecords(self):
315 status = {}
316 for cat in self.categories.keys():
317 status[cat] = 0
318
319 for a in self.patchsets[-1]['approvals']:
320 cur = status[a['type']]
321 cat_min, cat_max = self.categories[a['type']][1:]
322 new = int(a['value'])
323 if new == cat_min:
324 cur = new
325 elif abs(new) > abs(cur):
326 cur = new
327 status[a['type']] = cur
328
329 labels = []
330 ok = True
331 for typ, cat in self.categories.items():
332 cur = status[typ]
333 cat_min, cat_max = cat[1:]
334 if cur == cat_min:
335 value = 'REJECT'
336 ok = False
337 elif cur == cat_max:
338 value = 'OK'
339 else:
340 value = 'NEED'
341 ok = False
342 labels.append({'label': cat[0], 'status': value})
343 if ok:
344 return [{'status': 'OK'}]
345 return [{'status': 'NOT_READY',
346 'labels': labels}]
347
348 def setDependsOn(self, other, patchset):
349 self.depends_on_change = other
350 d = {'id': other.data['id'],
351 'number': other.data['number'],
352 'ref': other.patchsets[patchset - 1]['ref']
353 }
354 self.data['dependsOn'] = [d]
355
356 other.needed_by_changes.append(self)
357 needed = other.data.get('neededBy', [])
358 d = {'id': self.data['id'],
359 'number': self.data['number'],
360 'ref': self.patchsets[patchset - 1]['ref'],
361 'revision': self.patchsets[patchset - 1]['revision']
362 }
363 needed.append(d)
364 other.data['neededBy'] = needed
365
366 def query(self):
367 self.queried += 1
368 d = self.data.get('dependsOn')
369 if d:
370 d = d[0]
371 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
372 d['isCurrentPatchSet'] = True
373 else:
374 d['isCurrentPatchSet'] = False
375 return json.loads(json.dumps(self.data))
376
377 def setMerged(self):
378 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000379 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700380 return
381 if self.fail_merge:
382 return
383 self.data['status'] = 'MERGED'
384 self.open = False
385
386 path = os.path.join(self.upstream_root, self.project)
387 repo = git.Repo(path)
388 repo.heads[self.branch].commit = \
389 repo.commit(self.patchsets[-1]['revision'])
390
391 def setReported(self):
392 self.reported += 1
393
394
James E. Blaire511d2f2016-12-08 15:22:26 -0800395class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700396 """A Fake Gerrit connection for use in tests.
397
398 This subclasses
399 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
400 ability for tests to add changes to the fake Gerrit it represents.
401 """
402
Joshua Hesketh352264b2015-08-11 23:42:08 +1000403 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700404
James E. Blaire511d2f2016-12-08 15:22:26 -0800405 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700406 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800407 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000408 connection_config)
409
James E. Blair7fc8daa2016-08-08 15:37:15 -0700410 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700411 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
412 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000413 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700414 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200415 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700416
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700417 def addFakeChange(self, project, branch, subject, status='NEW',
418 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700419 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700420 self.change_number += 1
421 c = FakeChange(self, self.change_number, project, branch, subject,
422 upstream_root=self.upstream_root,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700423 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700424 self.changes[self.change_number] = c
425 return c
426
Clark Boylanb640e052014-04-03 16:41:46 -0700427 def review(self, project, changeid, message, action):
428 number, ps = changeid.split(',')
429 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000430
431 # Add the approval back onto the change (ie simulate what gerrit would
432 # do).
433 # Usually when zuul leaves a review it'll create a feedback loop where
434 # zuul's review enters another gerrit event (which is then picked up by
435 # zuul). However, we can't mimic this behaviour (by adding this
436 # approval event into the queue) as it stops jobs from checking what
437 # happens before this event is triggered. If a job needs to see what
438 # happens they can add their own verified event into the queue.
439 # Nevertheless, we can update change with the new review in gerrit.
440
James E. Blair8b5408c2016-08-08 15:37:46 -0700441 for cat in action.keys():
442 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000443 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000444
James E. Blair8b5408c2016-08-08 15:37:46 -0700445 # TODOv3(jeblair): can this be removed?
Joshua Hesketh642824b2014-07-01 17:54:59 +1000446 if 'label' in action:
447 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000448 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000449
Clark Boylanb640e052014-04-03 16:41:46 -0700450 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000451
Clark Boylanb640e052014-04-03 16:41:46 -0700452 if 'submit' in action:
453 change.setMerged()
454 if message:
455 change.setReported()
456
457 def query(self, number):
458 change = self.changes.get(int(number))
459 if change:
460 return change.query()
461 return {}
462
James E. Blairc494d542014-08-06 09:23:52 -0700463 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700464 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700465 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800466 if query.startswith('change:'):
467 # Query a specific changeid
468 changeid = query[len('change:'):]
469 l = [change.query() for change in self.changes.values()
470 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700471 elif query.startswith('message:'):
472 # Query the content of a commit message
473 msg = query[len('message:'):].strip()
474 l = [change.query() for change in self.changes.values()
475 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800476 else:
477 # Query all open changes
478 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700479 return l
James E. Blairc494d542014-08-06 09:23:52 -0700480
Joshua Hesketh352264b2015-08-11 23:42:08 +1000481 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700482 pass
483
Joshua Hesketh352264b2015-08-11 23:42:08 +1000484 def getGitUrl(self, project):
485 return os.path.join(self.upstream_root, project.name)
486
Clark Boylanb640e052014-04-03 16:41:46 -0700487
488class BuildHistory(object):
489 def __init__(self, **kw):
490 self.__dict__.update(kw)
491
492 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -0700493 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
494 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -0700495
496
497class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200498 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700499 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700500 self.url = url
501
502 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -0700503 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -0700504 path = res.path
505 project = '/'.join(path.split('/')[2:-2])
506 ret = '001e# service=git-upload-pack\n'
507 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
508 'multi_ack thin-pack side-band side-band-64k ofs-delta '
509 'shallow no-progress include-tag multi_ack_detailed no-done\n')
510 path = os.path.join(self.upstream_root, project)
511 repo = git.Repo(path)
512 for ref in repo.refs:
513 r = ref.object.hexsha + ' ' + ref.path + '\n'
514 ret += '%04x%s' % (len(r) + 4, r)
515 ret += '0000'
516 return ret
517
518
Clark Boylanb640e052014-04-03 16:41:46 -0700519class FakeStatsd(threading.Thread):
520 def __init__(self):
521 threading.Thread.__init__(self)
522 self.daemon = True
523 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
524 self.sock.bind(('', 0))
525 self.port = self.sock.getsockname()[1]
526 self.wake_read, self.wake_write = os.pipe()
527 self.stats = []
528
529 def run(self):
530 while True:
531 poll = select.poll()
532 poll.register(self.sock, select.POLLIN)
533 poll.register(self.wake_read, select.POLLIN)
534 ret = poll.poll()
535 for (fd, event) in ret:
536 if fd == self.sock.fileno():
537 data = self.sock.recvfrom(1024)
538 if not data:
539 return
540 self.stats.append(data[0])
541 if fd == self.wake_read:
542 return
543
544 def stop(self):
545 os.write(self.wake_write, '1\n')
546
547
James E. Blaire1767bc2016-08-02 10:00:27 -0700548class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -0700549 log = logging.getLogger("zuul.test")
550
James E. Blair34776ee2016-08-25 13:53:54 -0700551 def __init__(self, launch_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -0700552 self.daemon = True
James E. Blaire1767bc2016-08-02 10:00:27 -0700553 self.launch_server = launch_server
Clark Boylanb640e052014-04-03 16:41:46 -0700554 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -0700555 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -0700556 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -0700557 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -0700558 # TODOv3(jeblair): self.node is really "the image of the node
559 # assigned". We should rename it (self.node_image?) if we
560 # keep using it like this, or we may end up exposing more of
561 # the complexity around multi-node jobs here
562 # (self.nodes[0].image?)
563 self.node = None
564 if len(self.parameters.get('nodes')) == 1:
565 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -0700566 self.unique = self.parameters['ZUUL_UUID']
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100567 self.pipeline = self.parameters['ZUUL_PIPELINE']
568 self.project = self.parameters['ZUUL_PROJECT']
James E. Blair3f876d52016-07-22 13:07:14 -0700569 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -0700570 self.wait_condition = threading.Condition()
571 self.waiting = False
572 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -0500573 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -0700574 self.created = time.time()
James E. Blaire1767bc2016-08-02 10:00:27 -0700575 self.changes = None
576 if 'ZUUL_CHANGE_IDS' in self.parameters:
577 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -0700578
James E. Blair3158e282016-08-19 09:34:11 -0700579 def __repr__(self):
580 waiting = ''
581 if self.waiting:
582 waiting = ' [waiting]'
Jamie Lennoxd2e37332016-12-05 15:26:19 +1100583 return '<FakeBuild %s:%s %s%s>' % (self.pipeline, self.name,
584 self.changes, waiting)
James E. Blair3158e282016-08-19 09:34:11 -0700585
Clark Boylanb640e052014-04-03 16:41:46 -0700586 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700587 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -0700588 self.wait_condition.acquire()
589 self.wait_condition.notify()
590 self.waiting = False
591 self.log.debug("Build %s released" % self.unique)
592 self.wait_condition.release()
593
594 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700595 """Return whether this build is being held.
596
597 :returns: Whether the build is being held.
598 :rtype: bool
599 """
600
Clark Boylanb640e052014-04-03 16:41:46 -0700601 self.wait_condition.acquire()
602 if self.waiting:
603 ret = True
604 else:
605 ret = False
606 self.wait_condition.release()
607 return ret
608
609 def _wait(self):
610 self.wait_condition.acquire()
611 self.waiting = True
612 self.log.debug("Build %s waiting" % self.unique)
613 self.wait_condition.wait()
614 self.wait_condition.release()
615
616 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -0700617 self.log.debug('Running build %s' % self.unique)
618
James E. Blaire1767bc2016-08-02 10:00:27 -0700619 if self.launch_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -0700620 self.log.debug('Holding build %s' % self.unique)
621 self._wait()
622 self.log.debug("Build %s continuing" % self.unique)
623
James E. Blair412fba82017-01-26 15:00:50 -0800624 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
James E. Blaira5dba232016-08-08 15:53:24 -0700625 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
James E. Blair412fba82017-01-26 15:00:50 -0800626 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
Clark Boylanb640e052014-04-03 16:41:46 -0700627 if self.aborted:
James E. Blair412fba82017-01-26 15:00:50 -0800628 result = (RecordingAnsibleJob.RESULT_ABORTED, None)
Paul Belanger71d98172016-11-08 10:56:31 -0500629 if self.requeue:
James E. Blair412fba82017-01-26 15:00:50 -0800630 result = (RecordingAnsibleJob.RESULT_UNREACHABLE, None)
Clark Boylanb640e052014-04-03 16:41:46 -0700631
James E. Blaire1767bc2016-08-02 10:00:27 -0700632 return result
Clark Boylanb640e052014-04-03 16:41:46 -0700633
James E. Blaira5dba232016-08-08 15:53:24 -0700634 def shouldFail(self):
635 changes = self.launch_server.fail_tests.get(self.name, [])
636 for change in changes:
637 if self.hasChanges(change):
638 return True
639 return False
640
James E. Blaire7b99a02016-08-05 14:27:34 -0700641 def hasChanges(self, *changes):
642 """Return whether this build has certain changes in its git repos.
643
644 :arg FakeChange changes: One or more changes (varargs) that
645 are expected to be present (in order) in the git repository of
646 the active project.
647
648 :returns: Whether the build has the indicated changes.
649 :rtype: bool
650
651 """
Clint Byrum3343e3e2016-11-15 16:05:03 -0800652 for change in changes:
653 path = os.path.join(self.jobdir.git_root, change.project)
654 try:
655 repo = git.Repo(path)
656 except NoSuchPathError as e:
657 self.log.debug('%s' % e)
658 return False
659 ref = self.parameters['ZUUL_REF']
660 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
661 commit_message = '%s-1' % change.subject
662 self.log.debug("Checking if build %s has changes; commit_message "
663 "%s; repo_messages %s" % (self, commit_message,
664 repo_messages))
665 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -0700666 self.log.debug(" messages do not match")
667 return False
668 self.log.debug(" OK")
669 return True
670
Clark Boylanb640e052014-04-03 16:41:46 -0700671
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +1000672class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
James E. Blaire7b99a02016-08-05 14:27:34 -0700673 """An Ansible launcher to be used in tests.
674
675 :ivar bool hold_jobs_in_build: If true, when jobs are launched
676 they will report that they have started but then pause until
677 released before reporting completion. This attribute may be
678 changed at any time and will take effect for subsequently
679 launched builds, but previously held builds will still need to
680 be explicitly released.
681
682 """
James E. Blairf5dbd002015-12-23 15:26:17 -0800683 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -0700684 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -0800685 self._test_root = kw.pop('_test_root', False)
James E. Blairf5dbd002015-12-23 15:26:17 -0800686 super(RecordingLaunchServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -0700687 self.hold_jobs_in_build = False
688 self.lock = threading.Lock()
689 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -0700690 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -0700691 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -0700692 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -0800693
James E. Blaira5dba232016-08-08 15:53:24 -0700694 def failJob(self, name, change):
James E. Blaire7b99a02016-08-05 14:27:34 -0700695 """Instruct the launcher to report matching builds as failures.
696
697 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -0700698 :arg Change change: The :py:class:`~tests.base.FakeChange`
699 instance which should cause the job to fail. This job
700 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -0700701
702 """
James E. Blaire1767bc2016-08-02 10:00:27 -0700703 l = self.fail_tests.get(name, [])
704 l.append(change)
705 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -0800706
James E. Blair962220f2016-08-03 11:22:38 -0700707 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700708 """Release a held build.
709
710 :arg str regex: A regular expression which, if supplied, will
711 cause only builds with matching names to be released. If
712 not supplied, all builds will be released.
713
714 """
James E. Blair962220f2016-08-03 11:22:38 -0700715 builds = self.running_builds[:]
716 self.log.debug("Releasing build %s (%s)" % (regex,
717 len(self.running_builds)))
718 for build in builds:
719 if not regex or re.match(regex, build.name):
720 self.log.debug("Releasing build %s" %
721 (build.parameters['ZUUL_UUID']))
722 build.release()
723 else:
724 self.log.debug("Not releasing build %s" %
725 (build.parameters['ZUUL_UUID']))
726 self.log.debug("Done releasing builds %s (%s)" %
727 (regex, len(self.running_builds)))
728
James E. Blair17302972016-08-10 16:11:42 -0700729 def launchJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -0700730 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -0700731 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -0700732 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -0700733 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -0800734 args = json.loads(job.arguments)
735 args['zuul']['_test'] = dict(test_root=self._test_root)
736 job.arguments = json.dumps(args)
Joshua Hesketh50c21782016-10-13 21:34:14 +1100737 self.job_workers[job.unique] = RecordingAnsibleJob(self, job)
738 self.job_workers[job.unique].run()
James E. Blair17302972016-08-10 16:11:42 -0700739
740 def stopJob(self, job):
741 self.log.debug("handle stop")
742 parameters = json.loads(job.arguments)
743 uuid = parameters['uuid']
744 for build in self.running_builds:
745 if build.unique == uuid:
746 build.aborted = True
747 build.release()
748 super(RecordingLaunchServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -0700749
Joshua Hesketh50c21782016-10-13 21:34:14 +1100750
751class RecordingAnsibleJob(zuul.launcher.server.AnsibleJob):
James E. Blair412fba82017-01-26 15:00:50 -0800752 def runPlaybooks(self):
Joshua Hesketh50c21782016-10-13 21:34:14 +1100753 build = self.launcher_server.job_builds[self.job.unique]
James E. Blair412fba82017-01-26 15:00:50 -0800754 build.jobdir = self.jobdir
James E. Blaire1767bc2016-08-02 10:00:27 -0700755
James E. Blair412fba82017-01-26 15:00:50 -0800756 result = super(RecordingAnsibleJob, self).runPlaybooks()
757
Joshua Hesketh50c21782016-10-13 21:34:14 +1100758 self.launcher_server.lock.acquire()
759 self.launcher_server.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -0700760 BuildHistory(name=build.name, result=result, changes=build.changes,
761 node=build.node, uuid=build.unique,
762 parameters=build.parameters,
James E. Blaire1767bc2016-08-02 10:00:27 -0700763 pipeline=build.parameters['ZUUL_PIPELINE'])
764 )
Joshua Hesketh50c21782016-10-13 21:34:14 +1100765 self.launcher_server.running_builds.remove(build)
766 del self.launcher_server.job_builds[self.job.unique]
767 self.launcher_server.lock.release()
James E. Blair412fba82017-01-26 15:00:50 -0800768 return result
769
Monty Taylore6562aa2017-02-20 07:37:39 -0500770 def runAnsible(self, cmd, timeout, trusted=False):
James E. Blair412fba82017-01-26 15:00:50 -0800771 build = self.launcher_server.job_builds[self.job.unique]
772
773 if self.launcher_server._run_ansible:
Monty Taylorc231d932017-02-03 09:57:15 -0600774 result = super(RecordingAnsibleJob, self).runAnsible(
Monty Taylore6562aa2017-02-20 07:37:39 -0500775 cmd, timeout, trusted=trusted)
James E. Blair412fba82017-01-26 15:00:50 -0800776 else:
777 result = build.run()
James E. Blaire1767bc2016-08-02 10:00:27 -0700778 return result
James E. Blairf5dbd002015-12-23 15:26:17 -0800779
James E. Blairad8dca02017-02-21 11:48:32 -0500780 def getHostList(self, args):
781 self.log.debug("hostlist")
782 hosts = super(RecordingAnsibleJob, self).getHostList(args)
783 for name, d in hosts:
784 d['ansible_connection'] = 'local'
785 hosts.append(('localhost', dict(ansible_connection='local')))
786 return hosts
787
James E. Blairf5dbd002015-12-23 15:26:17 -0800788
Clark Boylanb640e052014-04-03 16:41:46 -0700789class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -0700790 """A Gearman server for use in tests.
791
792 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
793 added to the queue but will not be distributed to workers
794 until released. This attribute may be changed at any time and
795 will take effect for subsequently enqueued jobs, but
796 previously held jobs will still need to be explicitly
797 released.
798
799 """
800
Clark Boylanb640e052014-04-03 16:41:46 -0700801 def __init__(self):
802 self.hold_jobs_in_queue = False
803 super(FakeGearmanServer, self).__init__(0)
804
805 def getJobForConnection(self, connection, peek=False):
806 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
807 for job in queue:
808 if not hasattr(job, 'waiting'):
Paul Belanger6ab6af72016-11-06 11:32:59 -0500809 if job.name.startswith('launcher:launch'):
Clark Boylanb640e052014-04-03 16:41:46 -0700810 job.waiting = self.hold_jobs_in_queue
811 else:
812 job.waiting = False
813 if job.waiting:
814 continue
815 if job.name in connection.functions:
816 if not peek:
817 queue.remove(job)
818 connection.related_jobs[job.handle] = job
819 job.worker_connection = connection
820 job.running = True
821 return job
822 return None
823
824 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700825 """Release a held job.
826
827 :arg str regex: A regular expression which, if supplied, will
828 cause only jobs with matching names to be released. If
829 not supplied, all jobs will be released.
830 """
Clark Boylanb640e052014-04-03 16:41:46 -0700831 released = False
832 qlen = (len(self.high_queue) + len(self.normal_queue) +
833 len(self.low_queue))
834 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
835 for job in self.getQueue():
Paul Belanger6ab6af72016-11-06 11:32:59 -0500836 if job.name != 'launcher:launch':
Clark Boylanb640e052014-04-03 16:41:46 -0700837 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -0500838 parameters = json.loads(job.arguments)
839 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -0700840 self.log.debug("releasing queued job %s" %
841 job.unique)
842 job.waiting = False
843 released = True
844 else:
845 self.log.debug("not releasing queued job %s" %
846 job.unique)
847 if released:
848 self.wakeConnections()
849 qlen = (len(self.high_queue) + len(self.normal_queue) +
850 len(self.low_queue))
851 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
852
853
854class FakeSMTP(object):
855 log = logging.getLogger('zuul.FakeSMTP')
856
857 def __init__(self, messages, server, port):
858 self.server = server
859 self.port = port
860 self.messages = messages
861
862 def sendmail(self, from_email, to_email, msg):
863 self.log.info("Sending email from %s, to %s, with msg %s" % (
864 from_email, to_email, msg))
865
866 headers = msg.split('\n\n', 1)[0]
867 body = msg.split('\n\n', 1)[1]
868
869 self.messages.append(dict(
870 from_email=from_email,
871 to_email=to_email,
872 msg=msg,
873 headers=headers,
874 body=body,
875 ))
876
877 return True
878
879 def quit(self):
880 return True
881
882
883class FakeSwiftClientConnection(swiftclient.client.Connection):
884 def post_account(self, headers):
885 # Do nothing
886 pass
887
888 def get_auth(self):
889 # Returns endpoint and (unused) auth token
890 endpoint = os.path.join('https://storage.example.org', 'V1',
891 'AUTH_account')
892 return endpoint, ''
893
894
James E. Blairdce6cea2016-12-20 16:45:32 -0800895class FakeNodepool(object):
896 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -0800897 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -0800898
899 log = logging.getLogger("zuul.test.FakeNodepool")
900
901 def __init__(self, host, port, chroot):
902 self.client = kazoo.client.KazooClient(
903 hosts='%s:%s%s' % (host, port, chroot))
904 self.client.start()
905 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -0800906 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -0800907 self.thread = threading.Thread(target=self.run)
908 self.thread.daemon = True
909 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -0800910 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -0800911
912 def stop(self):
913 self._running = False
914 self.thread.join()
915 self.client.stop()
916 self.client.close()
917
918 def run(self):
919 while self._running:
920 self._run()
921 time.sleep(0.1)
922
923 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -0800924 if self.paused:
925 return
James E. Blairdce6cea2016-12-20 16:45:32 -0800926 for req in self.getNodeRequests():
927 self.fulfillRequest(req)
928
929 def getNodeRequests(self):
930 try:
931 reqids = self.client.get_children(self.REQUEST_ROOT)
932 except kazoo.exceptions.NoNodeError:
933 return []
934 reqs = []
935 for oid in sorted(reqids):
936 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -0800937 try:
938 data, stat = self.client.get(path)
939 data = json.loads(data)
940 data['_oid'] = oid
941 reqs.append(data)
942 except kazoo.exceptions.NoNodeError:
943 pass
James E. Blairdce6cea2016-12-20 16:45:32 -0800944 return reqs
945
James E. Blaire18d4602017-01-05 11:17:28 -0800946 def getNodes(self):
947 try:
948 nodeids = self.client.get_children(self.NODE_ROOT)
949 except kazoo.exceptions.NoNodeError:
950 return []
951 nodes = []
952 for oid in sorted(nodeids):
953 path = self.NODE_ROOT + '/' + oid
954 data, stat = self.client.get(path)
955 data = json.loads(data)
956 data['_oid'] = oid
957 try:
958 lockfiles = self.client.get_children(path + '/lock')
959 except kazoo.exceptions.NoNodeError:
960 lockfiles = []
961 if lockfiles:
962 data['_lock'] = True
963 else:
964 data['_lock'] = False
965 nodes.append(data)
966 return nodes
967
James E. Blaira38c28e2017-01-04 10:33:20 -0800968 def makeNode(self, request_id, node_type):
969 now = time.time()
970 path = '/nodepool/nodes/'
971 data = dict(type=node_type,
972 provider='test-provider',
973 region='test-region',
974 az=None,
975 public_ipv4='127.0.0.1',
976 private_ipv4=None,
977 public_ipv6=None,
978 allocated_to=request_id,
979 state='ready',
980 state_time=now,
981 created_time=now,
982 updated_time=now,
983 image_id=None,
984 launcher='fake-nodepool')
985 data = json.dumps(data)
986 path = self.client.create(path, data,
987 makepath=True,
988 sequence=True)
989 nodeid = path.split("/")[-1]
990 return nodeid
991
James E. Blair6ab79e02017-01-06 10:10:17 -0800992 def addFailRequest(self, request):
993 self.fail_requests.add(request['_oid'])
994
James E. Blairdce6cea2016-12-20 16:45:32 -0800995 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -0800996 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -0800997 return
998 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -0800999 oid = request['_oid']
1000 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -08001001
James E. Blair6ab79e02017-01-06 10:10:17 -08001002 if oid in self.fail_requests:
1003 request['state'] = 'failed'
1004 else:
1005 request['state'] = 'fulfilled'
1006 nodes = []
1007 for node in request['node_types']:
1008 nodeid = self.makeNode(oid, node)
1009 nodes.append(nodeid)
1010 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -08001011
James E. Blaira38c28e2017-01-04 10:33:20 -08001012 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -08001013 path = self.REQUEST_ROOT + '/' + oid
1014 data = json.dumps(request)
1015 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
1016 self.client.set(path, data)
1017
1018
James E. Blair498059b2016-12-20 13:50:13 -08001019class ChrootedKazooFixture(fixtures.Fixture):
1020 def __init__(self):
1021 super(ChrootedKazooFixture, self).__init__()
1022
1023 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1024 if ':' in zk_host:
1025 host, port = zk_host.split(':')
1026 else:
1027 host = zk_host
1028 port = None
1029
1030 self.zookeeper_host = host
1031
1032 if not port:
1033 self.zookeeper_port = 2181
1034 else:
1035 self.zookeeper_port = int(port)
1036
1037 def _setUp(self):
1038 # Make sure the test chroot paths do not conflict
1039 random_bits = ''.join(random.choice(string.ascii_lowercase +
1040 string.ascii_uppercase)
1041 for x in range(8))
1042
1043 rand_test_path = '%s_%s' % (random_bits, os.getpid())
1044 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1045
1046 # Ensure the chroot path exists and clean up any pre-existing znodes.
1047 _tmp_client = kazoo.client.KazooClient(
1048 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1049 _tmp_client.start()
1050
1051 if _tmp_client.exists(self.zookeeper_chroot):
1052 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1053
1054 _tmp_client.ensure_path(self.zookeeper_chroot)
1055 _tmp_client.stop()
1056 _tmp_client.close()
1057
1058 self.addCleanup(self._cleanup)
1059
1060 def _cleanup(self):
1061 '''Remove the chroot path.'''
1062 # Need a non-chroot'ed client to remove the chroot path
1063 _tmp_client = kazoo.client.KazooClient(
1064 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1065 _tmp_client.start()
1066 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1067 _tmp_client.stop()
1068
1069
Maru Newby3fe5f852015-01-13 04:22:14 +00001070class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001071 log = logging.getLogger("zuul.test")
1072
James E. Blair1c236df2017-02-01 14:07:24 -08001073 def attachLogs(self, *args):
1074 def reader():
1075 self._log_stream.seek(0)
1076 while True:
1077 x = self._log_stream.read(4096)
1078 if not x:
1079 break
1080 yield x.encode('utf8')
1081 content = testtools.content.content_from_reader(
1082 reader,
1083 testtools.content_type.UTF8_TEXT,
1084 False)
1085 self.addDetail('logging', content)
1086
Clark Boylanb640e052014-04-03 16:41:46 -07001087 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001088 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001089 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1090 try:
1091 test_timeout = int(test_timeout)
1092 except ValueError:
1093 # If timeout value is invalid do not set a timeout.
1094 test_timeout = 0
1095 if test_timeout > 0:
1096 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1097
1098 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1099 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1100 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1101 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1102 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1103 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1104 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1105 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1106 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1107 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001108 self._log_stream = StringIO()
1109 self.addOnException(self.attachLogs)
1110 else:
1111 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001112
James E. Blair1c236df2017-02-01 14:07:24 -08001113 handler = logging.StreamHandler(self._log_stream)
1114 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1115 '%(levelname)-8s %(message)s')
1116 handler.setFormatter(formatter)
1117
1118 logger = logging.getLogger()
1119 logger.setLevel(logging.DEBUG)
1120 logger.addHandler(handler)
1121
1122 # NOTE(notmorgan): Extract logging overrides for specific
1123 # libraries from the OS_LOG_DEFAULTS env and create loggers
1124 # for each. This is used to limit the output during test runs
1125 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001126 log_defaults_from_env = os.environ.get(
1127 'OS_LOG_DEFAULTS',
James E. Blair1c236df2017-02-01 14:07:24 -08001128 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001129
James E. Blairdce6cea2016-12-20 16:45:32 -08001130 if log_defaults_from_env:
1131 for default in log_defaults_from_env.split(','):
1132 try:
1133 name, level_str = default.split('=', 1)
1134 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001135 logger = logging.getLogger(name)
1136 logger.setLevel(level)
1137 logger.addHandler(handler)
1138 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001139 except ValueError:
1140 # NOTE(notmorgan): Invalid format of the log default,
1141 # skip and don't try and apply a logger for the
1142 # specified module
1143 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001144
Maru Newby3fe5f852015-01-13 04:22:14 +00001145
1146class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001147 """A test case with a functioning Zuul.
1148
1149 The following class variables are used during test setup and can
1150 be overidden by subclasses but are effectively read-only once a
1151 test method starts running:
1152
1153 :cvar str config_file: This points to the main zuul config file
1154 within the fixtures directory. Subclasses may override this
1155 to obtain a different behavior.
1156
1157 :cvar str tenant_config_file: This is the tenant config file
1158 (which specifies from what git repos the configuration should
1159 be loaded). It defaults to the value specified in
1160 `config_file` but can be overidden by subclasses to obtain a
1161 different tenant/project layout while using the standard main
1162 configuration.
1163
1164 The following are instance variables that are useful within test
1165 methods:
1166
1167 :ivar FakeGerritConnection fake_<connection>:
1168 A :py:class:`~tests.base.FakeGerritConnection` will be
1169 instantiated for each connection present in the config file
1170 and stored here. For instance, `fake_gerrit` will hold the
1171 FakeGerritConnection object for a connection named `gerrit`.
1172
1173 :ivar FakeGearmanServer gearman_server: An instance of
1174 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1175 server that all of the Zuul components in this test use to
1176 communicate with each other.
1177
1178 :ivar RecordingLaunchServer launch_server: An instance of
1179 :py:class:`~tests.base.RecordingLaunchServer` which is the
1180 Ansible launch server used to run jobs for this test.
1181
1182 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1183 representing currently running builds. They are appended to
1184 the list in the order they are launched, and removed from this
1185 list upon completion.
1186
1187 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1188 objects representing completed builds. They are appended to
1189 the list in the order they complete.
1190
1191 """
1192
James E. Blair83005782015-12-11 14:46:03 -08001193 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001194 run_ansible = False
James E. Blair3f876d52016-07-22 13:07:14 -07001195
1196 def _startMerger(self):
1197 self.merge_server = zuul.merger.server.MergeServer(self.config,
1198 self.connections)
1199 self.merge_server.start()
1200
Maru Newby3fe5f852015-01-13 04:22:14 +00001201 def setUp(self):
1202 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001203
1204 self.setupZK()
1205
James E. Blair97d902e2014-08-21 13:25:56 -07001206 if USE_TEMPDIR:
1207 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001208 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1209 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001210 else:
1211 tmp_root = os.environ.get("ZUUL_TEST_ROOT")
Clark Boylanb640e052014-04-03 16:41:46 -07001212 self.test_root = os.path.join(tmp_root, "zuul-test")
1213 self.upstream_root = os.path.join(self.test_root, "upstream")
James E. Blair8c1be532017-02-07 14:04:12 -08001214 self.merger_git_root = os.path.join(self.test_root, "merger-git")
1215 self.launcher_git_root = os.path.join(self.test_root, "launcher-git")
James E. Blairce8a2132016-05-19 15:21:52 -07001216 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001217
1218 if os.path.exists(self.test_root):
1219 shutil.rmtree(self.test_root)
1220 os.makedirs(self.test_root)
1221 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001222 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001223
1224 # Make per test copy of Configuration.
1225 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001226 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001227 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001228 self.config.get('zuul', 'tenant_config')))
James E. Blair8c1be532017-02-07 14:04:12 -08001229 self.config.set('merger', 'git_dir', self.merger_git_root)
1230 self.config.set('launcher', 'git_dir', self.launcher_git_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001231 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001232
1233 # For each project in config:
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001234 # TODOv3(jeblair): remove these and replace with new git
1235 # filesystem fixtures
Clark Boylanb640e052014-04-03 16:41:46 -07001236 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -07001237 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -07001238 self.init_repo("org/project5")
1239 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -07001240 self.init_repo("org/one-job-project")
1241 self.init_repo("org/nonvoting-project")
1242 self.init_repo("org/templated-project")
1243 self.init_repo("org/layered-project")
1244 self.init_repo("org/node-project")
1245 self.init_repo("org/conflict-project")
1246 self.init_repo("org/noop-project")
1247 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00001248 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -07001249
1250 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001251 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1252 # see: https://github.com/jsocol/pystatsd/issues/61
1253 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001254 os.environ['STATSD_PORT'] = str(self.statsd.port)
1255 self.statsd.start()
1256 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001257 reload_module(statsd)
1258 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001259
1260 self.gearman_server = FakeGearmanServer()
1261
1262 self.config.set('gearman', 'port', str(self.gearman_server.port))
James E. Blaire47eb772017-02-02 17:19:40 -08001263 self.log.info("Gearman server on port %s" %
1264 (self.gearman_server.port,))
Clark Boylanb640e052014-04-03 16:41:46 -07001265
James E. Blaire511d2f2016-12-08 15:22:26 -08001266 gerritsource.GerritSource.replication_timeout = 1.5
1267 gerritsource.GerritSource.replication_retry_interval = 0.5
1268 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001269
Joshua Hesketh352264b2015-08-11 23:42:08 +10001270 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001271
1272 self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
1273 FakeSwiftClientConnection))
James E. Blaire511d2f2016-12-08 15:22:26 -08001274
Clark Boylanb640e052014-04-03 16:41:46 -07001275 self.swift = zuul.lib.swift.Swift(self.config)
1276
Jan Hruban6b71aff2015-10-22 16:58:08 +02001277 self.event_queues = [
1278 self.sched.result_event_queue,
James E. Blair646322f2017-01-27 15:50:34 -08001279 self.sched.trigger_event_queue,
1280 self.sched.management_event_queue
Jan Hruban6b71aff2015-10-22 16:58:08 +02001281 ]
1282
James E. Blairfef78942016-03-11 16:28:56 -08001283 self.configure_connections()
Joshua Hesketh352264b2015-08-11 23:42:08 +10001284 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001285
Clark Boylanb640e052014-04-03 16:41:46 -07001286 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001287 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001288 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001289 return FakeURLOpener(self.upstream_root, *args, **kw)
1290
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001291 old_urlopen = urllib.request.urlopen
1292 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001293
James E. Blair3f876d52016-07-22 13:07:14 -07001294 self._startMerger()
James E. Blair3f876d52016-07-22 13:07:14 -07001295
James E. Blaire1767bc2016-08-02 10:00:27 -07001296 self.launch_server = RecordingLaunchServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001297 self.config, self.connections,
James E. Blair854f8892017-02-02 11:25:39 -08001298 jobdir_root=self.test_root,
James E. Blaira92cbc82017-01-23 14:56:49 -08001299 _run_ansible=self.run_ansible,
1300 _test_root=self.test_root)
James E. Blaire1767bc2016-08-02 10:00:27 -07001301 self.launch_server.start()
1302 self.history = self.launch_server.build_history
1303 self.builds = self.launch_server.running_builds
1304
1305 self.launch_client = zuul.launcher.client.LaunchClient(
James E. Blair82938472016-01-11 14:38:13 -08001306 self.config, self.sched, self.swift)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001307 self.merge_client = zuul.merger.client.MergeClient(
1308 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001309 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001310 self.zk = zuul.zk.ZooKeeper()
James E. Blair0d5a36e2017-02-21 10:53:44 -05001311 self.zk.connect(self.zk_config)
James E. Blairdce6cea2016-12-20 16:45:32 -08001312
James E. Blair0d5a36e2017-02-21 10:53:44 -05001313 self.fake_nodepool = FakeNodepool(
1314 self.zk_chroot_fixture.zookeeper_host,
1315 self.zk_chroot_fixture.zookeeper_port,
1316 self.zk_chroot_fixture.zookeeper_chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001317
James E. Blaire1767bc2016-08-02 10:00:27 -07001318 self.sched.setLauncher(self.launch_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001319 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001320 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001321 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001322
Paul Belanger88ef0ea2015-12-23 11:57:02 -05001323 self.webapp = zuul.webapp.WebApp(
1324 self.sched, port=0, listen_address='127.0.0.1')
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001325 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001326
1327 self.sched.start()
1328 self.sched.reconfigure(self.config)
1329 self.sched.resume()
1330 self.webapp.start()
1331 self.rpc.start()
James E. Blaire1767bc2016-08-02 10:00:27 -07001332 self.launch_client.gearman.waitForServer()
Clark Boylanb640e052014-04-03 16:41:46 -07001333
Clark Boylanb640e052014-04-03 16:41:46 -07001334 self.addCleanup(self.shutdown)
1335
James E. Blaire18d4602017-01-05 11:17:28 -08001336 def tearDown(self):
1337 super(ZuulTestCase, self).tearDown()
1338 self.assertFinalState()
1339
James E. Blairfef78942016-03-11 16:28:56 -08001340 def configure_connections(self):
James E. Blaire511d2f2016-12-08 15:22:26 -08001341 # Set up gerrit related fakes
1342 # Set a changes database so multiple FakeGerrit's can report back to
1343 # a virtual canonical database given by the configured hostname
1344 self.gerrit_changes_dbs = {}
1345
1346 def getGerritConnection(driver, name, config):
1347 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1348 con = FakeGerritConnection(driver, name, config,
1349 changes_db=db,
1350 upstream_root=self.upstream_root)
1351 self.event_queues.append(con.event_queue)
1352 setattr(self, 'fake_' + name, con)
1353 return con
1354
1355 self.useFixture(fixtures.MonkeyPatch(
1356 'zuul.driver.gerrit.GerritDriver.getConnection',
1357 getGerritConnection))
1358
1359 # Set up smtp related fakes
Joshua Hesketh352264b2015-08-11 23:42:08 +10001360 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001361
Joshua Hesketh352264b2015-08-11 23:42:08 +10001362 def FakeSMTPFactory(*args, **kw):
1363 args = [self.smtp_messages] + list(args)
1364 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001365
Joshua Hesketh352264b2015-08-11 23:42:08 +10001366 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001367
James E. Blaire511d2f2016-12-08 15:22:26 -08001368 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001369 self.connections = zuul.lib.connections.ConnectionRegistry()
James E. Blaire511d2f2016-12-08 15:22:26 -08001370 self.connections.configure(self.config)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001371
James E. Blair83005782015-12-11 14:46:03 -08001372 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001373 # This creates the per-test configuration object. It can be
1374 # overriden by subclasses, but should not need to be since it
1375 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001376 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001377 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair2a629ec2015-12-22 15:32:02 -08001378 if hasattr(self, 'tenant_config_file'):
1379 self.config.set('zuul', 'tenant_config', self.tenant_config_file)
James E. Blair96c6bf82016-01-15 16:20:40 -08001380 git_path = os.path.join(
1381 os.path.dirname(
1382 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1383 'git')
1384 if os.path.exists(git_path):
1385 for reponame in os.listdir(git_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001386 project = reponame.replace('_', '/')
1387 self.copyDirToRepo(project,
James E. Blair96c6bf82016-01-15 16:20:40 -08001388 os.path.join(git_path, reponame))
1389
James E. Blair498059b2016-12-20 13:50:13 -08001390 def setupZK(self):
1391 self.zk_chroot_fixture = self.useFixture(ChrootedKazooFixture())
James E. Blair0d5a36e2017-02-21 10:53:44 -05001392 self.zk_config = '%s:%s%s' % (
James E. Blairdce6cea2016-12-20 16:45:32 -08001393 self.zk_chroot_fixture.zookeeper_host,
1394 self.zk_chroot_fixture.zookeeper_port,
1395 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08001396
James E. Blair96c6bf82016-01-15 16:20:40 -08001397 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001398 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08001399
1400 files = {}
1401 for (dirpath, dirnames, filenames) in os.walk(source_path):
1402 for filename in filenames:
1403 test_tree_filepath = os.path.join(dirpath, filename)
1404 common_path = os.path.commonprefix([test_tree_filepath,
1405 source_path])
1406 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1407 with open(test_tree_filepath, 'r') as f:
1408 content = f.read()
1409 files[relative_filepath] = content
1410 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001411 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08001412
James E. Blaire18d4602017-01-05 11:17:28 -08001413 def assertNodepoolState(self):
1414 # Make sure that there are no pending requests
1415
1416 requests = self.fake_nodepool.getNodeRequests()
1417 self.assertEqual(len(requests), 0)
1418
1419 nodes = self.fake_nodepool.getNodes()
1420 for node in nodes:
1421 self.assertFalse(node['_lock'], "Node %s is locked" %
1422 (node['_oid'],))
1423
Clark Boylanb640e052014-04-03 16:41:46 -07001424 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001425 # Make sure that git.Repo objects have been garbage collected.
1426 repos = []
1427 gc.collect()
1428 for obj in gc.get_objects():
1429 if isinstance(obj, git.Repo):
1430 repos.append(obj)
1431 self.assertEqual(len(repos), 0)
1432 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08001433 self.assertNodepoolState()
James E. Blair83005782015-12-11 14:46:03 -08001434 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001435 for tenant in self.sched.abide.tenants.values():
1436 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001437 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001438 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001439
1440 def shutdown(self):
1441 self.log.debug("Shutting down after tests")
James E. Blaire1767bc2016-08-02 10:00:27 -07001442 self.launch_client.stop()
James E. Blair3f876d52016-07-22 13:07:14 -07001443 self.merge_server.stop()
1444 self.merge_server.join()
Clark Boylanb640e052014-04-03 16:41:46 -07001445 self.merge_client.stop()
James E. Blaire1767bc2016-08-02 10:00:27 -07001446 self.launch_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001447 self.sched.stop()
1448 self.sched.join()
1449 self.statsd.stop()
1450 self.statsd.join()
1451 self.webapp.stop()
1452 self.webapp.join()
1453 self.rpc.stop()
1454 self.rpc.join()
1455 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08001456 self.fake_nodepool.stop()
1457 self.zk.disconnect()
Clark Boylanb640e052014-04-03 16:41:46 -07001458 threads = threading.enumerate()
1459 if len(threads) > 1:
1460 self.log.error("More than one thread is running: %s" % threads)
James E. Blair6ac368c2016-12-22 18:07:20 -08001461 self.printHistory()
Clark Boylanb640e052014-04-03 16:41:46 -07001462
1463 def init_repo(self, project):
1464 parts = project.split('/')
1465 path = os.path.join(self.upstream_root, *parts[:-1])
1466 if not os.path.exists(path):
1467 os.makedirs(path)
1468 path = os.path.join(self.upstream_root, project)
1469 repo = git.Repo.init(path)
1470
Morgan Fainberg78c301a2016-07-14 13:47:01 -07001471 with repo.config_writer() as config_writer:
1472 config_writer.set_value('user', 'email', 'user@example.com')
1473 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07001474
Clark Boylanb640e052014-04-03 16:41:46 -07001475 repo.index.commit('initial commit')
1476 master = repo.create_head('master')
Clark Boylanb640e052014-04-03 16:41:46 -07001477
James E. Blair97d902e2014-08-21 13:25:56 -07001478 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001479 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001480 repo.git.clean('-x', '-f', '-d')
1481
James E. Blair97d902e2014-08-21 13:25:56 -07001482 def create_branch(self, project, branch):
1483 path = os.path.join(self.upstream_root, project)
1484 repo = git.Repo.init(path)
1485 fn = os.path.join(path, 'README')
1486
1487 branch_head = repo.create_head(branch)
1488 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001489 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001490 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001491 f.close()
1492 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001493 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001494
James E. Blair97d902e2014-08-21 13:25:56 -07001495 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001496 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001497 repo.git.clean('-x', '-f', '-d')
1498
Sachi King9f16d522016-03-16 12:20:45 +11001499 def create_commit(self, project):
1500 path = os.path.join(self.upstream_root, project)
1501 repo = git.Repo(path)
1502 repo.head.reference = repo.heads['master']
1503 file_name = os.path.join(path, 'README')
1504 with open(file_name, 'a') as f:
1505 f.write('creating fake commit\n')
1506 repo.index.add([file_name])
1507 commit = repo.index.commit('Creating a fake commit')
1508 return commit.hexsha
1509
James E. Blairb8c16472015-05-05 14:55:26 -07001510 def orderedRelease(self):
1511 # Run one build at a time to ensure non-race order:
1512 while len(self.builds):
1513 self.release(self.builds[0])
1514 self.waitUntilSettled()
1515
Clark Boylanb640e052014-04-03 16:41:46 -07001516 def release(self, job):
1517 if isinstance(job, FakeBuild):
1518 job.release()
1519 else:
1520 job.waiting = False
1521 self.log.debug("Queued job %s released" % job.unique)
1522 self.gearman_server.wakeConnections()
1523
1524 def getParameter(self, job, name):
1525 if isinstance(job, FakeBuild):
1526 return job.parameters[name]
1527 else:
1528 parameters = json.loads(job.arguments)
1529 return parameters[name]
1530
Clark Boylanb640e052014-04-03 16:41:46 -07001531 def haveAllBuildsReported(self):
1532 # See if Zuul is waiting on a meta job to complete
James E. Blaire1767bc2016-08-02 10:00:27 -07001533 if self.launch_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07001534 return False
1535 # Find out if every build that the worker has completed has been
1536 # reported back to Zuul. If it hasn't then that means a Gearman
1537 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07001538 for build in self.history:
James E. Blaire1767bc2016-08-02 10:00:27 -07001539 zbuild = self.launch_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07001540 if not zbuild:
1541 # It has already been reported
1542 continue
1543 # It hasn't been reported yet.
1544 return False
1545 # Make sure that none of the worker connections are in GRAB_WAIT
James E. Blaire1767bc2016-08-02 10:00:27 -07001546 for connection in self.launch_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001547 if connection.state == 'GRAB_WAIT':
1548 return False
1549 return True
1550
1551 def areAllBuildsWaiting(self):
James E. Blaire1767bc2016-08-02 10:00:27 -07001552 builds = self.launch_client.builds.values()
Clark Boylanb640e052014-04-03 16:41:46 -07001553 for build in builds:
1554 client_job = None
James E. Blaire1767bc2016-08-02 10:00:27 -07001555 for conn in self.launch_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001556 for j in conn.related_jobs.values():
1557 if j.unique == build.uuid:
1558 client_job = j
1559 break
1560 if not client_job:
1561 self.log.debug("%s is not known to the gearman client" %
1562 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001563 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001564 if not client_job.handle:
1565 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001566 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001567 server_job = self.gearman_server.jobs.get(client_job.handle)
1568 if not server_job:
1569 self.log.debug("%s is not known to the gearman server" %
1570 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001571 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001572 if not hasattr(server_job, 'waiting'):
1573 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001574 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001575 if server_job.waiting:
1576 continue
James E. Blair17302972016-08-10 16:11:42 -07001577 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08001578 self.log.debug("%s has not reported start" % build)
1579 return False
James E. Blairab7132b2016-08-05 12:36:22 -07001580 worker_build = self.launch_server.job_builds.get(server_job.unique)
James E. Blair962220f2016-08-03 11:22:38 -07001581 if worker_build:
1582 if worker_build.isWaiting():
1583 continue
1584 else:
1585 self.log.debug("%s is running" % worker_build)
1586 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001587 else:
James E. Blair962220f2016-08-03 11:22:38 -07001588 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001589 return False
1590 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001591
James E. Blairdce6cea2016-12-20 16:45:32 -08001592 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001593 if self.fake_nodepool.paused:
1594 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08001595 if self.sched.nodepool.requests:
1596 return False
1597 return True
1598
Jan Hruban6b71aff2015-10-22 16:58:08 +02001599 def eventQueuesEmpty(self):
1600 for queue in self.event_queues:
1601 yield queue.empty()
1602
1603 def eventQueuesJoin(self):
1604 for queue in self.event_queues:
1605 queue.join()
1606
Clark Boylanb640e052014-04-03 16:41:46 -07001607 def waitUntilSettled(self):
1608 self.log.debug("Waiting until settled...")
1609 start = time.time()
1610 while True:
James E. Blair71932482017-02-02 11:29:07 -08001611 if time.time() - start > 20:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001612 self.log.error("Timeout waiting for Zuul to settle")
1613 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07001614 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001615 self.log.error(" %s: %s" % (queue, queue.empty()))
1616 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07001617 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001618 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07001619 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001620 self.log.error("All requests completed: %s" %
1621 (self.areAllNodeRequestsComplete(),))
1622 self.log.error("Merge client jobs: %s" %
1623 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07001624 raise Exception("Timeout waiting for Zuul to settle")
1625 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07001626
James E. Blaire1767bc2016-08-02 10:00:27 -07001627 self.launch_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07001628 # have all build states propogated to zuul?
1629 if self.haveAllBuildsReported():
1630 # Join ensures that the queue is empty _and_ events have been
1631 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001632 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001633 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001634 if (not self.merge_client.jobs and
Clark Boylanb640e052014-04-03 16:41:46 -07001635 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08001636 self.areAllBuildsWaiting() and
James E. Blair36c611a2017-02-06 15:59:43 -08001637 self.areAllNodeRequestsComplete() and
1638 all(self.eventQueuesEmpty())):
1639 # The queue empty check is placed at the end to
1640 # ensure that if a component adds an event between
1641 # when locked the run handler and checked that the
1642 # components were stable, we don't erroneously
1643 # report that we are settled.
Clark Boylanb640e052014-04-03 16:41:46 -07001644 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001645 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001646 self.log.debug("...settled.")
1647 return
1648 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001649 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001650 self.sched.wake_event.wait(0.1)
1651
1652 def countJobResults(self, jobs, result):
1653 jobs = filter(lambda x: x.result == result, jobs)
1654 return len(jobs)
1655
James E. Blair96c6bf82016-01-15 16:20:40 -08001656 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07001657 for job in self.history:
1658 if (job.name == name and
1659 (project is None or
1660 job.parameters['ZUUL_PROJECT'] == project)):
1661 return job
Clark Boylanb640e052014-04-03 16:41:46 -07001662 raise Exception("Unable to find job %s in history" % name)
1663
1664 def assertEmptyQueues(self):
1665 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001666 for tenant in self.sched.abide.tenants.values():
1667 for pipeline in tenant.layout.pipelines.values():
1668 for queue in pipeline.queues:
1669 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10001670 print('pipeline %s queue %s contents %s' % (
1671 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08001672 self.assertEqual(len(queue.queue), 0,
1673 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001674
1675 def assertReportedStat(self, key, value=None, kind=None):
1676 start = time.time()
1677 while time.time() < (start + 5):
1678 for stat in self.statsd.stats:
Clark Boylanb640e052014-04-03 16:41:46 -07001679 k, v = stat.split(':')
1680 if key == k:
1681 if value is None and kind is None:
1682 return
1683 elif value:
1684 if value == v:
1685 return
1686 elif kind:
1687 if v.endswith('|' + kind):
1688 return
1689 time.sleep(0.1)
1690
Clark Boylanb640e052014-04-03 16:41:46 -07001691 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001692
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001693 def assertBuilds(self, builds):
1694 """Assert that the running builds are as described.
1695
1696 The list of running builds is examined and must match exactly
1697 the list of builds described by the input.
1698
1699 :arg list builds: A list of dictionaries. Each item in the
1700 list must match the corresponding build in the build
1701 history, and each element of the dictionary must match the
1702 corresponding attribute of the build.
1703
1704 """
James E. Blair3158e282016-08-19 09:34:11 -07001705 try:
1706 self.assertEqual(len(self.builds), len(builds))
1707 for i, d in enumerate(builds):
1708 for k, v in d.items():
1709 self.assertEqual(
1710 getattr(self.builds[i], k), v,
1711 "Element %i in builds does not match" % (i,))
1712 except Exception:
1713 for build in self.builds:
1714 self.log.error("Running build: %s" % build)
1715 else:
1716 self.log.error("No running builds")
1717 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001718
James E. Blairb536ecc2016-08-31 10:11:42 -07001719 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001720 """Assert that the completed builds are as described.
1721
1722 The list of completed builds is examined and must match
1723 exactly the list of builds described by the input.
1724
1725 :arg list history: A list of dictionaries. Each item in the
1726 list must match the corresponding build in the build
1727 history, and each element of the dictionary must match the
1728 corresponding attribute of the build.
1729
James E. Blairb536ecc2016-08-31 10:11:42 -07001730 :arg bool ordered: If true, the history must match the order
1731 supplied, if false, the builds are permitted to have
1732 arrived in any order.
1733
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001734 """
James E. Blairb536ecc2016-08-31 10:11:42 -07001735 def matches(history_item, item):
1736 for k, v in item.items():
1737 if getattr(history_item, k) != v:
1738 return False
1739 return True
James E. Blair3158e282016-08-19 09:34:11 -07001740 try:
1741 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07001742 if ordered:
1743 for i, d in enumerate(history):
1744 if not matches(self.history[i], d):
1745 raise Exception(
1746 "Element %i in history does not match" % (i,))
1747 else:
1748 unseen = self.history[:]
1749 for i, d in enumerate(history):
1750 found = False
1751 for unseen_item in unseen:
1752 if matches(unseen_item, d):
1753 found = True
1754 unseen.remove(unseen_item)
1755 break
1756 if not found:
1757 raise Exception("No match found for element %i "
1758 "in history" % (i,))
1759 if unseen:
1760 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07001761 except Exception:
1762 for build in self.history:
1763 self.log.error("Completed build: %s" % build)
1764 else:
1765 self.log.error("No completed builds")
1766 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001767
James E. Blair6ac368c2016-12-22 18:07:20 -08001768 def printHistory(self):
1769 """Log the build history.
1770
1771 This can be useful during tests to summarize what jobs have
1772 completed.
1773
1774 """
1775 self.log.debug("Build history:")
1776 for build in self.history:
1777 self.log.debug(build)
1778
James E. Blair59fdbac2015-12-07 17:08:06 -08001779 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001780 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1781
1782 def updateConfigLayout(self, path):
1783 root = os.path.join(self.test_root, "config")
Clint Byrumead6c562017-02-01 16:34:04 -08001784 if not os.path.exists(root):
1785 os.makedirs(root)
James E. Blairf84026c2015-12-08 16:11:46 -08001786 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1787 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05001788- tenant:
1789 name: openstack
1790 source:
1791 gerrit:
1792 config-repos:
1793 - %s
1794 """ % path)
James E. Blairf84026c2015-12-08 16:11:46 -08001795 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05001796 self.config.set('zuul', 'tenant_config',
1797 os.path.join(FIXTURE_DIR, f.name))
James E. Blair14abdf42015-12-09 16:11:53 -08001798
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001799 def addCommitToRepo(self, project, message, files,
1800 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08001801 path = os.path.join(self.upstream_root, project)
1802 repo = git.Repo(path)
1803 repo.head.reference = branch
1804 zuul.merger.merger.reset_repo_to_head(repo)
1805 for fn, content in files.items():
1806 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08001807 try:
1808 os.makedirs(os.path.dirname(fn))
1809 except OSError:
1810 pass
James E. Blair14abdf42015-12-09 16:11:53 -08001811 with open(fn, 'w') as f:
1812 f.write(content)
1813 repo.index.add([fn])
1814 commit = repo.index.commit(message)
1815 repo.heads[branch].commit = commit
1816 repo.head.reference = branch
1817 repo.git.clean('-x', '-f', '-d')
1818 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001819 if tag:
1820 repo.create_tag(tag)
James E. Blair3f876d52016-07-22 13:07:14 -07001821
James E. Blair7fc8daa2016-08-08 15:37:15 -07001822 def addEvent(self, connection, event):
1823 """Inject a Fake (Gerrit) event.
1824
1825 This method accepts a JSON-encoded event and simulates Zuul
1826 having received it from Gerrit. It could (and should)
1827 eventually apply to any connection type, but is currently only
1828 used with Gerrit connections. The name of the connection is
1829 used to look up the corresponding server, and the event is
1830 simulated as having been received by all Zuul connections
1831 attached to that server. So if two Gerrit connections in Zuul
1832 are connected to the same Gerrit server, and you invoke this
1833 method specifying the name of one of them, the event will be
1834 received by both.
1835
1836 .. note::
1837
1838 "self.fake_gerrit.addEvent" calls should be migrated to
1839 this method.
1840
1841 :arg str connection: The name of the connection corresponding
1842 to the gerrit server.
1843 :arg str event: The JSON-encoded event.
1844
1845 """
1846 specified_conn = self.connections.connections[connection]
1847 for conn in self.connections.connections.values():
1848 if (isinstance(conn, specified_conn.__class__) and
1849 specified_conn.server == conn.server):
1850 conn.addEvent(event)
1851
James E. Blair3f876d52016-07-22 13:07:14 -07001852
1853class AnsibleZuulTestCase(ZuulTestCase):
1854 """ZuulTestCase but with an actual ansible launcher running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07001855 run_ansible = True