blob: 1c3a86f1b398d64510caf0e68f16676d08cb9bd3 [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
James E. Blair498059b2016-12-20 13:50:13 -08004# Copyright 2016 Red Hat, Inc.
Clark Boylanb640e052014-04-03 16:41:46 -07005#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
Christian Berendtffba5df2014-06-07 21:30:22 +020018from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070019import gc
20import hashlib
21import json
22import logging
23import os
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Morgan Fainberg293f7f82016-05-30 14:01:22 -070025from six.moves import urllib
Clark Boylanb640e052014-04-03 16:41:46 -070026import random
27import re
28import select
29import shutil
Monty Taylor74fa3862016-06-02 07:39:49 +030030from six.moves import reload_module
James E. Blair1c236df2017-02-01 14:07:24 -080031from six import StringIO
Clark Boylanb640e052014-04-03 16:41:46 -070032import socket
33import string
34import subprocess
35import swiftclient
James E. Blair1c236df2017-02-01 14:07:24 -080036import sys
James E. Blairf84026c2015-12-08 16:11:46 -080037import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070038import threading
39import time
Clark Boylanb640e052014-04-03 16:41:46 -070040
41import git
42import gear
43import fixtures
James E. Blair498059b2016-12-20 13:50:13 -080044import kazoo.client
James E. Blairdce6cea2016-12-20 16:45:32 -080045import kazoo.exceptions
Clark Boylanb640e052014-04-03 16:41:46 -070046import statsd
47import testtools
James E. Blair1c236df2017-02-01 14:07:24 -080048import testtools.content
49import testtools.content_type
Clint Byrum3343e3e2016-11-15 16:05:03 -080050from git.exc import NoSuchPathError
Clark Boylanb640e052014-04-03 16:41:46 -070051
James E. Blaire511d2f2016-12-08 15:22:26 -080052import zuul.driver.gerrit.gerritsource as gerritsource
53import zuul.driver.gerrit.gerritconnection as gerritconnection
Clark Boylanb640e052014-04-03 16:41:46 -070054import zuul.scheduler
55import zuul.webapp
56import zuul.rpclistener
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +100057import zuul.launcher.server
58import zuul.launcher.client
Clark Boylanb640e052014-04-03 16:41:46 -070059import zuul.lib.swift
James E. Blair83005782015-12-11 14:46:03 -080060import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070061import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070062import zuul.merger.merger
63import zuul.merger.server
James E. Blair8d692392016-04-08 17:47:58 -070064import zuul.nodepool
James E. Blairdce6cea2016-12-20 16:45:32 -080065import zuul.zk
Clark Boylanb640e052014-04-03 16:41:46 -070066
67FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
68 'fixtures')
James E. Blair97d902e2014-08-21 13:25:56 -070069USE_TEMPDIR = True
Clark Boylanb640e052014-04-03 16:41:46 -070070
Clark Boylanb640e052014-04-03 16:41:46 -070071
72def repack_repo(path):
73 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
74 output = subprocess.Popen(cmd, close_fds=True,
75 stdout=subprocess.PIPE,
76 stderr=subprocess.PIPE)
77 out = output.communicate()
78 if output.returncode:
79 raise Exception("git repack returned %d" % output.returncode)
80 return out
81
82
83def random_sha1():
84 return hashlib.sha1(str(random.random())).hexdigest()
85
86
James E. Blaira190f3b2015-01-05 14:56:54 -080087def iterate_timeout(max_seconds, purpose):
88 start = time.time()
89 count = 0
90 while (time.time() < start + max_seconds):
91 count += 1
92 yield count
93 time.sleep(0)
94 raise Exception("Timeout waiting for %s" % purpose)
95
96
Clark Boylanb640e052014-04-03 16:41:46 -070097class ChangeReference(git.Reference):
98 _common_path_default = "refs/changes"
99 _points_to_commits_only = True
100
101
102class FakeChange(object):
James E. Blair8b5408c2016-08-08 15:37:46 -0700103 categories = {'approved': ('Approved', -1, 1),
104 'code-review': ('Code-Review', -2, 2),
105 'verified': ('Verified', -2, 2)}
Clark Boylanb640e052014-04-03 16:41:46 -0700106
107 def __init__(self, gerrit, number, project, branch, subject,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700108 status='NEW', upstream_root=None, files={}):
Clark Boylanb640e052014-04-03 16:41:46 -0700109 self.gerrit = gerrit
110 self.reported = 0
111 self.queried = 0
112 self.patchsets = []
113 self.number = number
114 self.project = project
115 self.branch = branch
116 self.subject = subject
117 self.latest_patchset = 0
118 self.depends_on_change = None
119 self.needed_by_changes = []
120 self.fail_merge = False
121 self.messages = []
122 self.data = {
123 'branch': branch,
124 'comments': [],
125 'commitMessage': subject,
126 'createdOn': time.time(),
127 'id': 'I' + random_sha1(),
128 'lastUpdated': time.time(),
129 'number': str(number),
130 'open': status == 'NEW',
131 'owner': {'email': 'user@example.com',
132 'name': 'User Name',
133 'username': 'username'},
134 'patchSets': self.patchsets,
135 'project': project,
136 'status': status,
137 'subject': subject,
138 'submitRecords': [],
139 'url': 'https://hostname/%s' % number}
140
141 self.upstream_root = upstream_root
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700142 self.addPatchset(files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700143 self.data['submitRecords'] = self.getSubmitRecords()
144 self.open = status == 'NEW'
145
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700146 def addFakeChangeToRepo(self, msg, files, large):
Clark Boylanb640e052014-04-03 16:41:46 -0700147 path = os.path.join(self.upstream_root, self.project)
148 repo = git.Repo(path)
149 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
150 self.latest_patchset),
151 'refs/tags/init')
152 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700153 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700154 repo.git.clean('-x', '-f', '-d')
155
156 path = os.path.join(self.upstream_root, self.project)
157 if not large:
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700158 for fn, content in files.items():
159 fn = os.path.join(path, fn)
160 with open(fn, 'w') as f:
161 f.write(content)
162 repo.index.add([fn])
Clark Boylanb640e052014-04-03 16:41:46 -0700163 else:
164 for fni in range(100):
165 fn = os.path.join(path, str(fni))
166 f = open(fn, 'w')
167 for ci in range(4096):
168 f.write(random.choice(string.printable))
169 f.close()
170 repo.index.add([fn])
171
172 r = repo.index.commit(msg)
173 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700174 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700175 repo.git.clean('-x', '-f', '-d')
176 repo.heads['master'].checkout()
177 return r
178
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700179 def addPatchset(self, files=None, large=False):
Clark Boylanb640e052014-04-03 16:41:46 -0700180 self.latest_patchset += 1
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700181 if not files:
James E. Blair97d902e2014-08-21 13:25:56 -0700182 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700183 data = ("test %s %s %s\n" %
184 (self.branch, self.number, self.latest_patchset))
185 files = {fn: data}
Clark Boylanb640e052014-04-03 16:41:46 -0700186 msg = self.subject + '-' + str(self.latest_patchset)
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700187 c = self.addFakeChangeToRepo(msg, files, large)
Clark Boylanb640e052014-04-03 16:41:46 -0700188 ps_files = [{'file': '/COMMIT_MSG',
189 'type': 'ADDED'},
190 {'file': 'README',
191 'type': 'MODIFIED'}]
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700192 for f in files.keys():
Clark Boylanb640e052014-04-03 16:41:46 -0700193 ps_files.append({'file': f, 'type': 'ADDED'})
194 d = {'approvals': [],
195 'createdOn': time.time(),
196 'files': ps_files,
197 'number': str(self.latest_patchset),
198 'ref': 'refs/changes/1/%s/%s' % (self.number,
199 self.latest_patchset),
200 'revision': c.hexsha,
201 'uploader': {'email': 'user@example.com',
202 'name': 'User name',
203 'username': 'user'}}
204 self.data['currentPatchSet'] = d
205 self.patchsets.append(d)
206 self.data['submitRecords'] = self.getSubmitRecords()
207
208 def getPatchsetCreatedEvent(self, patchset):
209 event = {"type": "patchset-created",
210 "change": {"project": self.project,
211 "branch": self.branch,
212 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
213 "number": str(self.number),
214 "subject": self.subject,
215 "owner": {"name": "User Name"},
216 "url": "https://hostname/3"},
217 "patchSet": self.patchsets[patchset - 1],
218 "uploader": {"name": "User Name"}}
219 return event
220
221 def getChangeRestoredEvent(self):
222 event = {"type": "change-restored",
223 "change": {"project": self.project,
224 "branch": self.branch,
225 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
226 "number": str(self.number),
227 "subject": self.subject,
228 "owner": {"name": "User Name"},
229 "url": "https://hostname/3"},
230 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100231 "patchSet": self.patchsets[-1],
232 "reason": ""}
233 return event
234
235 def getChangeAbandonedEvent(self):
236 event = {"type": "change-abandoned",
237 "change": {"project": self.project,
238 "branch": self.branch,
239 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
240 "number": str(self.number),
241 "subject": self.subject,
242 "owner": {"name": "User Name"},
243 "url": "https://hostname/3"},
244 "abandoner": {"name": "User Name"},
245 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700246 "reason": ""}
247 return event
248
249 def getChangeCommentEvent(self, patchset):
250 event = {"type": "comment-added",
251 "change": {"project": self.project,
252 "branch": self.branch,
253 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
254 "number": str(self.number),
255 "subject": self.subject,
256 "owner": {"name": "User Name"},
257 "url": "https://hostname/3"},
258 "patchSet": self.patchsets[patchset - 1],
259 "author": {"name": "User Name"},
James E. Blair8b5408c2016-08-08 15:37:46 -0700260 "approvals": [{"type": "code-review",
Clark Boylanb640e052014-04-03 16:41:46 -0700261 "description": "Code-Review",
262 "value": "0"}],
263 "comment": "This is a comment"}
264 return event
265
Joshua Hesketh642824b2014-07-01 17:54:59 +1000266 def addApproval(self, category, value, username='reviewer_john',
267 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700268 if not granted_on:
269 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000270 approval = {
271 'description': self.categories[category][0],
272 'type': category,
273 'value': str(value),
274 'by': {
275 'username': username,
276 'email': username + '@example.com',
277 },
278 'grantedOn': int(granted_on)
279 }
Clark Boylanb640e052014-04-03 16:41:46 -0700280 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
281 if x['by']['username'] == username and x['type'] == category:
282 del self.patchsets[-1]['approvals'][i]
283 self.patchsets[-1]['approvals'].append(approval)
284 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000285 'author': {'email': 'author@example.com',
286 'name': 'Patchset Author',
287 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700288 'change': {'branch': self.branch,
289 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
290 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000291 'owner': {'email': 'owner@example.com',
292 'name': 'Change Owner',
293 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700294 'project': self.project,
295 'subject': self.subject,
296 'topic': 'master',
297 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000298 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700299 'patchSet': self.patchsets[-1],
300 'type': 'comment-added'}
301 self.data['submitRecords'] = self.getSubmitRecords()
302 return json.loads(json.dumps(event))
303
304 def getSubmitRecords(self):
305 status = {}
306 for cat in self.categories.keys():
307 status[cat] = 0
308
309 for a in self.patchsets[-1]['approvals']:
310 cur = status[a['type']]
311 cat_min, cat_max = self.categories[a['type']][1:]
312 new = int(a['value'])
313 if new == cat_min:
314 cur = new
315 elif abs(new) > abs(cur):
316 cur = new
317 status[a['type']] = cur
318
319 labels = []
320 ok = True
321 for typ, cat in self.categories.items():
322 cur = status[typ]
323 cat_min, cat_max = cat[1:]
324 if cur == cat_min:
325 value = 'REJECT'
326 ok = False
327 elif cur == cat_max:
328 value = 'OK'
329 else:
330 value = 'NEED'
331 ok = False
332 labels.append({'label': cat[0], 'status': value})
333 if ok:
334 return [{'status': 'OK'}]
335 return [{'status': 'NOT_READY',
336 'labels': labels}]
337
338 def setDependsOn(self, other, patchset):
339 self.depends_on_change = other
340 d = {'id': other.data['id'],
341 'number': other.data['number'],
342 'ref': other.patchsets[patchset - 1]['ref']
343 }
344 self.data['dependsOn'] = [d]
345
346 other.needed_by_changes.append(self)
347 needed = other.data.get('neededBy', [])
348 d = {'id': self.data['id'],
349 'number': self.data['number'],
350 'ref': self.patchsets[patchset - 1]['ref'],
351 'revision': self.patchsets[patchset - 1]['revision']
352 }
353 needed.append(d)
354 other.data['neededBy'] = needed
355
356 def query(self):
357 self.queried += 1
358 d = self.data.get('dependsOn')
359 if d:
360 d = d[0]
361 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
362 d['isCurrentPatchSet'] = True
363 else:
364 d['isCurrentPatchSet'] = False
365 return json.loads(json.dumps(self.data))
366
367 def setMerged(self):
368 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000369 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700370 return
371 if self.fail_merge:
372 return
373 self.data['status'] = 'MERGED'
374 self.open = False
375
376 path = os.path.join(self.upstream_root, self.project)
377 repo = git.Repo(path)
378 repo.heads[self.branch].commit = \
379 repo.commit(self.patchsets[-1]['revision'])
380
381 def setReported(self):
382 self.reported += 1
383
384
James E. Blaire511d2f2016-12-08 15:22:26 -0800385class FakeGerritConnection(gerritconnection.GerritConnection):
James E. Blaire7b99a02016-08-05 14:27:34 -0700386 """A Fake Gerrit connection for use in tests.
387
388 This subclasses
389 :py:class:`~zuul.connection.gerrit.GerritConnection` to add the
390 ability for tests to add changes to the fake Gerrit it represents.
391 """
392
Joshua Hesketh352264b2015-08-11 23:42:08 +1000393 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700394
James E. Blaire511d2f2016-12-08 15:22:26 -0800395 def __init__(self, driver, connection_name, connection_config,
James E. Blair7fc8daa2016-08-08 15:37:15 -0700396 changes_db=None, upstream_root=None):
James E. Blaire511d2f2016-12-08 15:22:26 -0800397 super(FakeGerritConnection, self).__init__(driver, connection_name,
Joshua Hesketh352264b2015-08-11 23:42:08 +1000398 connection_config)
399
James E. Blair7fc8daa2016-08-08 15:37:15 -0700400 self.event_queue = Queue.Queue()
Clark Boylanb640e052014-04-03 16:41:46 -0700401 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
402 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000403 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700404 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200405 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700406
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700407 def addFakeChange(self, project, branch, subject, status='NEW',
408 files=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700409 """Add a change to the fake Gerrit."""
Clark Boylanb640e052014-04-03 16:41:46 -0700410 self.change_number += 1
411 c = FakeChange(self, self.change_number, project, branch, subject,
412 upstream_root=self.upstream_root,
James E. Blair8b1dc3f2016-07-05 16:49:00 -0700413 status=status, files=files)
Clark Boylanb640e052014-04-03 16:41:46 -0700414 self.changes[self.change_number] = c
415 return c
416
Clark Boylanb640e052014-04-03 16:41:46 -0700417 def review(self, project, changeid, message, action):
418 number, ps = changeid.split(',')
419 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000420
421 # Add the approval back onto the change (ie simulate what gerrit would
422 # do).
423 # Usually when zuul leaves a review it'll create a feedback loop where
424 # zuul's review enters another gerrit event (which is then picked up by
425 # zuul). However, we can't mimic this behaviour (by adding this
426 # approval event into the queue) as it stops jobs from checking what
427 # happens before this event is triggered. If a job needs to see what
428 # happens they can add their own verified event into the queue.
429 # Nevertheless, we can update change with the new review in gerrit.
430
James E. Blair8b5408c2016-08-08 15:37:46 -0700431 for cat in action.keys():
432 if cat != 'submit':
Joshua Hesketh352264b2015-08-11 23:42:08 +1000433 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000434
James E. Blair8b5408c2016-08-08 15:37:46 -0700435 # TODOv3(jeblair): can this be removed?
Joshua Hesketh642824b2014-07-01 17:54:59 +1000436 if 'label' in action:
437 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000438 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000439
Clark Boylanb640e052014-04-03 16:41:46 -0700440 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000441
Clark Boylanb640e052014-04-03 16:41:46 -0700442 if 'submit' in action:
443 change.setMerged()
444 if message:
445 change.setReported()
446
447 def query(self, number):
448 change = self.changes.get(int(number))
449 if change:
450 return change.query()
451 return {}
452
James E. Blairc494d542014-08-06 09:23:52 -0700453 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700454 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700455 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800456 if query.startswith('change:'):
457 # Query a specific changeid
458 changeid = query[len('change:'):]
459 l = [change.query() for change in self.changes.values()
460 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700461 elif query.startswith('message:'):
462 # Query the content of a commit message
463 msg = query[len('message:'):].strip()
464 l = [change.query() for change in self.changes.values()
465 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800466 else:
467 # Query all open changes
468 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700469 return l
James E. Blairc494d542014-08-06 09:23:52 -0700470
Joshua Hesketh352264b2015-08-11 23:42:08 +1000471 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700472 pass
473
Joshua Hesketh352264b2015-08-11 23:42:08 +1000474 def getGitUrl(self, project):
475 return os.path.join(self.upstream_root, project.name)
476
Adam Gandelmanc5e4f1d2016-11-29 14:27:17 -0800477 def _getGitwebUrl(self, project, sha=None):
478 return self.getGitwebUrl(project, sha)
479
Clark Boylanb640e052014-04-03 16:41:46 -0700480
481class BuildHistory(object):
482 def __init__(self, **kw):
483 self.__dict__.update(kw)
484
485 def __repr__(self):
James E. Blair17302972016-08-10 16:11:42 -0700486 return ("<Completed build, result: %s name: %s uuid: %s changes: %s>" %
487 (self.result, self.name, self.uuid, self.changes))
Clark Boylanb640e052014-04-03 16:41:46 -0700488
489
490class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200491 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700492 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700493 self.url = url
494
495 def read(self):
Morgan Fainberg293f7f82016-05-30 14:01:22 -0700496 res = urllib.parse.urlparse(self.url)
Clark Boylanb640e052014-04-03 16:41:46 -0700497 path = res.path
498 project = '/'.join(path.split('/')[2:-2])
499 ret = '001e# service=git-upload-pack\n'
500 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
501 'multi_ack thin-pack side-band side-band-64k ofs-delta '
502 'shallow no-progress include-tag multi_ack_detailed no-done\n')
503 path = os.path.join(self.upstream_root, project)
504 repo = git.Repo(path)
505 for ref in repo.refs:
506 r = ref.object.hexsha + ' ' + ref.path + '\n'
507 ret += '%04x%s' % (len(r) + 4, r)
508 ret += '0000'
509 return ret
510
511
Clark Boylanb640e052014-04-03 16:41:46 -0700512class FakeStatsd(threading.Thread):
513 def __init__(self):
514 threading.Thread.__init__(self)
515 self.daemon = True
516 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
517 self.sock.bind(('', 0))
518 self.port = self.sock.getsockname()[1]
519 self.wake_read, self.wake_write = os.pipe()
520 self.stats = []
521
522 def run(self):
523 while True:
524 poll = select.poll()
525 poll.register(self.sock, select.POLLIN)
526 poll.register(self.wake_read, select.POLLIN)
527 ret = poll.poll()
528 for (fd, event) in ret:
529 if fd == self.sock.fileno():
530 data = self.sock.recvfrom(1024)
531 if not data:
532 return
533 self.stats.append(data[0])
534 if fd == self.wake_read:
535 return
536
537 def stop(self):
538 os.write(self.wake_write, '1\n')
539
540
James E. Blaire1767bc2016-08-02 10:00:27 -0700541class FakeBuild(object):
Clark Boylanb640e052014-04-03 16:41:46 -0700542 log = logging.getLogger("zuul.test")
543
James E. Blair34776ee2016-08-25 13:53:54 -0700544 def __init__(self, launch_server, job):
Clark Boylanb640e052014-04-03 16:41:46 -0700545 self.daemon = True
James E. Blaire1767bc2016-08-02 10:00:27 -0700546 self.launch_server = launch_server
Clark Boylanb640e052014-04-03 16:41:46 -0700547 self.job = job
James E. Blairab7132b2016-08-05 12:36:22 -0700548 self.jobdir = None
James E. Blair17302972016-08-10 16:11:42 -0700549 self.uuid = job.unique
Clark Boylanb640e052014-04-03 16:41:46 -0700550 self.parameters = json.loads(job.arguments)
James E. Blair34776ee2016-08-25 13:53:54 -0700551 # TODOv3(jeblair): self.node is really "the image of the node
552 # assigned". We should rename it (self.node_image?) if we
553 # keep using it like this, or we may end up exposing more of
554 # the complexity around multi-node jobs here
555 # (self.nodes[0].image?)
556 self.node = None
557 if len(self.parameters.get('nodes')) == 1:
558 self.node = self.parameters['nodes'][0]['image']
Clark Boylanb640e052014-04-03 16:41:46 -0700559 self.unique = self.parameters['ZUUL_UUID']
James E. Blair3f876d52016-07-22 13:07:14 -0700560 self.name = self.parameters['job']
Clark Boylanb640e052014-04-03 16:41:46 -0700561 self.wait_condition = threading.Condition()
562 self.waiting = False
563 self.aborted = False
Paul Belanger71d98172016-11-08 10:56:31 -0500564 self.requeue = False
Clark Boylanb640e052014-04-03 16:41:46 -0700565 self.created = time.time()
Clark Boylanb640e052014-04-03 16:41:46 -0700566 self.run_error = False
James E. Blaire1767bc2016-08-02 10:00:27 -0700567 self.changes = None
568 if 'ZUUL_CHANGE_IDS' in self.parameters:
569 self.changes = self.parameters['ZUUL_CHANGE_IDS']
Clark Boylanb640e052014-04-03 16:41:46 -0700570
James E. Blair3158e282016-08-19 09:34:11 -0700571 def __repr__(self):
572 waiting = ''
573 if self.waiting:
574 waiting = ' [waiting]'
575 return '<FakeBuild %s %s%s>' % (self.name, self.changes, waiting)
576
Clark Boylanb640e052014-04-03 16:41:46 -0700577 def release(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700578 """Release this build."""
Clark Boylanb640e052014-04-03 16:41:46 -0700579 self.wait_condition.acquire()
580 self.wait_condition.notify()
581 self.waiting = False
582 self.log.debug("Build %s released" % self.unique)
583 self.wait_condition.release()
584
585 def isWaiting(self):
James E. Blaire7b99a02016-08-05 14:27:34 -0700586 """Return whether this build is being held.
587
588 :returns: Whether the build is being held.
589 :rtype: bool
590 """
591
Clark Boylanb640e052014-04-03 16:41:46 -0700592 self.wait_condition.acquire()
593 if self.waiting:
594 ret = True
595 else:
596 ret = False
597 self.wait_condition.release()
598 return ret
599
600 def _wait(self):
601 self.wait_condition.acquire()
602 self.waiting = True
603 self.log.debug("Build %s waiting" % self.unique)
604 self.wait_condition.wait()
605 self.wait_condition.release()
606
607 def run(self):
Clark Boylanb640e052014-04-03 16:41:46 -0700608 self.log.debug('Running build %s' % self.unique)
609
James E. Blaire1767bc2016-08-02 10:00:27 -0700610 if self.launch_server.hold_jobs_in_build:
Clark Boylanb640e052014-04-03 16:41:46 -0700611 self.log.debug('Holding build %s' % self.unique)
612 self._wait()
613 self.log.debug("Build %s continuing" % self.unique)
614
Clark Boylanb640e052014-04-03 16:41:46 -0700615 result = 'SUCCESS'
James E. Blaira5dba232016-08-08 15:53:24 -0700616 if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
Clark Boylanb640e052014-04-03 16:41:46 -0700617 result = 'FAILURE'
618 if self.aborted:
619 result = 'ABORTED'
Paul Belanger71d98172016-11-08 10:56:31 -0500620 if self.requeue:
621 result = None
Clark Boylanb640e052014-04-03 16:41:46 -0700622
623 if self.run_error:
Clark Boylanb640e052014-04-03 16:41:46 -0700624 result = 'RUN_ERROR'
Clark Boylanb640e052014-04-03 16:41:46 -0700625
James E. Blaire1767bc2016-08-02 10:00:27 -0700626 return result
Clark Boylanb640e052014-04-03 16:41:46 -0700627
James E. Blaira5dba232016-08-08 15:53:24 -0700628 def shouldFail(self):
629 changes = self.launch_server.fail_tests.get(self.name, [])
630 for change in changes:
631 if self.hasChanges(change):
632 return True
633 return False
634
James E. Blaire7b99a02016-08-05 14:27:34 -0700635 def hasChanges(self, *changes):
636 """Return whether this build has certain changes in its git repos.
637
638 :arg FakeChange changes: One or more changes (varargs) that
639 are expected to be present (in order) in the git repository of
640 the active project.
641
642 :returns: Whether the build has the indicated changes.
643 :rtype: bool
644
645 """
Clint Byrum3343e3e2016-11-15 16:05:03 -0800646 for change in changes:
647 path = os.path.join(self.jobdir.git_root, change.project)
648 try:
649 repo = git.Repo(path)
650 except NoSuchPathError as e:
651 self.log.debug('%s' % e)
652 return False
653 ref = self.parameters['ZUUL_REF']
654 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
655 commit_message = '%s-1' % change.subject
656 self.log.debug("Checking if build %s has changes; commit_message "
657 "%s; repo_messages %s" % (self, commit_message,
658 repo_messages))
659 if commit_message not in repo_messages:
James E. Blair962220f2016-08-03 11:22:38 -0700660 self.log.debug(" messages do not match")
661 return False
662 self.log.debug(" OK")
663 return True
664
Clark Boylanb640e052014-04-03 16:41:46 -0700665
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +1000666class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
James E. Blaire7b99a02016-08-05 14:27:34 -0700667 """An Ansible launcher to be used in tests.
668
669 :ivar bool hold_jobs_in_build: If true, when jobs are launched
670 they will report that they have started but then pause until
671 released before reporting completion. This attribute may be
672 changed at any time and will take effect for subsequently
673 launched builds, but previously held builds will still need to
674 be explicitly released.
675
676 """
James E. Blairf5dbd002015-12-23 15:26:17 -0800677 def __init__(self, *args, **kw):
James E. Blaire1767bc2016-08-02 10:00:27 -0700678 self._run_ansible = kw.pop('_run_ansible', False)
James E. Blaira92cbc82017-01-23 14:56:49 -0800679 self._test_root = kw.pop('_test_root', False)
James E. Blairf5dbd002015-12-23 15:26:17 -0800680 super(RecordingLaunchServer, self).__init__(*args, **kw)
James E. Blaire1767bc2016-08-02 10:00:27 -0700681 self.hold_jobs_in_build = False
682 self.lock = threading.Lock()
683 self.running_builds = []
James E. Blair3f876d52016-07-22 13:07:14 -0700684 self.build_history = []
James E. Blaire1767bc2016-08-02 10:00:27 -0700685 self.fail_tests = {}
James E. Blairab7132b2016-08-05 12:36:22 -0700686 self.job_builds = {}
James E. Blairf5dbd002015-12-23 15:26:17 -0800687
James E. Blaira5dba232016-08-08 15:53:24 -0700688 def failJob(self, name, change):
James E. Blaire7b99a02016-08-05 14:27:34 -0700689 """Instruct the launcher to report matching builds as failures.
690
691 :arg str name: The name of the job to fail.
James E. Blaira5dba232016-08-08 15:53:24 -0700692 :arg Change change: The :py:class:`~tests.base.FakeChange`
693 instance which should cause the job to fail. This job
694 will also fail for changes depending on this change.
James E. Blaire7b99a02016-08-05 14:27:34 -0700695
696 """
James E. Blaire1767bc2016-08-02 10:00:27 -0700697 l = self.fail_tests.get(name, [])
698 l.append(change)
699 self.fail_tests[name] = l
James E. Blairf5dbd002015-12-23 15:26:17 -0800700
James E. Blair962220f2016-08-03 11:22:38 -0700701 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700702 """Release a held build.
703
704 :arg str regex: A regular expression which, if supplied, will
705 cause only builds with matching names to be released. If
706 not supplied, all builds will be released.
707
708 """
James E. Blair962220f2016-08-03 11:22:38 -0700709 builds = self.running_builds[:]
710 self.log.debug("Releasing build %s (%s)" % (regex,
711 len(self.running_builds)))
712 for build in builds:
713 if not regex or re.match(regex, build.name):
714 self.log.debug("Releasing build %s" %
715 (build.parameters['ZUUL_UUID']))
716 build.release()
717 else:
718 self.log.debug("Not releasing build %s" %
719 (build.parameters['ZUUL_UUID']))
720 self.log.debug("Done releasing builds %s (%s)" %
721 (regex, len(self.running_builds)))
722
James E. Blair17302972016-08-10 16:11:42 -0700723 def launchJob(self, job):
James E. Blair34776ee2016-08-25 13:53:54 -0700724 build = FakeBuild(self, job)
James E. Blaire1767bc2016-08-02 10:00:27 -0700725 job.build = build
James E. Blaire1767bc2016-08-02 10:00:27 -0700726 self.running_builds.append(build)
James E. Blairab7132b2016-08-05 12:36:22 -0700727 self.job_builds[job.unique] = build
James E. Blaira92cbc82017-01-23 14:56:49 -0800728 args = json.loads(job.arguments)
729 args['zuul']['_test'] = dict(test_root=self._test_root)
730 job.arguments = json.dumps(args)
James E. Blair17302972016-08-10 16:11:42 -0700731 super(RecordingLaunchServer, self).launchJob(job)
732
733 def stopJob(self, job):
734 self.log.debug("handle stop")
735 parameters = json.loads(job.arguments)
736 uuid = parameters['uuid']
737 for build in self.running_builds:
738 if build.unique == uuid:
739 build.aborted = True
740 build.release()
741 super(RecordingLaunchServer, self).stopJob(job)
James E. Blairab7132b2016-08-05 12:36:22 -0700742
743 def runAnsible(self, jobdir, job):
744 build = self.job_builds[job.unique]
745 build.jobdir = jobdir
James E. Blaire1767bc2016-08-02 10:00:27 -0700746
747 if self._run_ansible:
748 result = super(RecordingLaunchServer, self).runAnsible(jobdir, job)
749 else:
750 result = build.run()
751
752 self.lock.acquire()
753 self.build_history.append(
James E. Blair17302972016-08-10 16:11:42 -0700754 BuildHistory(name=build.name, result=result, changes=build.changes,
755 node=build.node, uuid=build.unique,
756 parameters=build.parameters,
James E. Blaire1767bc2016-08-02 10:00:27 -0700757 pipeline=build.parameters['ZUUL_PIPELINE'])
758 )
James E. Blairab7132b2016-08-05 12:36:22 -0700759 self.running_builds.remove(build)
760 del self.job_builds[job.unique]
James E. Blaire1767bc2016-08-02 10:00:27 -0700761 self.lock.release()
Clint Byrum69e47122016-12-02 16:40:35 -0800762 if build.run_error:
763 result = None
James E. Blaire1767bc2016-08-02 10:00:27 -0700764 return result
James E. Blairf5dbd002015-12-23 15:26:17 -0800765
766
Clark Boylanb640e052014-04-03 16:41:46 -0700767class FakeGearmanServer(gear.Server):
James E. Blaire7b99a02016-08-05 14:27:34 -0700768 """A Gearman server for use in tests.
769
770 :ivar bool hold_jobs_in_queue: If true, submitted jobs will be
771 added to the queue but will not be distributed to workers
772 until released. This attribute may be changed at any time and
773 will take effect for subsequently enqueued jobs, but
774 previously held jobs will still need to be explicitly
775 released.
776
777 """
778
Clark Boylanb640e052014-04-03 16:41:46 -0700779 def __init__(self):
780 self.hold_jobs_in_queue = False
781 super(FakeGearmanServer, self).__init__(0)
782
783 def getJobForConnection(self, connection, peek=False):
784 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
785 for job in queue:
786 if not hasattr(job, 'waiting'):
Paul Belanger6ab6af72016-11-06 11:32:59 -0500787 if job.name.startswith('launcher:launch'):
Clark Boylanb640e052014-04-03 16:41:46 -0700788 job.waiting = self.hold_jobs_in_queue
789 else:
790 job.waiting = False
791 if job.waiting:
792 continue
793 if job.name in connection.functions:
794 if not peek:
795 queue.remove(job)
796 connection.related_jobs[job.handle] = job
797 job.worker_connection = connection
798 job.running = True
799 return job
800 return None
801
802 def release(self, regex=None):
James E. Blaire7b99a02016-08-05 14:27:34 -0700803 """Release a held job.
804
805 :arg str regex: A regular expression which, if supplied, will
806 cause only jobs with matching names to be released. If
807 not supplied, all jobs will be released.
808 """
Clark Boylanb640e052014-04-03 16:41:46 -0700809 released = False
810 qlen = (len(self.high_queue) + len(self.normal_queue) +
811 len(self.low_queue))
812 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
813 for job in self.getQueue():
Paul Belanger6ab6af72016-11-06 11:32:59 -0500814 if job.name != 'launcher:launch':
Clark Boylanb640e052014-04-03 16:41:46 -0700815 continue
Paul Belanger6ab6af72016-11-06 11:32:59 -0500816 parameters = json.loads(job.arguments)
817 if not regex or re.match(regex, parameters.get('job')):
Clark Boylanb640e052014-04-03 16:41:46 -0700818 self.log.debug("releasing queued job %s" %
819 job.unique)
820 job.waiting = False
821 released = True
822 else:
823 self.log.debug("not releasing queued job %s" %
824 job.unique)
825 if released:
826 self.wakeConnections()
827 qlen = (len(self.high_queue) + len(self.normal_queue) +
828 len(self.low_queue))
829 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
830
831
832class FakeSMTP(object):
833 log = logging.getLogger('zuul.FakeSMTP')
834
835 def __init__(self, messages, server, port):
836 self.server = server
837 self.port = port
838 self.messages = messages
839
840 def sendmail(self, from_email, to_email, msg):
841 self.log.info("Sending email from %s, to %s, with msg %s" % (
842 from_email, to_email, msg))
843
844 headers = msg.split('\n\n', 1)[0]
845 body = msg.split('\n\n', 1)[1]
846
847 self.messages.append(dict(
848 from_email=from_email,
849 to_email=to_email,
850 msg=msg,
851 headers=headers,
852 body=body,
853 ))
854
855 return True
856
857 def quit(self):
858 return True
859
860
861class FakeSwiftClientConnection(swiftclient.client.Connection):
862 def post_account(self, headers):
863 # Do nothing
864 pass
865
866 def get_auth(self):
867 # Returns endpoint and (unused) auth token
868 endpoint = os.path.join('https://storage.example.org', 'V1',
869 'AUTH_account')
870 return endpoint, ''
871
872
James E. Blairdce6cea2016-12-20 16:45:32 -0800873class FakeNodepool(object):
874 REQUEST_ROOT = '/nodepool/requests'
James E. Blaire18d4602017-01-05 11:17:28 -0800875 NODE_ROOT = '/nodepool/nodes'
James E. Blairdce6cea2016-12-20 16:45:32 -0800876
877 log = logging.getLogger("zuul.test.FakeNodepool")
878
879 def __init__(self, host, port, chroot):
880 self.client = kazoo.client.KazooClient(
881 hosts='%s:%s%s' % (host, port, chroot))
882 self.client.start()
883 self._running = True
James E. Blair15be0e12017-01-03 13:45:20 -0800884 self.paused = False
James E. Blairdce6cea2016-12-20 16:45:32 -0800885 self.thread = threading.Thread(target=self.run)
886 self.thread.daemon = True
887 self.thread.start()
James E. Blair6ab79e02017-01-06 10:10:17 -0800888 self.fail_requests = set()
James E. Blairdce6cea2016-12-20 16:45:32 -0800889
890 def stop(self):
891 self._running = False
892 self.thread.join()
893 self.client.stop()
894 self.client.close()
895
896 def run(self):
897 while self._running:
898 self._run()
899 time.sleep(0.1)
900
901 def _run(self):
James E. Blair15be0e12017-01-03 13:45:20 -0800902 if self.paused:
903 return
James E. Blairdce6cea2016-12-20 16:45:32 -0800904 for req in self.getNodeRequests():
905 self.fulfillRequest(req)
906
907 def getNodeRequests(self):
908 try:
909 reqids = self.client.get_children(self.REQUEST_ROOT)
910 except kazoo.exceptions.NoNodeError:
911 return []
912 reqs = []
913 for oid in sorted(reqids):
914 path = self.REQUEST_ROOT + '/' + oid
James E. Blair0ef64f82017-02-02 11:25:16 -0800915 try:
916 data, stat = self.client.get(path)
917 data = json.loads(data)
918 data['_oid'] = oid
919 reqs.append(data)
920 except kazoo.exceptions.NoNodeError:
921 pass
James E. Blairdce6cea2016-12-20 16:45:32 -0800922 return reqs
923
James E. Blaire18d4602017-01-05 11:17:28 -0800924 def getNodes(self):
925 try:
926 nodeids = self.client.get_children(self.NODE_ROOT)
927 except kazoo.exceptions.NoNodeError:
928 return []
929 nodes = []
930 for oid in sorted(nodeids):
931 path = self.NODE_ROOT + '/' + oid
932 data, stat = self.client.get(path)
933 data = json.loads(data)
934 data['_oid'] = oid
935 try:
936 lockfiles = self.client.get_children(path + '/lock')
937 except kazoo.exceptions.NoNodeError:
938 lockfiles = []
939 if lockfiles:
940 data['_lock'] = True
941 else:
942 data['_lock'] = False
943 nodes.append(data)
944 return nodes
945
James E. Blaira38c28e2017-01-04 10:33:20 -0800946 def makeNode(self, request_id, node_type):
947 now = time.time()
948 path = '/nodepool/nodes/'
949 data = dict(type=node_type,
950 provider='test-provider',
951 region='test-region',
952 az=None,
953 public_ipv4='127.0.0.1',
954 private_ipv4=None,
955 public_ipv6=None,
956 allocated_to=request_id,
957 state='ready',
958 state_time=now,
959 created_time=now,
960 updated_time=now,
961 image_id=None,
962 launcher='fake-nodepool')
963 data = json.dumps(data)
964 path = self.client.create(path, data,
965 makepath=True,
966 sequence=True)
967 nodeid = path.split("/")[-1]
968 return nodeid
969
James E. Blair6ab79e02017-01-06 10:10:17 -0800970 def addFailRequest(self, request):
971 self.fail_requests.add(request['_oid'])
972
James E. Blairdce6cea2016-12-20 16:45:32 -0800973 def fulfillRequest(self, request):
James E. Blair6ab79e02017-01-06 10:10:17 -0800974 if request['state'] != 'requested':
James E. Blairdce6cea2016-12-20 16:45:32 -0800975 return
976 request = request.copy()
James E. Blairdce6cea2016-12-20 16:45:32 -0800977 oid = request['_oid']
978 del request['_oid']
James E. Blaira38c28e2017-01-04 10:33:20 -0800979
James E. Blair6ab79e02017-01-06 10:10:17 -0800980 if oid in self.fail_requests:
981 request['state'] = 'failed'
982 else:
983 request['state'] = 'fulfilled'
984 nodes = []
985 for node in request['node_types']:
986 nodeid = self.makeNode(oid, node)
987 nodes.append(nodeid)
988 request['nodes'] = nodes
James E. Blaira38c28e2017-01-04 10:33:20 -0800989
James E. Blaira38c28e2017-01-04 10:33:20 -0800990 request['state_time'] = time.time()
James E. Blairdce6cea2016-12-20 16:45:32 -0800991 path = self.REQUEST_ROOT + '/' + oid
992 data = json.dumps(request)
993 self.log.debug("Fulfilling node request: %s %s" % (oid, data))
994 self.client.set(path, data)
995
996
James E. Blair498059b2016-12-20 13:50:13 -0800997class ChrootedKazooFixture(fixtures.Fixture):
998 def __init__(self):
999 super(ChrootedKazooFixture, self).__init__()
1000
1001 zk_host = os.environ.get('NODEPOOL_ZK_HOST', 'localhost')
1002 if ':' in zk_host:
1003 host, port = zk_host.split(':')
1004 else:
1005 host = zk_host
1006 port = None
1007
1008 self.zookeeper_host = host
1009
1010 if not port:
1011 self.zookeeper_port = 2181
1012 else:
1013 self.zookeeper_port = int(port)
1014
1015 def _setUp(self):
1016 # Make sure the test chroot paths do not conflict
1017 random_bits = ''.join(random.choice(string.ascii_lowercase +
1018 string.ascii_uppercase)
1019 for x in range(8))
1020
1021 rand_test_path = '%s_%s' % (random_bits, os.getpid())
1022 self.zookeeper_chroot = "/nodepool_test/%s" % rand_test_path
1023
1024 # Ensure the chroot path exists and clean up any pre-existing znodes.
1025 _tmp_client = kazoo.client.KazooClient(
1026 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1027 _tmp_client.start()
1028
1029 if _tmp_client.exists(self.zookeeper_chroot):
1030 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1031
1032 _tmp_client.ensure_path(self.zookeeper_chroot)
1033 _tmp_client.stop()
1034 _tmp_client.close()
1035
1036 self.addCleanup(self._cleanup)
1037
1038 def _cleanup(self):
1039 '''Remove the chroot path.'''
1040 # Need a non-chroot'ed client to remove the chroot path
1041 _tmp_client = kazoo.client.KazooClient(
1042 hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
1043 _tmp_client.start()
1044 _tmp_client.delete(self.zookeeper_chroot, recursive=True)
1045 _tmp_client.stop()
1046
1047
Maru Newby3fe5f852015-01-13 04:22:14 +00001048class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -07001049 log = logging.getLogger("zuul.test")
1050
James E. Blair1c236df2017-02-01 14:07:24 -08001051 def attachLogs(self, *args):
1052 def reader():
1053 self._log_stream.seek(0)
1054 while True:
1055 x = self._log_stream.read(4096)
1056 if not x:
1057 break
1058 yield x.encode('utf8')
1059 content = testtools.content.content_from_reader(
1060 reader,
1061 testtools.content_type.UTF8_TEXT,
1062 False)
1063 self.addDetail('logging', content)
1064
Clark Boylanb640e052014-04-03 16:41:46 -07001065 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +00001066 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -07001067 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
1068 try:
1069 test_timeout = int(test_timeout)
1070 except ValueError:
1071 # If timeout value is invalid do not set a timeout.
1072 test_timeout = 0
1073 if test_timeout > 0:
1074 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
1075
1076 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
1077 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
1078 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
1079 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
1080 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
1081 os.environ.get('OS_STDERR_CAPTURE') == '1'):
1082 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
1083 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
1084 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
1085 os.environ.get('OS_LOG_CAPTURE') == '1'):
James E. Blair1c236df2017-02-01 14:07:24 -08001086 self._log_stream = StringIO()
1087 self.addOnException(self.attachLogs)
1088 else:
1089 self._log_stream = sys.stdout
Maru Newby3fe5f852015-01-13 04:22:14 +00001090
James E. Blair1c236df2017-02-01 14:07:24 -08001091 handler = logging.StreamHandler(self._log_stream)
1092 formatter = logging.Formatter('%(asctime)s %(name)-32s '
1093 '%(levelname)-8s %(message)s')
1094 handler.setFormatter(formatter)
1095
1096 logger = logging.getLogger()
1097 logger.setLevel(logging.DEBUG)
1098 logger.addHandler(handler)
1099
1100 # NOTE(notmorgan): Extract logging overrides for specific
1101 # libraries from the OS_LOG_DEFAULTS env and create loggers
1102 # for each. This is used to limit the output during test runs
1103 # from libraries that zuul depends on such as gear.
James E. Blairdce6cea2016-12-20 16:45:32 -08001104 log_defaults_from_env = os.environ.get(
1105 'OS_LOG_DEFAULTS',
James E. Blair1c236df2017-02-01 14:07:24 -08001106 'git.cmd=INFO,kazoo.client=WARNING,gear=INFO')
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001107
James E. Blairdce6cea2016-12-20 16:45:32 -08001108 if log_defaults_from_env:
1109 for default in log_defaults_from_env.split(','):
1110 try:
1111 name, level_str = default.split('=', 1)
1112 level = getattr(logging, level_str, logging.DEBUG)
James E. Blair1c236df2017-02-01 14:07:24 -08001113 logger = logging.getLogger(name)
1114 logger.setLevel(level)
1115 logger.addHandler(handler)
1116 logger.propagate = False
James E. Blairdce6cea2016-12-20 16:45:32 -08001117 except ValueError:
1118 # NOTE(notmorgan): Invalid format of the log default,
1119 # skip and don't try and apply a logger for the
1120 # specified module
1121 pass
Morgan Fainbergd34e0b42016-06-09 19:10:38 -07001122
Maru Newby3fe5f852015-01-13 04:22:14 +00001123
1124class ZuulTestCase(BaseTestCase):
James E. Blaire7b99a02016-08-05 14:27:34 -07001125 """A test case with a functioning Zuul.
1126
1127 The following class variables are used during test setup and can
1128 be overidden by subclasses but are effectively read-only once a
1129 test method starts running:
1130
1131 :cvar str config_file: This points to the main zuul config file
1132 within the fixtures directory. Subclasses may override this
1133 to obtain a different behavior.
1134
1135 :cvar str tenant_config_file: This is the tenant config file
1136 (which specifies from what git repos the configuration should
1137 be loaded). It defaults to the value specified in
1138 `config_file` but can be overidden by subclasses to obtain a
1139 different tenant/project layout while using the standard main
1140 configuration.
1141
1142 The following are instance variables that are useful within test
1143 methods:
1144
1145 :ivar FakeGerritConnection fake_<connection>:
1146 A :py:class:`~tests.base.FakeGerritConnection` will be
1147 instantiated for each connection present in the config file
1148 and stored here. For instance, `fake_gerrit` will hold the
1149 FakeGerritConnection object for a connection named `gerrit`.
1150
1151 :ivar FakeGearmanServer gearman_server: An instance of
1152 :py:class:`~tests.base.FakeGearmanServer` which is the Gearman
1153 server that all of the Zuul components in this test use to
1154 communicate with each other.
1155
1156 :ivar RecordingLaunchServer launch_server: An instance of
1157 :py:class:`~tests.base.RecordingLaunchServer` which is the
1158 Ansible launch server used to run jobs for this test.
1159
1160 :ivar list builds: A list of :py:class:`~tests.base.FakeBuild` objects
1161 representing currently running builds. They are appended to
1162 the list in the order they are launched, and removed from this
1163 list upon completion.
1164
1165 :ivar list history: A list of :py:class:`~tests.base.BuildHistory`
1166 objects representing completed builds. They are appended to
1167 the list in the order they complete.
1168
1169 """
1170
James E. Blair83005782015-12-11 14:46:03 -08001171 config_file = 'zuul.conf'
James E. Blaire1767bc2016-08-02 10:00:27 -07001172 run_ansible = False
James E. Blair3f876d52016-07-22 13:07:14 -07001173
1174 def _startMerger(self):
1175 self.merge_server = zuul.merger.server.MergeServer(self.config,
1176 self.connections)
1177 self.merge_server.start()
1178
Maru Newby3fe5f852015-01-13 04:22:14 +00001179 def setUp(self):
1180 super(ZuulTestCase, self).setUp()
James E. Blair498059b2016-12-20 13:50:13 -08001181
1182 self.setupZK()
1183
James E. Blair97d902e2014-08-21 13:25:56 -07001184 if USE_TEMPDIR:
1185 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +10001186 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
1187 ).path
James E. Blair97d902e2014-08-21 13:25:56 -07001188 else:
1189 tmp_root = os.environ.get("ZUUL_TEST_ROOT")
Clark Boylanb640e052014-04-03 16:41:46 -07001190 self.test_root = os.path.join(tmp_root, "zuul-test")
1191 self.upstream_root = os.path.join(self.test_root, "upstream")
1192 self.git_root = os.path.join(self.test_root, "git")
James E. Blairce8a2132016-05-19 15:21:52 -07001193 self.state_root = os.path.join(self.test_root, "lib")
Clark Boylanb640e052014-04-03 16:41:46 -07001194
1195 if os.path.exists(self.test_root):
1196 shutil.rmtree(self.test_root)
1197 os.makedirs(self.test_root)
1198 os.makedirs(self.upstream_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001199 os.makedirs(self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001200
1201 # Make per test copy of Configuration.
1202 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -08001203 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +11001204 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -08001205 self.config.get('zuul', 'tenant_config')))
Clark Boylanb640e052014-04-03 16:41:46 -07001206 self.config.set('merger', 'git_dir', self.git_root)
James E. Blairce8a2132016-05-19 15:21:52 -07001207 self.config.set('zuul', 'state_dir', self.state_root)
Clark Boylanb640e052014-04-03 16:41:46 -07001208
1209 # For each project in config:
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001210 # TODOv3(jeblair): remove these and replace with new git
1211 # filesystem fixtures
Clark Boylanb640e052014-04-03 16:41:46 -07001212 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -07001213 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -07001214 self.init_repo("org/project5")
1215 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -07001216 self.init_repo("org/one-job-project")
1217 self.init_repo("org/nonvoting-project")
1218 self.init_repo("org/templated-project")
1219 self.init_repo("org/layered-project")
1220 self.init_repo("org/node-project")
1221 self.init_repo("org/conflict-project")
1222 self.init_repo("org/noop-project")
1223 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +00001224 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -07001225
1226 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +10001227 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
1228 # see: https://github.com/jsocol/pystatsd/issues/61
1229 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -07001230 os.environ['STATSD_PORT'] = str(self.statsd.port)
1231 self.statsd.start()
1232 # the statsd client object is configured in the statsd module import
Monty Taylor74fa3862016-06-02 07:39:49 +03001233 reload_module(statsd)
1234 reload_module(zuul.scheduler)
Clark Boylanb640e052014-04-03 16:41:46 -07001235
1236 self.gearman_server = FakeGearmanServer()
1237
1238 self.config.set('gearman', 'port', str(self.gearman_server.port))
1239
James E. Blaire511d2f2016-12-08 15:22:26 -08001240 gerritsource.GerritSource.replication_timeout = 1.5
1241 gerritsource.GerritSource.replication_retry_interval = 0.5
1242 gerritconnection.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -07001243
Joshua Hesketh352264b2015-08-11 23:42:08 +10001244 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -07001245
1246 self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
1247 FakeSwiftClientConnection))
James E. Blaire511d2f2016-12-08 15:22:26 -08001248
Clark Boylanb640e052014-04-03 16:41:46 -07001249 self.swift = zuul.lib.swift.Swift(self.config)
1250
Jan Hruban6b71aff2015-10-22 16:58:08 +02001251 self.event_queues = [
1252 self.sched.result_event_queue,
1253 self.sched.trigger_event_queue
1254 ]
1255
James E. Blairfef78942016-03-11 16:28:56 -08001256 self.configure_connections()
Joshua Hesketh352264b2015-08-11 23:42:08 +10001257 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +10001258
Clark Boylanb640e052014-04-03 16:41:46 -07001259 def URLOpenerFactory(*args, **kw):
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001260 if isinstance(args[0], urllib.request.Request):
Clark Boylanb640e052014-04-03 16:41:46 -07001261 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -07001262 return FakeURLOpener(self.upstream_root, *args, **kw)
1263
Morgan Fainberg293f7f82016-05-30 14:01:22 -07001264 old_urlopen = urllib.request.urlopen
1265 urllib.request.urlopen = URLOpenerFactory
Clark Boylanb640e052014-04-03 16:41:46 -07001266
James E. Blair3f876d52016-07-22 13:07:14 -07001267 self._startMerger()
James E. Blair3f876d52016-07-22 13:07:14 -07001268
James E. Blaire1767bc2016-08-02 10:00:27 -07001269 self.launch_server = RecordingLaunchServer(
James E. Blaira92cbc82017-01-23 14:56:49 -08001270 self.config, self.connections,
1271 _run_ansible=self.run_ansible,
1272 _test_root=self.test_root)
James E. Blaire1767bc2016-08-02 10:00:27 -07001273 self.launch_server.start()
1274 self.history = self.launch_server.build_history
1275 self.builds = self.launch_server.running_builds
1276
1277 self.launch_client = zuul.launcher.client.LaunchClient(
James E. Blair82938472016-01-11 14:38:13 -08001278 self.config, self.sched, self.swift)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001279 self.merge_client = zuul.merger.client.MergeClient(
1280 self.config, self.sched)
James E. Blair8d692392016-04-08 17:47:58 -07001281 self.nodepool = zuul.nodepool.Nodepool(self.sched)
James E. Blairdce6cea2016-12-20 16:45:32 -08001282 self.zk = zuul.zk.ZooKeeper()
1283 self.zk.connect([self.zk_config])
1284
1285 self.fake_nodepool = FakeNodepool(self.zk_config.host,
1286 self.zk_config.port,
1287 self.zk_config.chroot)
Clark Boylanb640e052014-04-03 16:41:46 -07001288
James E. Blaire1767bc2016-08-02 10:00:27 -07001289 self.sched.setLauncher(self.launch_client)
Clark Boylanb640e052014-04-03 16:41:46 -07001290 self.sched.setMerger(self.merge_client)
James E. Blair8d692392016-04-08 17:47:58 -07001291 self.sched.setNodepool(self.nodepool)
James E. Blairdce6cea2016-12-20 16:45:32 -08001292 self.sched.setZooKeeper(self.zk)
Clark Boylanb640e052014-04-03 16:41:46 -07001293
Paul Belanger88ef0ea2015-12-23 11:57:02 -05001294 self.webapp = zuul.webapp.WebApp(
1295 self.sched, port=0, listen_address='127.0.0.1')
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001296 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -07001297
1298 self.sched.start()
1299 self.sched.reconfigure(self.config)
1300 self.sched.resume()
1301 self.webapp.start()
1302 self.rpc.start()
James E. Blaire1767bc2016-08-02 10:00:27 -07001303 self.launch_client.gearman.waitForServer()
Clark Boylanb640e052014-04-03 16:41:46 -07001304
Clark Boylanb640e052014-04-03 16:41:46 -07001305 self.addCleanup(self.shutdown)
1306
James E. Blaire18d4602017-01-05 11:17:28 -08001307 def tearDown(self):
1308 super(ZuulTestCase, self).tearDown()
1309 self.assertFinalState()
1310
James E. Blairfef78942016-03-11 16:28:56 -08001311 def configure_connections(self):
James E. Blaire511d2f2016-12-08 15:22:26 -08001312 # Set up gerrit related fakes
1313 # Set a changes database so multiple FakeGerrit's can report back to
1314 # a virtual canonical database given by the configured hostname
1315 self.gerrit_changes_dbs = {}
1316
1317 def getGerritConnection(driver, name, config):
1318 db = self.gerrit_changes_dbs.setdefault(config['server'], {})
1319 con = FakeGerritConnection(driver, name, config,
1320 changes_db=db,
1321 upstream_root=self.upstream_root)
1322 self.event_queues.append(con.event_queue)
1323 setattr(self, 'fake_' + name, con)
1324 return con
1325
1326 self.useFixture(fixtures.MonkeyPatch(
1327 'zuul.driver.gerrit.GerritDriver.getConnection',
1328 getGerritConnection))
1329
1330 # Set up smtp related fakes
Joshua Hesketh352264b2015-08-11 23:42:08 +10001331 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001332
Joshua Hesketh352264b2015-08-11 23:42:08 +10001333 def FakeSMTPFactory(*args, **kw):
1334 args = [self.smtp_messages] + list(args)
1335 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001336
Joshua Hesketh352264b2015-08-11 23:42:08 +10001337 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001338
James E. Blaire511d2f2016-12-08 15:22:26 -08001339 # Register connections from the config using fakes
James E. Blairfef78942016-03-11 16:28:56 -08001340 self.connections = zuul.lib.connections.ConnectionRegistry()
James E. Blaire511d2f2016-12-08 15:22:26 -08001341 self.connections.configure(self.config)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001342
James E. Blair83005782015-12-11 14:46:03 -08001343 def setup_config(self):
James E. Blaire7b99a02016-08-05 14:27:34 -07001344 # This creates the per-test configuration object. It can be
1345 # overriden by subclasses, but should not need to be since it
1346 # obeys the config_file and tenant_config_file attributes.
Clark Boylanb640e052014-04-03 16:41:46 -07001347 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001348 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair2a629ec2015-12-22 15:32:02 -08001349 if hasattr(self, 'tenant_config_file'):
1350 self.config.set('zuul', 'tenant_config', self.tenant_config_file)
James E. Blair96c6bf82016-01-15 16:20:40 -08001351 git_path = os.path.join(
1352 os.path.dirname(
1353 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1354 'git')
1355 if os.path.exists(git_path):
1356 for reponame in os.listdir(git_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001357 project = reponame.replace('_', '/')
1358 self.copyDirToRepo(project,
James E. Blair96c6bf82016-01-15 16:20:40 -08001359 os.path.join(git_path, reponame))
1360
James E. Blair498059b2016-12-20 13:50:13 -08001361 def setupZK(self):
1362 self.zk_chroot_fixture = self.useFixture(ChrootedKazooFixture())
James E. Blairdce6cea2016-12-20 16:45:32 -08001363 self.zk_config = zuul.zk.ZooKeeperConnectionConfig(
1364 self.zk_chroot_fixture.zookeeper_host,
1365 self.zk_chroot_fixture.zookeeper_port,
1366 self.zk_chroot_fixture.zookeeper_chroot)
James E. Blair498059b2016-12-20 13:50:13 -08001367
James E. Blair96c6bf82016-01-15 16:20:40 -08001368 def copyDirToRepo(self, project, source_path):
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001369 self.init_repo(project)
James E. Blair96c6bf82016-01-15 16:20:40 -08001370
1371 files = {}
1372 for (dirpath, dirnames, filenames) in os.walk(source_path):
1373 for filename in filenames:
1374 test_tree_filepath = os.path.join(dirpath, filename)
1375 common_path = os.path.commonprefix([test_tree_filepath,
1376 source_path])
1377 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1378 with open(test_tree_filepath, 'r') as f:
1379 content = f.read()
1380 files[relative_filepath] = content
1381 self.addCommitToRepo(project, 'add content from fixture',
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001382 files, branch='master', tag='init')
James E. Blair83005782015-12-11 14:46:03 -08001383
James E. Blaire18d4602017-01-05 11:17:28 -08001384 def assertNodepoolState(self):
1385 # Make sure that there are no pending requests
1386
1387 requests = self.fake_nodepool.getNodeRequests()
1388 self.assertEqual(len(requests), 0)
1389
1390 nodes = self.fake_nodepool.getNodes()
1391 for node in nodes:
1392 self.assertFalse(node['_lock'], "Node %s is locked" %
1393 (node['_oid'],))
1394
Clark Boylanb640e052014-04-03 16:41:46 -07001395 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001396 # Make sure that git.Repo objects have been garbage collected.
1397 repos = []
1398 gc.collect()
1399 for obj in gc.get_objects():
1400 if isinstance(obj, git.Repo):
1401 repos.append(obj)
1402 self.assertEqual(len(repos), 0)
1403 self.assertEmptyQueues()
James E. Blaire18d4602017-01-05 11:17:28 -08001404 self.assertNodepoolState()
James E. Blair83005782015-12-11 14:46:03 -08001405 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001406 for tenant in self.sched.abide.tenants.values():
1407 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001408 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001409 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001410
1411 def shutdown(self):
1412 self.log.debug("Shutting down after tests")
James E. Blaire1767bc2016-08-02 10:00:27 -07001413 self.launch_client.stop()
James E. Blair3f876d52016-07-22 13:07:14 -07001414 self.merge_server.stop()
1415 self.merge_server.join()
Clark Boylanb640e052014-04-03 16:41:46 -07001416 self.merge_client.stop()
James E. Blaire1767bc2016-08-02 10:00:27 -07001417 self.launch_server.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001418 self.sched.stop()
1419 self.sched.join()
1420 self.statsd.stop()
1421 self.statsd.join()
1422 self.webapp.stop()
1423 self.webapp.join()
1424 self.rpc.stop()
1425 self.rpc.join()
1426 self.gearman_server.shutdown()
James E. Blairdce6cea2016-12-20 16:45:32 -08001427 self.fake_nodepool.stop()
1428 self.zk.disconnect()
Clark Boylanb640e052014-04-03 16:41:46 -07001429 threads = threading.enumerate()
1430 if len(threads) > 1:
1431 self.log.error("More than one thread is running: %s" % threads)
James E. Blair6ac368c2016-12-22 18:07:20 -08001432 self.printHistory()
Clark Boylanb640e052014-04-03 16:41:46 -07001433
1434 def init_repo(self, project):
1435 parts = project.split('/')
1436 path = os.path.join(self.upstream_root, *parts[:-1])
1437 if not os.path.exists(path):
1438 os.makedirs(path)
1439 path = os.path.join(self.upstream_root, project)
1440 repo = git.Repo.init(path)
1441
Morgan Fainberg78c301a2016-07-14 13:47:01 -07001442 with repo.config_writer() as config_writer:
1443 config_writer.set_value('user', 'email', 'user@example.com')
1444 config_writer.set_value('user', 'name', 'User Name')
Clark Boylanb640e052014-04-03 16:41:46 -07001445
Clark Boylanb640e052014-04-03 16:41:46 -07001446 repo.index.commit('initial commit')
1447 master = repo.create_head('master')
Clark Boylanb640e052014-04-03 16:41:46 -07001448
James E. Blair97d902e2014-08-21 13:25:56 -07001449 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001450 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001451 repo.git.clean('-x', '-f', '-d')
1452
James E. Blair97d902e2014-08-21 13:25:56 -07001453 def create_branch(self, project, branch):
1454 path = os.path.join(self.upstream_root, project)
1455 repo = git.Repo.init(path)
1456 fn = os.path.join(path, 'README')
1457
1458 branch_head = repo.create_head(branch)
1459 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001460 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001461 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001462 f.close()
1463 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001464 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001465
James E. Blair97d902e2014-08-21 13:25:56 -07001466 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001467 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001468 repo.git.clean('-x', '-f', '-d')
1469
Sachi King9f16d522016-03-16 12:20:45 +11001470 def create_commit(self, project):
1471 path = os.path.join(self.upstream_root, project)
1472 repo = git.Repo(path)
1473 repo.head.reference = repo.heads['master']
1474 file_name = os.path.join(path, 'README')
1475 with open(file_name, 'a') as f:
1476 f.write('creating fake commit\n')
1477 repo.index.add([file_name])
1478 commit = repo.index.commit('Creating a fake commit')
1479 return commit.hexsha
1480
James E. Blairb8c16472015-05-05 14:55:26 -07001481 def orderedRelease(self):
1482 # Run one build at a time to ensure non-race order:
1483 while len(self.builds):
1484 self.release(self.builds[0])
1485 self.waitUntilSettled()
1486
Clark Boylanb640e052014-04-03 16:41:46 -07001487 def release(self, job):
1488 if isinstance(job, FakeBuild):
1489 job.release()
1490 else:
1491 job.waiting = False
1492 self.log.debug("Queued job %s released" % job.unique)
1493 self.gearman_server.wakeConnections()
1494
1495 def getParameter(self, job, name):
1496 if isinstance(job, FakeBuild):
1497 return job.parameters[name]
1498 else:
1499 parameters = json.loads(job.arguments)
1500 return parameters[name]
1501
Clark Boylanb640e052014-04-03 16:41:46 -07001502 def haveAllBuildsReported(self):
1503 # See if Zuul is waiting on a meta job to complete
James E. Blaire1767bc2016-08-02 10:00:27 -07001504 if self.launch_client.meta_jobs:
Clark Boylanb640e052014-04-03 16:41:46 -07001505 return False
1506 # Find out if every build that the worker has completed has been
1507 # reported back to Zuul. If it hasn't then that means a Gearman
1508 # event is still in transit and the system is not stable.
James E. Blair3f876d52016-07-22 13:07:14 -07001509 for build in self.history:
James E. Blaire1767bc2016-08-02 10:00:27 -07001510 zbuild = self.launch_client.builds.get(build.uuid)
Clark Boylanb640e052014-04-03 16:41:46 -07001511 if not zbuild:
1512 # It has already been reported
1513 continue
1514 # It hasn't been reported yet.
1515 return False
1516 # Make sure that none of the worker connections are in GRAB_WAIT
James E. Blaire1767bc2016-08-02 10:00:27 -07001517 for connection in self.launch_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001518 if connection.state == 'GRAB_WAIT':
1519 return False
1520 return True
1521
1522 def areAllBuildsWaiting(self):
James E. Blaire1767bc2016-08-02 10:00:27 -07001523 builds = self.launch_client.builds.values()
Clark Boylanb640e052014-04-03 16:41:46 -07001524 for build in builds:
1525 client_job = None
James E. Blaire1767bc2016-08-02 10:00:27 -07001526 for conn in self.launch_client.gearman.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001527 for j in conn.related_jobs.values():
1528 if j.unique == build.uuid:
1529 client_job = j
1530 break
1531 if not client_job:
1532 self.log.debug("%s is not known to the gearman client" %
1533 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001534 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001535 if not client_job.handle:
1536 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001537 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001538 server_job = self.gearman_server.jobs.get(client_job.handle)
1539 if not server_job:
1540 self.log.debug("%s is not known to the gearman server" %
1541 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001542 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001543 if not hasattr(server_job, 'waiting'):
1544 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001545 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001546 if server_job.waiting:
1547 continue
James E. Blair17302972016-08-10 16:11:42 -07001548 if build.url is None:
James E. Blairbbda4702016-03-09 15:19:56 -08001549 self.log.debug("%s has not reported start" % build)
1550 return False
James E. Blairab7132b2016-08-05 12:36:22 -07001551 worker_build = self.launch_server.job_builds.get(server_job.unique)
James E. Blair962220f2016-08-03 11:22:38 -07001552 if worker_build:
1553 if worker_build.isWaiting():
1554 continue
1555 else:
1556 self.log.debug("%s is running" % worker_build)
1557 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001558 else:
James E. Blair962220f2016-08-03 11:22:38 -07001559 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001560 return False
1561 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001562
James E. Blairdce6cea2016-12-20 16:45:32 -08001563 def areAllNodeRequestsComplete(self):
James E. Blair15be0e12017-01-03 13:45:20 -08001564 if self.fake_nodepool.paused:
1565 return True
James E. Blairdce6cea2016-12-20 16:45:32 -08001566 if self.sched.nodepool.requests:
1567 return False
1568 return True
1569
Jan Hruban6b71aff2015-10-22 16:58:08 +02001570 def eventQueuesEmpty(self):
1571 for queue in self.event_queues:
1572 yield queue.empty()
1573
1574 def eventQueuesJoin(self):
1575 for queue in self.event_queues:
1576 queue.join()
1577
Clark Boylanb640e052014-04-03 16:41:46 -07001578 def waitUntilSettled(self):
1579 self.log.debug("Waiting until settled...")
1580 start = time.time()
1581 while True:
James E. Blair71932482017-02-02 11:29:07 -08001582 if time.time() - start > 20:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001583 self.log.error("Timeout waiting for Zuul to settle")
1584 self.log.error("Queue status:")
James E. Blair622c9682016-06-09 08:14:53 -07001585 for queue in self.event_queues:
James E. Blair10fc1eb2016-12-21 16:16:25 -08001586 self.log.error(" %s: %s" % (queue, queue.empty()))
1587 self.log.error("All builds waiting: %s" %
James E. Blair622c9682016-06-09 08:14:53 -07001588 (self.areAllBuildsWaiting(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001589 self.log.error("All builds reported: %s" %
James E. Blairf3156c92016-08-10 15:32:19 -07001590 (self.haveAllBuildsReported(),))
James E. Blair10fc1eb2016-12-21 16:16:25 -08001591 self.log.error("All requests completed: %s" %
1592 (self.areAllNodeRequestsComplete(),))
1593 self.log.error("Merge client jobs: %s" %
1594 (self.merge_client.jobs,))
Clark Boylanb640e052014-04-03 16:41:46 -07001595 raise Exception("Timeout waiting for Zuul to settle")
1596 # Make sure no new events show up while we're checking
James E. Blair3f876d52016-07-22 13:07:14 -07001597
James E. Blaire1767bc2016-08-02 10:00:27 -07001598 self.launch_server.lock.acquire()
Clark Boylanb640e052014-04-03 16:41:46 -07001599 # have all build states propogated to zuul?
1600 if self.haveAllBuildsReported():
1601 # Join ensures that the queue is empty _and_ events have been
1602 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001603 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001604 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001605 if (not self.merge_client.jobs and
Jan Hruban6b71aff2015-10-22 16:58:08 +02001606 all(self.eventQueuesEmpty()) and
Clark Boylanb640e052014-04-03 16:41:46 -07001607 self.haveAllBuildsReported() and
James E. Blairdce6cea2016-12-20 16:45:32 -08001608 self.areAllBuildsWaiting() and
1609 self.areAllNodeRequestsComplete()):
Clark Boylanb640e052014-04-03 16:41:46 -07001610 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001611 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001612 self.log.debug("...settled.")
1613 return
1614 self.sched.run_handler_lock.release()
James E. Blaire1767bc2016-08-02 10:00:27 -07001615 self.launch_server.lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001616 self.sched.wake_event.wait(0.1)
1617
1618 def countJobResults(self, jobs, result):
1619 jobs = filter(lambda x: x.result == result, jobs)
1620 return len(jobs)
1621
James E. Blair96c6bf82016-01-15 16:20:40 -08001622 def getJobFromHistory(self, name, project=None):
James E. Blair3f876d52016-07-22 13:07:14 -07001623 for job in self.history:
1624 if (job.name == name and
1625 (project is None or
1626 job.parameters['ZUUL_PROJECT'] == project)):
1627 return job
Clark Boylanb640e052014-04-03 16:41:46 -07001628 raise Exception("Unable to find job %s in history" % name)
1629
1630 def assertEmptyQueues(self):
1631 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001632 for tenant in self.sched.abide.tenants.values():
1633 for pipeline in tenant.layout.pipelines.values():
1634 for queue in pipeline.queues:
1635 if len(queue.queue) != 0:
Joshua Hesketh0aa7e8b2016-07-14 00:12:25 +10001636 print('pipeline %s queue %s contents %s' % (
1637 pipeline.name, queue.name, queue.queue))
James E. Blair59fdbac2015-12-07 17:08:06 -08001638 self.assertEqual(len(queue.queue), 0,
1639 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001640
1641 def assertReportedStat(self, key, value=None, kind=None):
1642 start = time.time()
1643 while time.time() < (start + 5):
1644 for stat in self.statsd.stats:
Clark Boylanb640e052014-04-03 16:41:46 -07001645 k, v = stat.split(':')
1646 if key == k:
1647 if value is None and kind is None:
1648 return
1649 elif value:
1650 if value == v:
1651 return
1652 elif kind:
1653 if v.endswith('|' + kind):
1654 return
1655 time.sleep(0.1)
1656
Clark Boylanb640e052014-04-03 16:41:46 -07001657 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001658
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001659 def assertBuilds(self, builds):
1660 """Assert that the running builds are as described.
1661
1662 The list of running builds is examined and must match exactly
1663 the list of builds described by the input.
1664
1665 :arg list builds: A list of dictionaries. Each item in the
1666 list must match the corresponding build in the build
1667 history, and each element of the dictionary must match the
1668 corresponding attribute of the build.
1669
1670 """
James E. Blair3158e282016-08-19 09:34:11 -07001671 try:
1672 self.assertEqual(len(self.builds), len(builds))
1673 for i, d in enumerate(builds):
1674 for k, v in d.items():
1675 self.assertEqual(
1676 getattr(self.builds[i], k), v,
1677 "Element %i in builds does not match" % (i,))
1678 except Exception:
1679 for build in self.builds:
1680 self.log.error("Running build: %s" % build)
1681 else:
1682 self.log.error("No running builds")
1683 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001684
James E. Blairb536ecc2016-08-31 10:11:42 -07001685 def assertHistory(self, history, ordered=True):
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001686 """Assert that the completed builds are as described.
1687
1688 The list of completed builds is examined and must match
1689 exactly the list of builds described by the input.
1690
1691 :arg list history: A list of dictionaries. Each item in the
1692 list must match the corresponding build in the build
1693 history, and each element of the dictionary must match the
1694 corresponding attribute of the build.
1695
James E. Blairb536ecc2016-08-31 10:11:42 -07001696 :arg bool ordered: If true, the history must match the order
1697 supplied, if false, the builds are permitted to have
1698 arrived in any order.
1699
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001700 """
James E. Blairb536ecc2016-08-31 10:11:42 -07001701 def matches(history_item, item):
1702 for k, v in item.items():
1703 if getattr(history_item, k) != v:
1704 return False
1705 return True
James E. Blair3158e282016-08-19 09:34:11 -07001706 try:
1707 self.assertEqual(len(self.history), len(history))
James E. Blairb536ecc2016-08-31 10:11:42 -07001708 if ordered:
1709 for i, d in enumerate(history):
1710 if not matches(self.history[i], d):
1711 raise Exception(
1712 "Element %i in history does not match" % (i,))
1713 else:
1714 unseen = self.history[:]
1715 for i, d in enumerate(history):
1716 found = False
1717 for unseen_item in unseen:
1718 if matches(unseen_item, d):
1719 found = True
1720 unseen.remove(unseen_item)
1721 break
1722 if not found:
1723 raise Exception("No match found for element %i "
1724 "in history" % (i,))
1725 if unseen:
1726 raise Exception("Unexpected items in history")
James E. Blair3158e282016-08-19 09:34:11 -07001727 except Exception:
1728 for build in self.history:
1729 self.log.error("Completed build: %s" % build)
1730 else:
1731 self.log.error("No completed builds")
1732 raise
James E. Blair2b2a8ab2016-08-11 14:39:11 -07001733
James E. Blair6ac368c2016-12-22 18:07:20 -08001734 def printHistory(self):
1735 """Log the build history.
1736
1737 This can be useful during tests to summarize what jobs have
1738 completed.
1739
1740 """
1741 self.log.debug("Build history:")
1742 for build in self.history:
1743 self.log.debug(build)
1744
James E. Blair59fdbac2015-12-07 17:08:06 -08001745 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001746 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1747
1748 def updateConfigLayout(self, path):
1749 root = os.path.join(self.test_root, "config")
1750 os.makedirs(root)
1751 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1752 f.write("""
Paul Belanger66e95962016-11-11 12:11:06 -05001753- tenant:
1754 name: openstack
1755 source:
1756 gerrit:
1757 config-repos:
1758 - %s
1759 """ % path)
James E. Blairf84026c2015-12-08 16:11:46 -08001760 f.close()
Paul Belanger66e95962016-11-11 12:11:06 -05001761 self.config.set('zuul', 'tenant_config',
1762 os.path.join(FIXTURE_DIR, f.name))
James E. Blair14abdf42015-12-09 16:11:53 -08001763
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001764 def addCommitToRepo(self, project, message, files,
1765 branch='master', tag=None):
James E. Blair14abdf42015-12-09 16:11:53 -08001766 path = os.path.join(self.upstream_root, project)
1767 repo = git.Repo(path)
1768 repo.head.reference = branch
1769 zuul.merger.merger.reset_repo_to_head(repo)
1770 for fn, content in files.items():
1771 fn = os.path.join(path, fn)
James E. Blairc73c73a2017-01-20 15:15:15 -08001772 try:
1773 os.makedirs(os.path.dirname(fn))
1774 except OSError:
1775 pass
James E. Blair14abdf42015-12-09 16:11:53 -08001776 with open(fn, 'w') as f:
1777 f.write(content)
1778 repo.index.add([fn])
1779 commit = repo.index.commit(message)
1780 repo.heads[branch].commit = commit
1781 repo.head.reference = branch
1782 repo.git.clean('-x', '-f', '-d')
1783 repo.heads[branch].checkout()
James E. Blair8b1dc3f2016-07-05 16:49:00 -07001784 if tag:
1785 repo.create_tag(tag)
James E. Blair3f876d52016-07-22 13:07:14 -07001786
James E. Blair7fc8daa2016-08-08 15:37:15 -07001787 def addEvent(self, connection, event):
1788 """Inject a Fake (Gerrit) event.
1789
1790 This method accepts a JSON-encoded event and simulates Zuul
1791 having received it from Gerrit. It could (and should)
1792 eventually apply to any connection type, but is currently only
1793 used with Gerrit connections. The name of the connection is
1794 used to look up the corresponding server, and the event is
1795 simulated as having been received by all Zuul connections
1796 attached to that server. So if two Gerrit connections in Zuul
1797 are connected to the same Gerrit server, and you invoke this
1798 method specifying the name of one of them, the event will be
1799 received by both.
1800
1801 .. note::
1802
1803 "self.fake_gerrit.addEvent" calls should be migrated to
1804 this method.
1805
1806 :arg str connection: The name of the connection corresponding
1807 to the gerrit server.
1808 :arg str event: The JSON-encoded event.
1809
1810 """
1811 specified_conn = self.connections.connections[connection]
1812 for conn in self.connections.connections.values():
1813 if (isinstance(conn, specified_conn.__class__) and
1814 specified_conn.server == conn.server):
1815 conn.addEvent(event)
1816
James E. Blair3f876d52016-07-22 13:07:14 -07001817
1818class AnsibleZuulTestCase(ZuulTestCase):
1819 """ZuulTestCase but with an actual ansible launcher running"""
James E. Blaire1767bc2016-08-02 10:00:27 -07001820 run_ansible = True