blob: d888904511f24ec7762e936655458d62a8245af7 [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may
6# not use this file except in compliance with the License. You may obtain
7# a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations
15# under the License.
16
Christian Berendtffba5df2014-06-07 21:30:22 +020017from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070018import gc
19import hashlib
20import json
21import logging
22import os
23import pprint
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Clark Boylanb640e052014-04-03 16:41:46 -070025import random
26import re
27import select
28import shutil
29import socket
30import string
31import subprocess
32import swiftclient
James E. Blairf84026c2015-12-08 16:11:46 -080033import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070034import threading
35import time
36import urllib2
37
38import git
39import gear
40import fixtures
41import six.moves.urllib.parse as urlparse
42import statsd
43import testtools
Mike Heald8225f522014-11-21 09:52:33 +000044from git import GitCommandError
Clark Boylanb640e052014-04-03 16:41:46 -070045
Joshua Hesketh352264b2015-08-11 23:42:08 +100046import zuul.connection.gerrit
47import zuul.connection.smtp
Clark Boylanb640e052014-04-03 16:41:46 -070048import zuul.scheduler
49import zuul.webapp
50import zuul.rpclistener
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +100051import zuul.launcher.server
52import zuul.launcher.client
Clark Boylanb640e052014-04-03 16:41:46 -070053import zuul.lib.swift
James E. Blair83005782015-12-11 14:46:03 -080054import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070055import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070056import zuul.merger.merger
57import zuul.merger.server
Clark Boylanb640e052014-04-03 16:41:46 -070058import zuul.reporter.gerrit
59import zuul.reporter.smtp
Joshua Hesketh850ccb62014-11-27 11:31:02 +110060import zuul.source.gerrit
Clark Boylanb640e052014-04-03 16:41:46 -070061import zuul.trigger.gerrit
62import zuul.trigger.timer
James E. Blairc494d542014-08-06 09:23:52 -070063import zuul.trigger.zuultrigger
Clark Boylanb640e052014-04-03 16:41:46 -070064
65FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
66 'fixtures')
James E. Blair97d902e2014-08-21 13:25:56 -070067USE_TEMPDIR = True
Clark Boylanb640e052014-04-03 16:41:46 -070068
69logging.basicConfig(level=logging.DEBUG,
70 format='%(asctime)s %(name)-32s '
71 '%(levelname)-8s %(message)s')
72
73
74def repack_repo(path):
75 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
76 output = subprocess.Popen(cmd, close_fds=True,
77 stdout=subprocess.PIPE,
78 stderr=subprocess.PIPE)
79 out = output.communicate()
80 if output.returncode:
81 raise Exception("git repack returned %d" % output.returncode)
82 return out
83
84
85def random_sha1():
86 return hashlib.sha1(str(random.random())).hexdigest()
87
88
James E. Blaira190f3b2015-01-05 14:56:54 -080089def iterate_timeout(max_seconds, purpose):
90 start = time.time()
91 count = 0
92 while (time.time() < start + max_seconds):
93 count += 1
94 yield count
95 time.sleep(0)
96 raise Exception("Timeout waiting for %s" % purpose)
97
98
Clark Boylanb640e052014-04-03 16:41:46 -070099class ChangeReference(git.Reference):
100 _common_path_default = "refs/changes"
101 _points_to_commits_only = True
102
103
104class FakeChange(object):
105 categories = {'APRV': ('Approved', -1, 1),
106 'CRVW': ('Code-Review', -2, 2),
107 'VRFY': ('Verified', -2, 2)}
108
109 def __init__(self, gerrit, number, project, branch, subject,
110 status='NEW', upstream_root=None):
111 self.gerrit = gerrit
112 self.reported = 0
113 self.queried = 0
114 self.patchsets = []
115 self.number = number
116 self.project = project
117 self.branch = branch
118 self.subject = subject
119 self.latest_patchset = 0
120 self.depends_on_change = None
121 self.needed_by_changes = []
122 self.fail_merge = False
123 self.messages = []
124 self.data = {
125 'branch': branch,
126 'comments': [],
127 'commitMessage': subject,
128 'createdOn': time.time(),
129 'id': 'I' + random_sha1(),
130 'lastUpdated': time.time(),
131 'number': str(number),
132 'open': status == 'NEW',
133 'owner': {'email': 'user@example.com',
134 'name': 'User Name',
135 'username': 'username'},
136 'patchSets': self.patchsets,
137 'project': project,
138 'status': status,
139 'subject': subject,
140 'submitRecords': [],
141 'url': 'https://hostname/%s' % number}
142
143 self.upstream_root = upstream_root
144 self.addPatchset()
145 self.data['submitRecords'] = self.getSubmitRecords()
146 self.open = status == 'NEW'
147
148 def add_fake_change_to_repo(self, msg, fn, large):
149 path = os.path.join(self.upstream_root, self.project)
150 repo = git.Repo(path)
151 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
152 self.latest_patchset),
153 'refs/tags/init')
154 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700155 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700156 repo.git.clean('-x', '-f', '-d')
157
158 path = os.path.join(self.upstream_root, self.project)
159 if not large:
160 fn = os.path.join(path, fn)
161 f = open(fn, 'w')
162 f.write("test %s %s %s\n" %
163 (self.branch, self.number, self.latest_patchset))
164 f.close()
165 repo.index.add([fn])
166 else:
167 for fni in range(100):
168 fn = os.path.join(path, str(fni))
169 f = open(fn, 'w')
170 for ci in range(4096):
171 f.write(random.choice(string.printable))
172 f.close()
173 repo.index.add([fn])
174
175 r = repo.index.commit(msg)
176 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700177 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700178 repo.git.clean('-x', '-f', '-d')
179 repo.heads['master'].checkout()
180 return r
181
182 def addPatchset(self, files=[], large=False):
183 self.latest_patchset += 1
184 if files:
185 fn = files[0]
186 else:
James E. Blair97d902e2014-08-21 13:25:56 -0700187 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
Clark Boylanb640e052014-04-03 16:41:46 -0700188 msg = self.subject + '-' + str(self.latest_patchset)
189 c = self.add_fake_change_to_repo(msg, fn, large)
190 ps_files = [{'file': '/COMMIT_MSG',
191 'type': 'ADDED'},
192 {'file': 'README',
193 'type': 'MODIFIED'}]
194 for f in files:
195 ps_files.append({'file': f, 'type': 'ADDED'})
196 d = {'approvals': [],
197 'createdOn': time.time(),
198 'files': ps_files,
199 'number': str(self.latest_patchset),
200 'ref': 'refs/changes/1/%s/%s' % (self.number,
201 self.latest_patchset),
202 'revision': c.hexsha,
203 'uploader': {'email': 'user@example.com',
204 'name': 'User name',
205 'username': 'user'}}
206 self.data['currentPatchSet'] = d
207 self.patchsets.append(d)
208 self.data['submitRecords'] = self.getSubmitRecords()
209
210 def getPatchsetCreatedEvent(self, patchset):
211 event = {"type": "patchset-created",
212 "change": {"project": self.project,
213 "branch": self.branch,
214 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
215 "number": str(self.number),
216 "subject": self.subject,
217 "owner": {"name": "User Name"},
218 "url": "https://hostname/3"},
219 "patchSet": self.patchsets[patchset - 1],
220 "uploader": {"name": "User Name"}}
221 return event
222
223 def getChangeRestoredEvent(self):
224 event = {"type": "change-restored",
225 "change": {"project": self.project,
226 "branch": self.branch,
227 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
228 "number": str(self.number),
229 "subject": self.subject,
230 "owner": {"name": "User Name"},
231 "url": "https://hostname/3"},
232 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100233 "patchSet": self.patchsets[-1],
234 "reason": ""}
235 return event
236
237 def getChangeAbandonedEvent(self):
238 event = {"type": "change-abandoned",
239 "change": {"project": self.project,
240 "branch": self.branch,
241 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
242 "number": str(self.number),
243 "subject": self.subject,
244 "owner": {"name": "User Name"},
245 "url": "https://hostname/3"},
246 "abandoner": {"name": "User Name"},
247 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700248 "reason": ""}
249 return event
250
251 def getChangeCommentEvent(self, patchset):
252 event = {"type": "comment-added",
253 "change": {"project": self.project,
254 "branch": self.branch,
255 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
256 "number": str(self.number),
257 "subject": self.subject,
258 "owner": {"name": "User Name"},
259 "url": "https://hostname/3"},
260 "patchSet": self.patchsets[patchset - 1],
261 "author": {"name": "User Name"},
262 "approvals": [{"type": "Code-Review",
263 "description": "Code-Review",
264 "value": "0"}],
265 "comment": "This is a comment"}
266 return event
267
Joshua Hesketh642824b2014-07-01 17:54:59 +1000268 def addApproval(self, category, value, username='reviewer_john',
269 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700270 if not granted_on:
271 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000272 approval = {
273 'description': self.categories[category][0],
274 'type': category,
275 'value': str(value),
276 'by': {
277 'username': username,
278 'email': username + '@example.com',
279 },
280 'grantedOn': int(granted_on)
281 }
Clark Boylanb640e052014-04-03 16:41:46 -0700282 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
283 if x['by']['username'] == username and x['type'] == category:
284 del self.patchsets[-1]['approvals'][i]
285 self.patchsets[-1]['approvals'].append(approval)
286 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000287 'author': {'email': 'author@example.com',
288 'name': 'Patchset Author',
289 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700290 'change': {'branch': self.branch,
291 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
292 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000293 'owner': {'email': 'owner@example.com',
294 'name': 'Change Owner',
295 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700296 'project': self.project,
297 'subject': self.subject,
298 'topic': 'master',
299 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000300 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700301 'patchSet': self.patchsets[-1],
302 'type': 'comment-added'}
303 self.data['submitRecords'] = self.getSubmitRecords()
304 return json.loads(json.dumps(event))
305
306 def getSubmitRecords(self):
307 status = {}
308 for cat in self.categories.keys():
309 status[cat] = 0
310
311 for a in self.patchsets[-1]['approvals']:
312 cur = status[a['type']]
313 cat_min, cat_max = self.categories[a['type']][1:]
314 new = int(a['value'])
315 if new == cat_min:
316 cur = new
317 elif abs(new) > abs(cur):
318 cur = new
319 status[a['type']] = cur
320
321 labels = []
322 ok = True
323 for typ, cat in self.categories.items():
324 cur = status[typ]
325 cat_min, cat_max = cat[1:]
326 if cur == cat_min:
327 value = 'REJECT'
328 ok = False
329 elif cur == cat_max:
330 value = 'OK'
331 else:
332 value = 'NEED'
333 ok = False
334 labels.append({'label': cat[0], 'status': value})
335 if ok:
336 return [{'status': 'OK'}]
337 return [{'status': 'NOT_READY',
338 'labels': labels}]
339
340 def setDependsOn(self, other, patchset):
341 self.depends_on_change = other
342 d = {'id': other.data['id'],
343 'number': other.data['number'],
344 'ref': other.patchsets[patchset - 1]['ref']
345 }
346 self.data['dependsOn'] = [d]
347
348 other.needed_by_changes.append(self)
349 needed = other.data.get('neededBy', [])
350 d = {'id': self.data['id'],
351 'number': self.data['number'],
352 'ref': self.patchsets[patchset - 1]['ref'],
353 'revision': self.patchsets[patchset - 1]['revision']
354 }
355 needed.append(d)
356 other.data['neededBy'] = needed
357
358 def query(self):
359 self.queried += 1
360 d = self.data.get('dependsOn')
361 if d:
362 d = d[0]
363 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
364 d['isCurrentPatchSet'] = True
365 else:
366 d['isCurrentPatchSet'] = False
367 return json.loads(json.dumps(self.data))
368
369 def setMerged(self):
370 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000371 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700372 return
373 if self.fail_merge:
374 return
375 self.data['status'] = 'MERGED'
376 self.open = False
377
378 path = os.path.join(self.upstream_root, self.project)
379 repo = git.Repo(path)
380 repo.heads[self.branch].commit = \
381 repo.commit(self.patchsets[-1]['revision'])
382
383 def setReported(self):
384 self.reported += 1
385
386
Joshua Hesketh352264b2015-08-11 23:42:08 +1000387class FakeGerritConnection(zuul.connection.gerrit.GerritConnection):
388 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700389
Joshua Hesketh352264b2015-08-11 23:42:08 +1000390 def __init__(self, connection_name, connection_config,
Jan Hruban6b71aff2015-10-22 16:58:08 +0200391 changes_db=None, queues_db=None, upstream_root=None):
Joshua Hesketh352264b2015-08-11 23:42:08 +1000392 super(FakeGerritConnection, self).__init__(connection_name,
393 connection_config)
394
395 self.event_queue = queues_db
Clark Boylanb640e052014-04-03 16:41:46 -0700396 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
397 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000398 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700399 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200400 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700401
402 def addFakeChange(self, project, branch, subject, status='NEW'):
403 self.change_number += 1
404 c = FakeChange(self, self.change_number, project, branch, subject,
405 upstream_root=self.upstream_root,
406 status=status)
407 self.changes[self.change_number] = c
408 return c
409
Clark Boylanb640e052014-04-03 16:41:46 -0700410 def review(self, project, changeid, message, action):
411 number, ps = changeid.split(',')
412 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000413
414 # Add the approval back onto the change (ie simulate what gerrit would
415 # do).
416 # Usually when zuul leaves a review it'll create a feedback loop where
417 # zuul's review enters another gerrit event (which is then picked up by
418 # zuul). However, we can't mimic this behaviour (by adding this
419 # approval event into the queue) as it stops jobs from checking what
420 # happens before this event is triggered. If a job needs to see what
421 # happens they can add their own verified event into the queue.
422 # Nevertheless, we can update change with the new review in gerrit.
423
424 for cat in ['CRVW', 'VRFY', 'APRV']:
425 if cat in action:
Joshua Hesketh352264b2015-08-11 23:42:08 +1000426 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000427
428 if 'label' in action:
429 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000430 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000431
Clark Boylanb640e052014-04-03 16:41:46 -0700432 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000433
Clark Boylanb640e052014-04-03 16:41:46 -0700434 if 'submit' in action:
435 change.setMerged()
436 if message:
437 change.setReported()
438
439 def query(self, number):
440 change = self.changes.get(int(number))
441 if change:
442 return change.query()
443 return {}
444
James E. Blairc494d542014-08-06 09:23:52 -0700445 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700446 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700447 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800448 if query.startswith('change:'):
449 # Query a specific changeid
450 changeid = query[len('change:'):]
451 l = [change.query() for change in self.changes.values()
452 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700453 elif query.startswith('message:'):
454 # Query the content of a commit message
455 msg = query[len('message:'):].strip()
456 l = [change.query() for change in self.changes.values()
457 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800458 else:
459 # Query all open changes
460 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700461 return l
James E. Blairc494d542014-08-06 09:23:52 -0700462
Joshua Hesketh352264b2015-08-11 23:42:08 +1000463 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700464 pass
465
Joshua Hesketh352264b2015-08-11 23:42:08 +1000466 def getGitUrl(self, project):
467 return os.path.join(self.upstream_root, project.name)
468
Clark Boylanb640e052014-04-03 16:41:46 -0700469
470class BuildHistory(object):
471 def __init__(self, **kw):
472 self.__dict__.update(kw)
473
474 def __repr__(self):
475 return ("<Completed build, result: %s name: %s #%s changes: %s>" %
476 (self.result, self.name, self.number, self.changes))
477
478
479class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200480 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700481 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700482 self.url = url
483
484 def read(self):
485 res = urlparse.urlparse(self.url)
486 path = res.path
487 project = '/'.join(path.split('/')[2:-2])
488 ret = '001e# service=git-upload-pack\n'
489 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
490 'multi_ack thin-pack side-band side-band-64k ofs-delta '
491 'shallow no-progress include-tag multi_ack_detailed no-done\n')
492 path = os.path.join(self.upstream_root, project)
493 repo = git.Repo(path)
494 for ref in repo.refs:
495 r = ref.object.hexsha + ' ' + ref.path + '\n'
496 ret += '%04x%s' % (len(r) + 4, r)
497 ret += '0000'
498 return ret
499
500
Clark Boylanb640e052014-04-03 16:41:46 -0700501class FakeStatsd(threading.Thread):
502 def __init__(self):
503 threading.Thread.__init__(self)
504 self.daemon = True
505 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
506 self.sock.bind(('', 0))
507 self.port = self.sock.getsockname()[1]
508 self.wake_read, self.wake_write = os.pipe()
509 self.stats = []
510
511 def run(self):
512 while True:
513 poll = select.poll()
514 poll.register(self.sock, select.POLLIN)
515 poll.register(self.wake_read, select.POLLIN)
516 ret = poll.poll()
517 for (fd, event) in ret:
518 if fd == self.sock.fileno():
519 data = self.sock.recvfrom(1024)
520 if not data:
521 return
522 self.stats.append(data[0])
523 if fd == self.wake_read:
524 return
525
526 def stop(self):
527 os.write(self.wake_write, '1\n')
528
529
530class FakeBuild(threading.Thread):
531 log = logging.getLogger("zuul.test")
532
533 def __init__(self, worker, job, number, node):
534 threading.Thread.__init__(self)
535 self.daemon = True
536 self.worker = worker
537 self.job = job
538 self.name = job.name.split(':')[1]
539 self.number = number
540 self.node = node
541 self.parameters = json.loads(job.arguments)
542 self.unique = self.parameters['ZUUL_UUID']
543 self.wait_condition = threading.Condition()
544 self.waiting = False
545 self.aborted = False
546 self.created = time.time()
547 self.description = ''
548 self.run_error = False
549
550 def release(self):
551 self.wait_condition.acquire()
552 self.wait_condition.notify()
553 self.waiting = False
554 self.log.debug("Build %s released" % self.unique)
555 self.wait_condition.release()
556
557 def isWaiting(self):
558 self.wait_condition.acquire()
559 if self.waiting:
560 ret = True
561 else:
562 ret = False
563 self.wait_condition.release()
564 return ret
565
566 def _wait(self):
567 self.wait_condition.acquire()
568 self.waiting = True
569 self.log.debug("Build %s waiting" % self.unique)
570 self.wait_condition.wait()
571 self.wait_condition.release()
572
573 def run(self):
574 data = {
575 'url': 'https://server/job/%s/%s/' % (self.name, self.number),
576 'name': self.name,
577 'number': self.number,
578 'manager': self.worker.worker_id,
579 'worker_name': 'My Worker',
580 'worker_hostname': 'localhost',
581 'worker_ips': ['127.0.0.1', '192.168.1.1'],
582 'worker_fqdn': 'zuul.example.org',
583 'worker_program': 'FakeBuilder',
584 'worker_version': 'v1.1',
585 'worker_extra': {'something': 'else'}
586 }
587
588 self.log.debug('Running build %s' % self.unique)
589
590 self.job.sendWorkData(json.dumps(data))
591 self.log.debug('Sent WorkData packet with %s' % json.dumps(data))
592 self.job.sendWorkStatus(0, 100)
593
594 if self.worker.hold_jobs_in_build:
595 self.log.debug('Holding build %s' % self.unique)
596 self._wait()
597 self.log.debug("Build %s continuing" % self.unique)
598
599 self.worker.lock.acquire()
600
601 result = 'SUCCESS'
602 if (('ZUUL_REF' in self.parameters) and
603 self.worker.shouldFailTest(self.name,
604 self.parameters['ZUUL_REF'])):
605 result = 'FAILURE'
606 if self.aborted:
607 result = 'ABORTED'
608
609 if self.run_error:
610 work_fail = True
611 result = 'RUN_ERROR'
612 else:
613 data['result'] = result
Timothy Chavezb2332082015-08-07 20:08:04 -0500614 data['node_labels'] = ['bare-necessities']
615 data['node_name'] = 'foo'
Clark Boylanb640e052014-04-03 16:41:46 -0700616 work_fail = False
617
618 changes = None
619 if 'ZUUL_CHANGE_IDS' in self.parameters:
620 changes = self.parameters['ZUUL_CHANGE_IDS']
621
622 self.worker.build_history.append(
623 BuildHistory(name=self.name, number=self.number,
624 result=result, changes=changes, node=self.node,
625 uuid=self.unique, description=self.description,
James E. Blair456f2fb2016-02-09 09:29:33 -0800626 parameters=self.parameters,
Clark Boylanb640e052014-04-03 16:41:46 -0700627 pipeline=self.parameters['ZUUL_PIPELINE'])
628 )
629
630 self.job.sendWorkData(json.dumps(data))
631 if work_fail:
632 self.job.sendWorkFail()
633 else:
634 self.job.sendWorkComplete(json.dumps(data))
635 del self.worker.gearman_jobs[self.job.unique]
636 self.worker.running_builds.remove(self)
637 self.worker.lock.release()
638
639
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +1000640class RecordingLaunchServer(zuul.launcher.server.LaunchServer):
James E. Blairf5dbd002015-12-23 15:26:17 -0800641 def __init__(self, *args, **kw):
642 super(RecordingLaunchServer, self).__init__(*args, **kw)
643 self.job_history = []
644
645 def launch(self, job):
646 self.job_history.append(job)
647 job.data = []
648
649 def sendWorkComplete(data=b''):
650 job.data.append(data)
651 gear.WorkerJob.sendWorkComplete(job, data)
652
653 job.sendWorkComplete = sendWorkComplete
654 super(RecordingLaunchServer, self).launch(job)
655
656
Clark Boylanb640e052014-04-03 16:41:46 -0700657class FakeWorker(gear.Worker):
658 def __init__(self, worker_id, test):
659 super(FakeWorker, self).__init__(worker_id)
660 self.gearman_jobs = {}
661 self.build_history = []
662 self.running_builds = []
663 self.build_counter = 0
664 self.fail_tests = {}
665 self.test = test
666
667 self.hold_jobs_in_build = False
668 self.lock = threading.Lock()
669 self.__work_thread = threading.Thread(target=self.work)
670 self.__work_thread.daemon = True
671 self.__work_thread.start()
672
673 def handleJob(self, job):
674 parts = job.name.split(":")
675 cmd = parts[0]
676 name = parts[1]
677 if len(parts) > 2:
678 node = parts[2]
679 else:
680 node = None
681 if cmd == 'build':
682 self.handleBuild(job, name, node)
683 elif cmd == 'stop':
684 self.handleStop(job, name)
685 elif cmd == 'set_description':
686 self.handleSetDescription(job, name)
687
688 def handleBuild(self, job, name, node):
689 build = FakeBuild(self, job, self.build_counter, node)
690 job.build = build
691 self.gearman_jobs[job.unique] = job
692 self.build_counter += 1
693
694 self.running_builds.append(build)
695 build.start()
696
697 def handleStop(self, job, name):
698 self.log.debug("handle stop")
699 parameters = json.loads(job.arguments)
700 name = parameters['name']
701 number = parameters['number']
702 for build in self.running_builds:
703 if build.name == name and build.number == number:
704 build.aborted = True
705 build.release()
706 job.sendWorkComplete()
707 return
708 job.sendWorkFail()
709
710 def handleSetDescription(self, job, name):
711 self.log.debug("handle set description")
712 parameters = json.loads(job.arguments)
713 name = parameters['name']
714 number = parameters['number']
715 descr = parameters['html_description']
716 for build in self.running_builds:
717 if build.name == name and build.number == number:
718 build.description = descr
719 job.sendWorkComplete()
720 return
721 for build in self.build_history:
722 if build.name == name and build.number == number:
723 build.description = descr
724 job.sendWorkComplete()
725 return
726 job.sendWorkFail()
727
728 def work(self):
729 while self.running:
730 try:
731 job = self.getJob()
732 except gear.InterruptedError:
733 continue
734 try:
735 self.handleJob(job)
736 except:
737 self.log.exception("Worker exception:")
738
739 def addFailTest(self, name, change):
740 l = self.fail_tests.get(name, [])
741 l.append(change)
742 self.fail_tests[name] = l
743
744 def shouldFailTest(self, name, ref):
745 l = self.fail_tests.get(name, [])
746 for change in l:
747 if self.test.ref_has_change(ref, change):
748 return True
749 return False
750
751 def release(self, regex=None):
752 builds = self.running_builds[:]
753 self.log.debug("releasing build %s (%s)" % (regex,
754 len(self.running_builds)))
755 for build in builds:
756 if not regex or re.match(regex, build.name):
757 self.log.debug("releasing build %s" %
758 (build.parameters['ZUUL_UUID']))
759 build.release()
760 else:
761 self.log.debug("not releasing build %s" %
762 (build.parameters['ZUUL_UUID']))
763 self.log.debug("done releasing builds %s (%s)" %
764 (regex, len(self.running_builds)))
765
766
767class FakeGearmanServer(gear.Server):
768 def __init__(self):
769 self.hold_jobs_in_queue = False
770 super(FakeGearmanServer, self).__init__(0)
771
772 def getJobForConnection(self, connection, peek=False):
773 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
774 for job in queue:
775 if not hasattr(job, 'waiting'):
776 if job.name.startswith('build:'):
777 job.waiting = self.hold_jobs_in_queue
778 else:
779 job.waiting = False
780 if job.waiting:
781 continue
782 if job.name in connection.functions:
783 if not peek:
784 queue.remove(job)
785 connection.related_jobs[job.handle] = job
786 job.worker_connection = connection
787 job.running = True
788 return job
789 return None
790
791 def release(self, regex=None):
792 released = False
793 qlen = (len(self.high_queue) + len(self.normal_queue) +
794 len(self.low_queue))
795 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
796 for job in self.getQueue():
797 cmd, name = job.name.split(':')
798 if cmd != 'build':
799 continue
800 if not regex or re.match(regex, name):
801 self.log.debug("releasing queued job %s" %
802 job.unique)
803 job.waiting = False
804 released = True
805 else:
806 self.log.debug("not releasing queued job %s" %
807 job.unique)
808 if released:
809 self.wakeConnections()
810 qlen = (len(self.high_queue) + len(self.normal_queue) +
811 len(self.low_queue))
812 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
813
814
815class FakeSMTP(object):
816 log = logging.getLogger('zuul.FakeSMTP')
817
818 def __init__(self, messages, server, port):
819 self.server = server
820 self.port = port
821 self.messages = messages
822
823 def sendmail(self, from_email, to_email, msg):
824 self.log.info("Sending email from %s, to %s, with msg %s" % (
825 from_email, to_email, msg))
826
827 headers = msg.split('\n\n', 1)[0]
828 body = msg.split('\n\n', 1)[1]
829
830 self.messages.append(dict(
831 from_email=from_email,
832 to_email=to_email,
833 msg=msg,
834 headers=headers,
835 body=body,
836 ))
837
838 return True
839
840 def quit(self):
841 return True
842
843
844class FakeSwiftClientConnection(swiftclient.client.Connection):
845 def post_account(self, headers):
846 # Do nothing
847 pass
848
849 def get_auth(self):
850 # Returns endpoint and (unused) auth token
851 endpoint = os.path.join('https://storage.example.org', 'V1',
852 'AUTH_account')
853 return endpoint, ''
854
855
Maru Newby3fe5f852015-01-13 04:22:14 +0000856class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -0700857 log = logging.getLogger("zuul.test")
858
859 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +0000860 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -0700861 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
862 try:
863 test_timeout = int(test_timeout)
864 except ValueError:
865 # If timeout value is invalid do not set a timeout.
866 test_timeout = 0
867 if test_timeout > 0:
868 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
869
870 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
871 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
872 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
873 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
874 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
875 os.environ.get('OS_STDERR_CAPTURE') == '1'):
876 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
877 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
878 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
879 os.environ.get('OS_LOG_CAPTURE') == '1'):
880 self.useFixture(fixtures.FakeLogger(
881 level=logging.DEBUG,
882 format='%(asctime)s %(name)-32s '
883 '%(levelname)-8s %(message)s'))
Maru Newby3fe5f852015-01-13 04:22:14 +0000884
885
886class ZuulTestCase(BaseTestCase):
James E. Blair83005782015-12-11 14:46:03 -0800887 config_file = 'zuul.conf'
Maru Newby3fe5f852015-01-13 04:22:14 +0000888
889 def setUp(self):
890 super(ZuulTestCase, self).setUp()
James E. Blair97d902e2014-08-21 13:25:56 -0700891 if USE_TEMPDIR:
892 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000893 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
894 ).path
James E. Blair97d902e2014-08-21 13:25:56 -0700895 else:
896 tmp_root = os.environ.get("ZUUL_TEST_ROOT")
Clark Boylanb640e052014-04-03 16:41:46 -0700897 self.test_root = os.path.join(tmp_root, "zuul-test")
898 self.upstream_root = os.path.join(self.test_root, "upstream")
899 self.git_root = os.path.join(self.test_root, "git")
900
901 if os.path.exists(self.test_root):
902 shutil.rmtree(self.test_root)
903 os.makedirs(self.test_root)
904 os.makedirs(self.upstream_root)
Clark Boylanb640e052014-04-03 16:41:46 -0700905
906 # Make per test copy of Configuration.
907 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -0800908 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +1100909 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -0800910 self.config.get('zuul', 'tenant_config')))
Clark Boylanb640e052014-04-03 16:41:46 -0700911 self.config.set('merger', 'git_dir', self.git_root)
912
913 # For each project in config:
914 self.init_repo("org/project")
915 self.init_repo("org/project1")
916 self.init_repo("org/project2")
917 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -0700918 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -0700919 self.init_repo("org/project5")
920 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -0700921 self.init_repo("org/one-job-project")
922 self.init_repo("org/nonvoting-project")
923 self.init_repo("org/templated-project")
924 self.init_repo("org/layered-project")
925 self.init_repo("org/node-project")
926 self.init_repo("org/conflict-project")
927 self.init_repo("org/noop-project")
928 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +0000929 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -0700930
James E. Blair83005782015-12-11 14:46:03 -0800931 self.setup_repos()
932
Clark Boylanb640e052014-04-03 16:41:46 -0700933 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +1000934 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
935 # see: https://github.com/jsocol/pystatsd/issues/61
936 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -0700937 os.environ['STATSD_PORT'] = str(self.statsd.port)
938 self.statsd.start()
939 # the statsd client object is configured in the statsd module import
940 reload(statsd)
941 reload(zuul.scheduler)
942
943 self.gearman_server = FakeGearmanServer()
944
945 self.config.set('gearman', 'port', str(self.gearman_server.port))
946
Joshua Hesketh352264b2015-08-11 23:42:08 +1000947 zuul.source.gerrit.GerritSource.replication_timeout = 1.5
948 zuul.source.gerrit.GerritSource.replication_retry_interval = 0.5
949 zuul.connection.gerrit.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -0700950
Joshua Hesketh352264b2015-08-11 23:42:08 +1000951 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -0700952
953 self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
954 FakeSwiftClientConnection))
955 self.swift = zuul.lib.swift.Swift(self.config)
956
Jan Hruban6b71aff2015-10-22 16:58:08 +0200957 self.event_queues = [
958 self.sched.result_event_queue,
959 self.sched.trigger_event_queue
960 ]
961
James E. Blairfef78942016-03-11 16:28:56 -0800962 self.configure_connections()
Joshua Hesketh352264b2015-08-11 23:42:08 +1000963 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +1000964
James E. Blairf5dbd002015-12-23 15:26:17 -0800965 self.ansible_server = RecordingLaunchServer(
966 self.config, self.connections)
967 self.ansible_server.start()
968
Clark Boylanb640e052014-04-03 16:41:46 -0700969 def URLOpenerFactory(*args, **kw):
970 if isinstance(args[0], urllib2.Request):
971 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -0700972 return FakeURLOpener(self.upstream_root, *args, **kw)
973
974 old_urlopen = urllib2.urlopen
975 urllib2.urlopen = URLOpenerFactory
976
Joshua Hesketh0c54b2a2016-04-11 21:23:33 +1000977 self.launcher = zuul.launcher.client.LaunchClient(
James E. Blair82938472016-01-11 14:38:13 -0800978 self.config, self.sched, self.swift)
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100979 self.merge_client = zuul.merger.client.MergeClient(
980 self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -0700981
982 self.sched.setLauncher(self.launcher)
983 self.sched.setMerger(self.merge_client)
Clark Boylanb640e052014-04-03 16:41:46 -0700984
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100985 self.webapp = zuul.webapp.WebApp(self.sched, port=0)
986 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -0700987
988 self.sched.start()
989 self.sched.reconfigure(self.config)
990 self.sched.resume()
991 self.webapp.start()
992 self.rpc.start()
993 self.launcher.gearman.waitForServer()
Clark Boylanb640e052014-04-03 16:41:46 -0700994
995 self.addCleanup(self.assertFinalState)
996 self.addCleanup(self.shutdown)
997
James E. Blairfef78942016-03-11 16:28:56 -0800998 def configure_connections(self):
Joshua Hesketh352264b2015-08-11 23:42:08 +1000999 # Register connections from the config
1000 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001001
Joshua Hesketh352264b2015-08-11 23:42:08 +10001002 def FakeSMTPFactory(*args, **kw):
1003 args = [self.smtp_messages] + list(args)
1004 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001005
Joshua Hesketh352264b2015-08-11 23:42:08 +10001006 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001007
Joshua Hesketh352264b2015-08-11 23:42:08 +10001008 # Set a changes database so multiple FakeGerrit's can report back to
1009 # a virtual canonical database given by the configured hostname
1010 self.gerrit_changes_dbs = {}
1011 self.gerrit_queues_dbs = {}
James E. Blairfef78942016-03-11 16:28:56 -08001012 self.connections = zuul.lib.connections.ConnectionRegistry()
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001013
Joshua Hesketh352264b2015-08-11 23:42:08 +10001014 for section_name in self.config.sections():
1015 con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
1016 section_name, re.I)
1017 if not con_match:
1018 continue
1019 con_name = con_match.group(2)
1020 con_config = dict(self.config.items(section_name))
1021
1022 if 'driver' not in con_config:
1023 raise Exception("No driver specified for connection %s."
1024 % con_name)
1025
1026 con_driver = con_config['driver']
1027
1028 # TODO(jhesketh): load the required class automatically
1029 if con_driver == 'gerrit':
Joshua Heskethacccffc2015-03-31 23:38:17 +11001030 if con_config['server'] not in self.gerrit_changes_dbs.keys():
1031 self.gerrit_changes_dbs[con_config['server']] = {}
1032 if con_config['server'] not in self.gerrit_queues_dbs.keys():
1033 self.gerrit_queues_dbs[con_config['server']] = \
1034 Queue.Queue()
1035 self.event_queues.append(
1036 self.gerrit_queues_dbs[con_config['server']])
James E. Blair83005782015-12-11 14:46:03 -08001037 self.connections.connections[con_name] = FakeGerritConnection(
Joshua Hesketh352264b2015-08-11 23:42:08 +10001038 con_name, con_config,
Joshua Heskethacccffc2015-03-31 23:38:17 +11001039 changes_db=self.gerrit_changes_dbs[con_config['server']],
1040 queues_db=self.gerrit_queues_dbs[con_config['server']],
Jan Hruban6b71aff2015-10-22 16:58:08 +02001041 upstream_root=self.upstream_root
Joshua Hesketh352264b2015-08-11 23:42:08 +10001042 )
James E. Blair83005782015-12-11 14:46:03 -08001043 setattr(self, 'fake_' + con_name,
1044 self.connections.connections[con_name])
Joshua Hesketh352264b2015-08-11 23:42:08 +10001045 elif con_driver == 'smtp':
James E. Blair83005782015-12-11 14:46:03 -08001046 self.connections.connections[con_name] = \
Joshua Hesketh352264b2015-08-11 23:42:08 +10001047 zuul.connection.smtp.SMTPConnection(con_name, con_config)
1048 else:
1049 raise Exception("Unknown driver, %s, for connection %s"
1050 % (con_config['driver'], con_name))
1051
1052 # If the [gerrit] or [smtp] sections still exist, load them in as a
1053 # connection named 'gerrit' or 'smtp' respectfully
1054
1055 if 'gerrit' in self.config.sections():
1056 self.gerrit_changes_dbs['gerrit'] = {}
1057 self.gerrit_queues_dbs['gerrit'] = Queue.Queue()
Jan Hruban6b71aff2015-10-22 16:58:08 +02001058 self.event_queues.append(self.gerrit_queues_dbs['gerrit'])
James E. Blair83005782015-12-11 14:46:03 -08001059 self.connections.connections['gerrit'] = FakeGerritConnection(
Joshua Hesketh352264b2015-08-11 23:42:08 +10001060 '_legacy_gerrit', dict(self.config.items('gerrit')),
1061 changes_db=self.gerrit_changes_dbs['gerrit'],
1062 queues_db=self.gerrit_queues_dbs['gerrit'])
1063
1064 if 'smtp' in self.config.sections():
James E. Blair83005782015-12-11 14:46:03 -08001065 self.connections.connections['smtp'] = \
Joshua Hesketh352264b2015-08-11 23:42:08 +10001066 zuul.connection.smtp.SMTPConnection(
1067 '_legacy_smtp', dict(self.config.items('smtp')))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001068
James E. Blair83005782015-12-11 14:46:03 -08001069 def setup_config(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001070 """Per test config object. Override to set different config."""
1071 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001072 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
James E. Blair2a629ec2015-12-22 15:32:02 -08001073 if hasattr(self, 'tenant_config_file'):
1074 self.config.set('zuul', 'tenant_config', self.tenant_config_file)
James E. Blair96c6bf82016-01-15 16:20:40 -08001075 git_path = os.path.join(
1076 os.path.dirname(
1077 os.path.join(FIXTURE_DIR, self.tenant_config_file)),
1078 'git')
1079 if os.path.exists(git_path):
1080 for reponame in os.listdir(git_path):
1081 self.copyDirToRepo(reponame,
1082 os.path.join(git_path, reponame))
1083
1084 def copyDirToRepo(self, project, source_path):
1085 repo_path = os.path.join(self.upstream_root, project)
1086 if not os.path.exists(repo_path):
1087 self.init_repo(project)
1088
1089 files = {}
1090 for (dirpath, dirnames, filenames) in os.walk(source_path):
1091 for filename in filenames:
1092 test_tree_filepath = os.path.join(dirpath, filename)
1093 common_path = os.path.commonprefix([test_tree_filepath,
1094 source_path])
1095 relative_filepath = test_tree_filepath[len(common_path) + 1:]
1096 with open(test_tree_filepath, 'r') as f:
1097 content = f.read()
1098 files[relative_filepath] = content
1099 self.addCommitToRepo(project, 'add content from fixture',
1100 files, branch='master')
James E. Blair83005782015-12-11 14:46:03 -08001101
1102 def setup_repos(self):
1103 """Subclasses can override to manipulate repos before tests"""
1104 pass
Clark Boylanb640e052014-04-03 16:41:46 -07001105
1106 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001107 # Make sure that git.Repo objects have been garbage collected.
1108 repos = []
1109 gc.collect()
1110 for obj in gc.get_objects():
1111 if isinstance(obj, git.Repo):
1112 repos.append(obj)
1113 self.assertEqual(len(repos), 0)
1114 self.assertEmptyQueues()
James E. Blair83005782015-12-11 14:46:03 -08001115 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001116 for tenant in self.sched.abide.tenants.values():
1117 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001118 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001119 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001120
1121 def shutdown(self):
1122 self.log.debug("Shutting down after tests")
1123 self.launcher.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001124 self.merge_client.stop()
Clark Boylanb640e052014-04-03 16:41:46 -07001125 self.sched.stop()
1126 self.sched.join()
1127 self.statsd.stop()
1128 self.statsd.join()
1129 self.webapp.stop()
1130 self.webapp.join()
1131 self.rpc.stop()
1132 self.rpc.join()
1133 self.gearman_server.shutdown()
1134 threads = threading.enumerate()
1135 if len(threads) > 1:
1136 self.log.error("More than one thread is running: %s" % threads)
Clark Boylanb640e052014-04-03 16:41:46 -07001137
1138 def init_repo(self, project):
1139 parts = project.split('/')
1140 path = os.path.join(self.upstream_root, *parts[:-1])
1141 if not os.path.exists(path):
1142 os.makedirs(path)
1143 path = os.path.join(self.upstream_root, project)
1144 repo = git.Repo.init(path)
1145
1146 repo.config_writer().set_value('user', 'email', 'user@example.com')
1147 repo.config_writer().set_value('user', 'name', 'User Name')
1148 repo.config_writer().write()
1149
1150 fn = os.path.join(path, 'README')
1151 f = open(fn, 'w')
1152 f.write("test\n")
1153 f.close()
1154 repo.index.add([fn])
1155 repo.index.commit('initial commit')
1156 master = repo.create_head('master')
1157 repo.create_tag('init')
1158
James E. Blair97d902e2014-08-21 13:25:56 -07001159 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001160 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001161 repo.git.clean('-x', '-f', '-d')
1162
1163 self.create_branch(project, 'mp')
1164
1165 def create_branch(self, project, branch):
1166 path = os.path.join(self.upstream_root, project)
1167 repo = git.Repo.init(path)
1168 fn = os.path.join(path, 'README')
1169
1170 branch_head = repo.create_head(branch)
1171 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001172 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001173 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001174 f.close()
1175 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001176 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001177
James E. Blair97d902e2014-08-21 13:25:56 -07001178 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001179 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001180 repo.git.clean('-x', '-f', '-d')
1181
1182 def ref_has_change(self, ref, change):
1183 path = os.path.join(self.git_root, change.project)
1184 repo = git.Repo(path)
Mike Heald8225f522014-11-21 09:52:33 +00001185 try:
1186 for commit in repo.iter_commits(ref):
1187 if commit.message.strip() == ('%s-1' % change.subject):
1188 return True
1189 except GitCommandError:
1190 pass
Clark Boylanb640e052014-04-03 16:41:46 -07001191 return False
1192
1193 def job_has_changes(self, *args):
1194 job = args[0]
1195 commits = args[1:]
1196 if isinstance(job, FakeBuild):
1197 parameters = job.parameters
1198 else:
1199 parameters = json.loads(job.arguments)
1200 project = parameters['ZUUL_PROJECT']
1201 path = os.path.join(self.git_root, project)
1202 repo = git.Repo(path)
1203 ref = parameters['ZUUL_REF']
1204 sha = parameters['ZUUL_COMMIT']
1205 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
1206 repo_shas = [c.hexsha for c in repo.iter_commits(ref)]
1207 commit_messages = ['%s-1' % commit.subject for commit in commits]
1208 self.log.debug("Checking if job %s has changes; commit_messages %s;"
1209 " repo_messages %s; sha %s" % (job, commit_messages,
1210 repo_messages, sha))
1211 for msg in commit_messages:
1212 if msg not in repo_messages:
1213 self.log.debug(" messages do not match")
1214 return False
1215 if repo_shas[0] != sha:
1216 self.log.debug(" sha does not match")
1217 return False
1218 self.log.debug(" OK")
1219 return True
1220
James E. Blairb8c16472015-05-05 14:55:26 -07001221 def orderedRelease(self):
1222 # Run one build at a time to ensure non-race order:
1223 while len(self.builds):
1224 self.release(self.builds[0])
1225 self.waitUntilSettled()
1226
Clark Boylanb640e052014-04-03 16:41:46 -07001227 def release(self, job):
1228 if isinstance(job, FakeBuild):
1229 job.release()
1230 else:
1231 job.waiting = False
1232 self.log.debug("Queued job %s released" % job.unique)
1233 self.gearman_server.wakeConnections()
1234
1235 def getParameter(self, job, name):
1236 if isinstance(job, FakeBuild):
1237 return job.parameters[name]
1238 else:
1239 parameters = json.loads(job.arguments)
1240 return parameters[name]
1241
1242 def resetGearmanServer(self):
1243 self.worker.setFunctions([])
1244 while True:
1245 done = True
1246 for connection in self.gearman_server.active_connections:
1247 if (connection.functions and
1248 connection.client_id not in ['Zuul RPC Listener',
1249 'Zuul Merger']):
1250 done = False
1251 if done:
1252 break
1253 time.sleep(0)
1254 self.gearman_server.functions = set()
1255 self.rpc.register()
Clark Boylanb640e052014-04-03 16:41:46 -07001256
1257 def haveAllBuildsReported(self):
1258 # See if Zuul is waiting on a meta job to complete
1259 if self.launcher.meta_jobs:
1260 return False
1261 # Find out if every build that the worker has completed has been
1262 # reported back to Zuul. If it hasn't then that means a Gearman
1263 # event is still in transit and the system is not stable.
James E. Blairf5dbd002015-12-23 15:26:17 -08001264 for job in self.ansible_server.job_history:
1265 zbuild = self.launcher.builds.get(job.unique)
Clark Boylanb640e052014-04-03 16:41:46 -07001266 if not zbuild:
1267 # It has already been reported
1268 continue
1269 # It hasn't been reported yet.
1270 return False
1271 # Make sure that none of the worker connections are in GRAB_WAIT
James E. Blairf5dbd002015-12-23 15:26:17 -08001272 for connection in self.ansible_server.worker.active_connections:
Clark Boylanb640e052014-04-03 16:41:46 -07001273 if connection.state == 'GRAB_WAIT':
1274 return False
1275 return True
1276
1277 def areAllBuildsWaiting(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001278 builds = self.launcher.builds.values()
1279 for build in builds:
1280 client_job = None
1281 for conn in self.launcher.gearman.active_connections:
1282 for j in conn.related_jobs.values():
1283 if j.unique == build.uuid:
1284 client_job = j
1285 break
1286 if not client_job:
1287 self.log.debug("%s is not known to the gearman client" %
1288 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001289 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001290 if not client_job.handle:
1291 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001292 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001293 server_job = self.gearman_server.jobs.get(client_job.handle)
1294 if not server_job:
1295 self.log.debug("%s is not known to the gearman server" %
1296 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001297 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001298 if not hasattr(server_job, 'waiting'):
1299 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001300 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001301 if server_job.waiting:
1302 continue
James E. Blairbbda4702016-03-09 15:19:56 -08001303 if build.number is None:
1304 self.log.debug("%s has not reported start" % build)
1305 return False
1306 if False: # worker_job.build.isWaiting():
1307 # TODOv3: when we grow the ability to have fake
1308 # ansible jobs wait, check for that here.
1309 continue
Clark Boylanb640e052014-04-03 16:41:46 -07001310 else:
James E. Blairbbda4702016-03-09 15:19:56 -08001311 self.log.debug("%s is running" % build)
James E. Blairf15139b2015-04-02 16:37:15 -07001312 return False
1313 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001314
Jan Hruban6b71aff2015-10-22 16:58:08 +02001315 def eventQueuesEmpty(self):
1316 for queue in self.event_queues:
1317 yield queue.empty()
1318
1319 def eventQueuesJoin(self):
1320 for queue in self.event_queues:
1321 queue.join()
1322
Clark Boylanb640e052014-04-03 16:41:46 -07001323 def waitUntilSettled(self):
1324 self.log.debug("Waiting until settled...")
1325 start = time.time()
1326 while True:
1327 if time.time() - start > 10:
1328 print 'queue status:',
Jan Hruban6b71aff2015-10-22 16:58:08 +02001329 print ' '.join(self.eventQueuesEmpty())
Clark Boylanb640e052014-04-03 16:41:46 -07001330 print self.areAllBuildsWaiting()
1331 raise Exception("Timeout waiting for Zuul to settle")
1332 # Make sure no new events show up while we're checking
Clark Boylanb640e052014-04-03 16:41:46 -07001333 # have all build states propogated to zuul?
1334 if self.haveAllBuildsReported():
1335 # Join ensures that the queue is empty _and_ events have been
1336 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001337 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001338 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001339 if (not self.merge_client.jobs and
Jan Hruban6b71aff2015-10-22 16:58:08 +02001340 all(self.eventQueuesEmpty()) and
Clark Boylanb640e052014-04-03 16:41:46 -07001341 self.haveAllBuildsReported() and
1342 self.areAllBuildsWaiting()):
1343 self.sched.run_handler_lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001344 self.log.debug("...settled.")
1345 return
1346 self.sched.run_handler_lock.release()
Clark Boylanb640e052014-04-03 16:41:46 -07001347 self.sched.wake_event.wait(0.1)
1348
1349 def countJobResults(self, jobs, result):
1350 jobs = filter(lambda x: x.result == result, jobs)
1351 return len(jobs)
1352
James E. Blair96c6bf82016-01-15 16:20:40 -08001353 def getJobFromHistory(self, name, project=None):
James E. Blairf5dbd002015-12-23 15:26:17 -08001354 history = self.ansible_server.job_history
Clark Boylanb640e052014-04-03 16:41:46 -07001355 for job in history:
James E. Blairf5dbd002015-12-23 15:26:17 -08001356 params = json.loads(job.arguments)
James E. Blair96c6bf82016-01-15 16:20:40 -08001357 if (params['job'] == name and
1358 (project is None or params['ZUUL_PROJECT'] == project)):
James E. Blairf5dbd002015-12-23 15:26:17 -08001359 result = json.loads(job.data[-1])
James E. Blairf5dbd002015-12-23 15:26:17 -08001360 ret = BuildHistory(job=job,
1361 name=params['job'],
1362 result=result['result'])
1363 return ret
Clark Boylanb640e052014-04-03 16:41:46 -07001364 raise Exception("Unable to find job %s in history" % name)
1365
1366 def assertEmptyQueues(self):
1367 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001368 for tenant in self.sched.abide.tenants.values():
1369 for pipeline in tenant.layout.pipelines.values():
1370 for queue in pipeline.queues:
1371 if len(queue.queue) != 0:
1372 print 'pipeline %s queue %s contents %s' % (
1373 pipeline.name, queue.name, queue.queue)
1374 self.assertEqual(len(queue.queue), 0,
1375 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001376
1377 def assertReportedStat(self, key, value=None, kind=None):
1378 start = time.time()
1379 while time.time() < (start + 5):
1380 for stat in self.statsd.stats:
1381 pprint.pprint(self.statsd.stats)
1382 k, v = stat.split(':')
1383 if key == k:
1384 if value is None and kind is None:
1385 return
1386 elif value:
1387 if value == v:
1388 return
1389 elif kind:
1390 if v.endswith('|' + kind):
1391 return
1392 time.sleep(0.1)
1393
1394 pprint.pprint(self.statsd.stats)
1395 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001396
1397 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001398 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1399
1400 def updateConfigLayout(self, path):
1401 root = os.path.join(self.test_root, "config")
1402 os.makedirs(root)
1403 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1404 f.write("""
1405tenants:
1406 - name: openstack
1407 include:
1408 - %s
1409 """ % os.path.abspath(path))
1410 f.close()
1411 self.config.set('zuul', 'tenant_config', f.name)
James E. Blair14abdf42015-12-09 16:11:53 -08001412
1413 def addCommitToRepo(self, project, message, files, branch='master'):
1414 path = os.path.join(self.upstream_root, project)
1415 repo = git.Repo(path)
1416 repo.head.reference = branch
1417 zuul.merger.merger.reset_repo_to_head(repo)
1418 for fn, content in files.items():
1419 fn = os.path.join(path, fn)
1420 with open(fn, 'w') as f:
1421 f.write(content)
1422 repo.index.add([fn])
1423 commit = repo.index.commit(message)
1424 repo.heads[branch].commit = commit
1425 repo.head.reference = branch
1426 repo.git.clean('-x', '-f', '-d')
1427 repo.heads[branch].checkout()