blob: 8efdfd13ba104520f4b487cc828ed47433f9504b [file] [log] [blame]
Clark Boylanb640e052014-04-03 16:41:46 -07001#!/usr/bin/env python
2
3# Copyright 2012 Hewlett-Packard Development Company, L.P.
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may
6# not use this file except in compliance with the License. You may obtain
7# a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations
15# under the License.
16
Christian Berendtffba5df2014-06-07 21:30:22 +020017from six.moves import configparser as ConfigParser
Clark Boylanb640e052014-04-03 16:41:46 -070018import gc
19import hashlib
20import json
21import logging
22import os
23import pprint
Christian Berendt12d4d722014-06-07 21:03:45 +020024from six.moves import queue as Queue
Clark Boylanb640e052014-04-03 16:41:46 -070025import random
26import re
27import select
28import shutil
29import socket
30import string
31import subprocess
32import swiftclient
James E. Blairf84026c2015-12-08 16:11:46 -080033import tempfile
Clark Boylanb640e052014-04-03 16:41:46 -070034import threading
35import time
36import urllib2
37
38import git
39import gear
40import fixtures
41import six.moves.urllib.parse as urlparse
42import statsd
43import testtools
Mike Heald8225f522014-11-21 09:52:33 +000044from git import GitCommandError
Clark Boylanb640e052014-04-03 16:41:46 -070045
Joshua Hesketh352264b2015-08-11 23:42:08 +100046import zuul.connection.gerrit
47import zuul.connection.smtp
Clark Boylanb640e052014-04-03 16:41:46 -070048import zuul.scheduler
49import zuul.webapp
50import zuul.rpclistener
51import zuul.launcher.gearman
52import zuul.lib.swift
James E. Blair83005782015-12-11 14:46:03 -080053import zuul.lib.connections
Clark Boylanb640e052014-04-03 16:41:46 -070054import zuul.merger.client
James E. Blair879dafb2015-07-17 14:04:49 -070055import zuul.merger.merger
56import zuul.merger.server
Clark Boylanb640e052014-04-03 16:41:46 -070057import zuul.reporter.gerrit
58import zuul.reporter.smtp
Joshua Hesketh850ccb62014-11-27 11:31:02 +110059import zuul.source.gerrit
Clark Boylanb640e052014-04-03 16:41:46 -070060import zuul.trigger.gerrit
61import zuul.trigger.timer
James E. Blairc494d542014-08-06 09:23:52 -070062import zuul.trigger.zuultrigger
Clark Boylanb640e052014-04-03 16:41:46 -070063
64FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
65 'fixtures')
James E. Blair97d902e2014-08-21 13:25:56 -070066USE_TEMPDIR = True
Clark Boylanb640e052014-04-03 16:41:46 -070067
68logging.basicConfig(level=logging.DEBUG,
69 format='%(asctime)s %(name)-32s '
70 '%(levelname)-8s %(message)s')
71
72
73def repack_repo(path):
74 cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
75 output = subprocess.Popen(cmd, close_fds=True,
76 stdout=subprocess.PIPE,
77 stderr=subprocess.PIPE)
78 out = output.communicate()
79 if output.returncode:
80 raise Exception("git repack returned %d" % output.returncode)
81 return out
82
83
84def random_sha1():
85 return hashlib.sha1(str(random.random())).hexdigest()
86
87
James E. Blaira190f3b2015-01-05 14:56:54 -080088def iterate_timeout(max_seconds, purpose):
89 start = time.time()
90 count = 0
91 while (time.time() < start + max_seconds):
92 count += 1
93 yield count
94 time.sleep(0)
95 raise Exception("Timeout waiting for %s" % purpose)
96
97
Clark Boylanb640e052014-04-03 16:41:46 -070098class ChangeReference(git.Reference):
99 _common_path_default = "refs/changes"
100 _points_to_commits_only = True
101
102
103class FakeChange(object):
104 categories = {'APRV': ('Approved', -1, 1),
105 'CRVW': ('Code-Review', -2, 2),
106 'VRFY': ('Verified', -2, 2)}
107
108 def __init__(self, gerrit, number, project, branch, subject,
109 status='NEW', upstream_root=None):
110 self.gerrit = gerrit
111 self.reported = 0
112 self.queried = 0
113 self.patchsets = []
114 self.number = number
115 self.project = project
116 self.branch = branch
117 self.subject = subject
118 self.latest_patchset = 0
119 self.depends_on_change = None
120 self.needed_by_changes = []
121 self.fail_merge = False
122 self.messages = []
123 self.data = {
124 'branch': branch,
125 'comments': [],
126 'commitMessage': subject,
127 'createdOn': time.time(),
128 'id': 'I' + random_sha1(),
129 'lastUpdated': time.time(),
130 'number': str(number),
131 'open': status == 'NEW',
132 'owner': {'email': 'user@example.com',
133 'name': 'User Name',
134 'username': 'username'},
135 'patchSets': self.patchsets,
136 'project': project,
137 'status': status,
138 'subject': subject,
139 'submitRecords': [],
140 'url': 'https://hostname/%s' % number}
141
142 self.upstream_root = upstream_root
143 self.addPatchset()
144 self.data['submitRecords'] = self.getSubmitRecords()
145 self.open = status == 'NEW'
146
147 def add_fake_change_to_repo(self, msg, fn, large):
148 path = os.path.join(self.upstream_root, self.project)
149 repo = git.Repo(path)
150 ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
151 self.latest_patchset),
152 'refs/tags/init')
153 repo.head.reference = ref
James E. Blair879dafb2015-07-17 14:04:49 -0700154 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700155 repo.git.clean('-x', '-f', '-d')
156
157 path = os.path.join(self.upstream_root, self.project)
158 if not large:
159 fn = os.path.join(path, fn)
160 f = open(fn, 'w')
161 f.write("test %s %s %s\n" %
162 (self.branch, self.number, self.latest_patchset))
163 f.close()
164 repo.index.add([fn])
165 else:
166 for fni in range(100):
167 fn = os.path.join(path, str(fni))
168 f = open(fn, 'w')
169 for ci in range(4096):
170 f.write(random.choice(string.printable))
171 f.close()
172 repo.index.add([fn])
173
174 r = repo.index.commit(msg)
175 repo.head.reference = 'master'
James E. Blair879dafb2015-07-17 14:04:49 -0700176 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -0700177 repo.git.clean('-x', '-f', '-d')
178 repo.heads['master'].checkout()
179 return r
180
181 def addPatchset(self, files=[], large=False):
182 self.latest_patchset += 1
183 if files:
184 fn = files[0]
185 else:
James E. Blair97d902e2014-08-21 13:25:56 -0700186 fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
Clark Boylanb640e052014-04-03 16:41:46 -0700187 msg = self.subject + '-' + str(self.latest_patchset)
188 c = self.add_fake_change_to_repo(msg, fn, large)
189 ps_files = [{'file': '/COMMIT_MSG',
190 'type': 'ADDED'},
191 {'file': 'README',
192 'type': 'MODIFIED'}]
193 for f in files:
194 ps_files.append({'file': f, 'type': 'ADDED'})
195 d = {'approvals': [],
196 'createdOn': time.time(),
197 'files': ps_files,
198 'number': str(self.latest_patchset),
199 'ref': 'refs/changes/1/%s/%s' % (self.number,
200 self.latest_patchset),
201 'revision': c.hexsha,
202 'uploader': {'email': 'user@example.com',
203 'name': 'User name',
204 'username': 'user'}}
205 self.data['currentPatchSet'] = d
206 self.patchsets.append(d)
207 self.data['submitRecords'] = self.getSubmitRecords()
208
209 def getPatchsetCreatedEvent(self, patchset):
210 event = {"type": "patchset-created",
211 "change": {"project": self.project,
212 "branch": self.branch,
213 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
214 "number": str(self.number),
215 "subject": self.subject,
216 "owner": {"name": "User Name"},
217 "url": "https://hostname/3"},
218 "patchSet": self.patchsets[patchset - 1],
219 "uploader": {"name": "User Name"}}
220 return event
221
222 def getChangeRestoredEvent(self):
223 event = {"type": "change-restored",
224 "change": {"project": self.project,
225 "branch": self.branch,
226 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
227 "number": str(self.number),
228 "subject": self.subject,
229 "owner": {"name": "User Name"},
230 "url": "https://hostname/3"},
231 "restorer": {"name": "User Name"},
Antoine Mussobd86a312014-01-08 14:51:33 +0100232 "patchSet": self.patchsets[-1],
233 "reason": ""}
234 return event
235
236 def getChangeAbandonedEvent(self):
237 event = {"type": "change-abandoned",
238 "change": {"project": self.project,
239 "branch": self.branch,
240 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
241 "number": str(self.number),
242 "subject": self.subject,
243 "owner": {"name": "User Name"},
244 "url": "https://hostname/3"},
245 "abandoner": {"name": "User Name"},
246 "patchSet": self.patchsets[-1],
Clark Boylanb640e052014-04-03 16:41:46 -0700247 "reason": ""}
248 return event
249
250 def getChangeCommentEvent(self, patchset):
251 event = {"type": "comment-added",
252 "change": {"project": self.project,
253 "branch": self.branch,
254 "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
255 "number": str(self.number),
256 "subject": self.subject,
257 "owner": {"name": "User Name"},
258 "url": "https://hostname/3"},
259 "patchSet": self.patchsets[patchset - 1],
260 "author": {"name": "User Name"},
261 "approvals": [{"type": "Code-Review",
262 "description": "Code-Review",
263 "value": "0"}],
264 "comment": "This is a comment"}
265 return event
266
Joshua Hesketh642824b2014-07-01 17:54:59 +1000267 def addApproval(self, category, value, username='reviewer_john',
268 granted_on=None, message=''):
Clark Boylanb640e052014-04-03 16:41:46 -0700269 if not granted_on:
270 granted_on = time.time()
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000271 approval = {
272 'description': self.categories[category][0],
273 'type': category,
274 'value': str(value),
275 'by': {
276 'username': username,
277 'email': username + '@example.com',
278 },
279 'grantedOn': int(granted_on)
280 }
Clark Boylanb640e052014-04-03 16:41:46 -0700281 for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
282 if x['by']['username'] == username and x['type'] == category:
283 del self.patchsets[-1]['approvals'][i]
284 self.patchsets[-1]['approvals'].append(approval)
285 event = {'approvals': [approval],
Joshua Hesketh642824b2014-07-01 17:54:59 +1000286 'author': {'email': 'author@example.com',
287 'name': 'Patchset Author',
288 'username': 'author_phil'},
Clark Boylanb640e052014-04-03 16:41:46 -0700289 'change': {'branch': self.branch,
290 'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
291 'number': str(self.number),
Joshua Hesketh642824b2014-07-01 17:54:59 +1000292 'owner': {'email': 'owner@example.com',
293 'name': 'Change Owner',
294 'username': 'owner_jane'},
Clark Boylanb640e052014-04-03 16:41:46 -0700295 'project': self.project,
296 'subject': self.subject,
297 'topic': 'master',
298 'url': 'https://hostname/459'},
Joshua Hesketh642824b2014-07-01 17:54:59 +1000299 'comment': message,
Clark Boylanb640e052014-04-03 16:41:46 -0700300 'patchSet': self.patchsets[-1],
301 'type': 'comment-added'}
302 self.data['submitRecords'] = self.getSubmitRecords()
303 return json.loads(json.dumps(event))
304
305 def getSubmitRecords(self):
306 status = {}
307 for cat in self.categories.keys():
308 status[cat] = 0
309
310 for a in self.patchsets[-1]['approvals']:
311 cur = status[a['type']]
312 cat_min, cat_max = self.categories[a['type']][1:]
313 new = int(a['value'])
314 if new == cat_min:
315 cur = new
316 elif abs(new) > abs(cur):
317 cur = new
318 status[a['type']] = cur
319
320 labels = []
321 ok = True
322 for typ, cat in self.categories.items():
323 cur = status[typ]
324 cat_min, cat_max = cat[1:]
325 if cur == cat_min:
326 value = 'REJECT'
327 ok = False
328 elif cur == cat_max:
329 value = 'OK'
330 else:
331 value = 'NEED'
332 ok = False
333 labels.append({'label': cat[0], 'status': value})
334 if ok:
335 return [{'status': 'OK'}]
336 return [{'status': 'NOT_READY',
337 'labels': labels}]
338
339 def setDependsOn(self, other, patchset):
340 self.depends_on_change = other
341 d = {'id': other.data['id'],
342 'number': other.data['number'],
343 'ref': other.patchsets[patchset - 1]['ref']
344 }
345 self.data['dependsOn'] = [d]
346
347 other.needed_by_changes.append(self)
348 needed = other.data.get('neededBy', [])
349 d = {'id': self.data['id'],
350 'number': self.data['number'],
351 'ref': self.patchsets[patchset - 1]['ref'],
352 'revision': self.patchsets[patchset - 1]['revision']
353 }
354 needed.append(d)
355 other.data['neededBy'] = needed
356
357 def query(self):
358 self.queried += 1
359 d = self.data.get('dependsOn')
360 if d:
361 d = d[0]
362 if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
363 d['isCurrentPatchSet'] = True
364 else:
365 d['isCurrentPatchSet'] = False
366 return json.loads(json.dumps(self.data))
367
368 def setMerged(self):
369 if (self.depends_on_change and
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000370 self.depends_on_change.data['status'] != 'MERGED'):
Clark Boylanb640e052014-04-03 16:41:46 -0700371 return
372 if self.fail_merge:
373 return
374 self.data['status'] = 'MERGED'
375 self.open = False
376
377 path = os.path.join(self.upstream_root, self.project)
378 repo = git.Repo(path)
379 repo.heads[self.branch].commit = \
380 repo.commit(self.patchsets[-1]['revision'])
381
382 def setReported(self):
383 self.reported += 1
384
385
Joshua Hesketh352264b2015-08-11 23:42:08 +1000386class FakeGerritConnection(zuul.connection.gerrit.GerritConnection):
387 log = logging.getLogger("zuul.test.FakeGerritConnection")
James E. Blair96698e22015-04-02 07:48:21 -0700388
Joshua Hesketh352264b2015-08-11 23:42:08 +1000389 def __init__(self, connection_name, connection_config,
Jan Hruban6b71aff2015-10-22 16:58:08 +0200390 changes_db=None, queues_db=None, upstream_root=None):
Joshua Hesketh352264b2015-08-11 23:42:08 +1000391 super(FakeGerritConnection, self).__init__(connection_name,
392 connection_config)
393
394 self.event_queue = queues_db
Clark Boylanb640e052014-04-03 16:41:46 -0700395 self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
396 self.change_number = 0
Joshua Hesketh352264b2015-08-11 23:42:08 +1000397 self.changes = changes_db
James E. Blairf8ff9932014-08-15 15:24:24 -0700398 self.queries = []
Jan Hruban6b71aff2015-10-22 16:58:08 +0200399 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700400
401 def addFakeChange(self, project, branch, subject, status='NEW'):
402 self.change_number += 1
403 c = FakeChange(self, self.change_number, project, branch, subject,
404 upstream_root=self.upstream_root,
405 status=status)
406 self.changes[self.change_number] = c
407 return c
408
Clark Boylanb640e052014-04-03 16:41:46 -0700409 def review(self, project, changeid, message, action):
410 number, ps = changeid.split(',')
411 change = self.changes[int(number)]
Joshua Hesketh642824b2014-07-01 17:54:59 +1000412
413 # Add the approval back onto the change (ie simulate what gerrit would
414 # do).
415 # Usually when zuul leaves a review it'll create a feedback loop where
416 # zuul's review enters another gerrit event (which is then picked up by
417 # zuul). However, we can't mimic this behaviour (by adding this
418 # approval event into the queue) as it stops jobs from checking what
419 # happens before this event is triggered. If a job needs to see what
420 # happens they can add their own verified event into the queue.
421 # Nevertheless, we can update change with the new review in gerrit.
422
423 for cat in ['CRVW', 'VRFY', 'APRV']:
424 if cat in action:
Joshua Hesketh352264b2015-08-11 23:42:08 +1000425 change.addApproval(cat, action[cat], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000426
427 if 'label' in action:
428 parts = action['label'].split('=')
Joshua Hesketh352264b2015-08-11 23:42:08 +1000429 change.addApproval(parts[0], parts[2], username=self.user)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000430
Clark Boylanb640e052014-04-03 16:41:46 -0700431 change.messages.append(message)
Joshua Hesketh642824b2014-07-01 17:54:59 +1000432
Clark Boylanb640e052014-04-03 16:41:46 -0700433 if 'submit' in action:
434 change.setMerged()
435 if message:
436 change.setReported()
437
438 def query(self, number):
439 change = self.changes.get(int(number))
440 if change:
441 return change.query()
442 return {}
443
James E. Blairc494d542014-08-06 09:23:52 -0700444 def simpleQuery(self, query):
James E. Blair96698e22015-04-02 07:48:21 -0700445 self.log.debug("simpleQuery: %s" % query)
James E. Blairf8ff9932014-08-15 15:24:24 -0700446 self.queries.append(query)
James E. Blair5ee24252014-12-30 10:12:29 -0800447 if query.startswith('change:'):
448 # Query a specific changeid
449 changeid = query[len('change:'):]
450 l = [change.query() for change in self.changes.values()
451 if change.data['id'] == changeid]
James E. Blair96698e22015-04-02 07:48:21 -0700452 elif query.startswith('message:'):
453 # Query the content of a commit message
454 msg = query[len('message:'):].strip()
455 l = [change.query() for change in self.changes.values()
456 if msg in change.data['commitMessage']]
James E. Blair5ee24252014-12-30 10:12:29 -0800457 else:
458 # Query all open changes
459 l = [change.query() for change in self.changes.values()]
James E. Blairf8ff9932014-08-15 15:24:24 -0700460 return l
James E. Blairc494d542014-08-06 09:23:52 -0700461
Joshua Hesketh352264b2015-08-11 23:42:08 +1000462 def _start_watcher_thread(self, *args, **kw):
Clark Boylanb640e052014-04-03 16:41:46 -0700463 pass
464
Joshua Hesketh352264b2015-08-11 23:42:08 +1000465 def getGitUrl(self, project):
466 return os.path.join(self.upstream_root, project.name)
467
Clark Boylanb640e052014-04-03 16:41:46 -0700468
469class BuildHistory(object):
470 def __init__(self, **kw):
471 self.__dict__.update(kw)
472
473 def __repr__(self):
474 return ("<Completed build, result: %s name: %s #%s changes: %s>" %
475 (self.result, self.name, self.number, self.changes))
476
477
478class FakeURLOpener(object):
Jan Hruban6b71aff2015-10-22 16:58:08 +0200479 def __init__(self, upstream_root, url):
Clark Boylanb640e052014-04-03 16:41:46 -0700480 self.upstream_root = upstream_root
Clark Boylanb640e052014-04-03 16:41:46 -0700481 self.url = url
482
483 def read(self):
484 res = urlparse.urlparse(self.url)
485 path = res.path
486 project = '/'.join(path.split('/')[2:-2])
487 ret = '001e# service=git-upload-pack\n'
488 ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
489 'multi_ack thin-pack side-band side-band-64k ofs-delta '
490 'shallow no-progress include-tag multi_ack_detailed no-done\n')
491 path = os.path.join(self.upstream_root, project)
492 repo = git.Repo(path)
493 for ref in repo.refs:
494 r = ref.object.hexsha + ' ' + ref.path + '\n'
495 ret += '%04x%s' % (len(r) + 4, r)
496 ret += '0000'
497 return ret
498
499
Clark Boylanb640e052014-04-03 16:41:46 -0700500class FakeStatsd(threading.Thread):
501 def __init__(self):
502 threading.Thread.__init__(self)
503 self.daemon = True
504 self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
505 self.sock.bind(('', 0))
506 self.port = self.sock.getsockname()[1]
507 self.wake_read, self.wake_write = os.pipe()
508 self.stats = []
509
510 def run(self):
511 while True:
512 poll = select.poll()
513 poll.register(self.sock, select.POLLIN)
514 poll.register(self.wake_read, select.POLLIN)
515 ret = poll.poll()
516 for (fd, event) in ret:
517 if fd == self.sock.fileno():
518 data = self.sock.recvfrom(1024)
519 if not data:
520 return
521 self.stats.append(data[0])
522 if fd == self.wake_read:
523 return
524
525 def stop(self):
526 os.write(self.wake_write, '1\n')
527
528
529class FakeBuild(threading.Thread):
530 log = logging.getLogger("zuul.test")
531
532 def __init__(self, worker, job, number, node):
533 threading.Thread.__init__(self)
534 self.daemon = True
535 self.worker = worker
536 self.job = job
537 self.name = job.name.split(':')[1]
538 self.number = number
539 self.node = node
540 self.parameters = json.loads(job.arguments)
541 self.unique = self.parameters['ZUUL_UUID']
542 self.wait_condition = threading.Condition()
543 self.waiting = False
544 self.aborted = False
545 self.created = time.time()
546 self.description = ''
547 self.run_error = False
548
549 def release(self):
550 self.wait_condition.acquire()
551 self.wait_condition.notify()
552 self.waiting = False
553 self.log.debug("Build %s released" % self.unique)
554 self.wait_condition.release()
555
556 def isWaiting(self):
557 self.wait_condition.acquire()
558 if self.waiting:
559 ret = True
560 else:
561 ret = False
562 self.wait_condition.release()
563 return ret
564
565 def _wait(self):
566 self.wait_condition.acquire()
567 self.waiting = True
568 self.log.debug("Build %s waiting" % self.unique)
569 self.wait_condition.wait()
570 self.wait_condition.release()
571
572 def run(self):
573 data = {
574 'url': 'https://server/job/%s/%s/' % (self.name, self.number),
575 'name': self.name,
576 'number': self.number,
577 'manager': self.worker.worker_id,
578 'worker_name': 'My Worker',
579 'worker_hostname': 'localhost',
580 'worker_ips': ['127.0.0.1', '192.168.1.1'],
581 'worker_fqdn': 'zuul.example.org',
582 'worker_program': 'FakeBuilder',
583 'worker_version': 'v1.1',
584 'worker_extra': {'something': 'else'}
585 }
586
587 self.log.debug('Running build %s' % self.unique)
588
589 self.job.sendWorkData(json.dumps(data))
590 self.log.debug('Sent WorkData packet with %s' % json.dumps(data))
591 self.job.sendWorkStatus(0, 100)
592
593 if self.worker.hold_jobs_in_build:
594 self.log.debug('Holding build %s' % self.unique)
595 self._wait()
596 self.log.debug("Build %s continuing" % self.unique)
597
598 self.worker.lock.acquire()
599
600 result = 'SUCCESS'
601 if (('ZUUL_REF' in self.parameters) and
602 self.worker.shouldFailTest(self.name,
603 self.parameters['ZUUL_REF'])):
604 result = 'FAILURE'
605 if self.aborted:
606 result = 'ABORTED'
607
608 if self.run_error:
609 work_fail = True
610 result = 'RUN_ERROR'
611 else:
612 data['result'] = result
Timothy Chavezb2332082015-08-07 20:08:04 -0500613 data['node_labels'] = ['bare-necessities']
614 data['node_name'] = 'foo'
Clark Boylanb640e052014-04-03 16:41:46 -0700615 work_fail = False
616
617 changes = None
618 if 'ZUUL_CHANGE_IDS' in self.parameters:
619 changes = self.parameters['ZUUL_CHANGE_IDS']
620
621 self.worker.build_history.append(
622 BuildHistory(name=self.name, number=self.number,
623 result=result, changes=changes, node=self.node,
624 uuid=self.unique, description=self.description,
625 pipeline=self.parameters['ZUUL_PIPELINE'])
626 )
627
628 self.job.sendWorkData(json.dumps(data))
629 if work_fail:
630 self.job.sendWorkFail()
631 else:
632 self.job.sendWorkComplete(json.dumps(data))
633 del self.worker.gearman_jobs[self.job.unique]
634 self.worker.running_builds.remove(self)
635 self.worker.lock.release()
636
637
638class FakeWorker(gear.Worker):
639 def __init__(self, worker_id, test):
640 super(FakeWorker, self).__init__(worker_id)
641 self.gearman_jobs = {}
642 self.build_history = []
643 self.running_builds = []
644 self.build_counter = 0
645 self.fail_tests = {}
646 self.test = test
647
648 self.hold_jobs_in_build = False
649 self.lock = threading.Lock()
650 self.__work_thread = threading.Thread(target=self.work)
651 self.__work_thread.daemon = True
652 self.__work_thread.start()
653
654 def handleJob(self, job):
655 parts = job.name.split(":")
656 cmd = parts[0]
657 name = parts[1]
658 if len(parts) > 2:
659 node = parts[2]
660 else:
661 node = None
662 if cmd == 'build':
663 self.handleBuild(job, name, node)
664 elif cmd == 'stop':
665 self.handleStop(job, name)
666 elif cmd == 'set_description':
667 self.handleSetDescription(job, name)
668
669 def handleBuild(self, job, name, node):
670 build = FakeBuild(self, job, self.build_counter, node)
671 job.build = build
672 self.gearman_jobs[job.unique] = job
673 self.build_counter += 1
674
675 self.running_builds.append(build)
676 build.start()
677
678 def handleStop(self, job, name):
679 self.log.debug("handle stop")
680 parameters = json.loads(job.arguments)
681 name = parameters['name']
682 number = parameters['number']
683 for build in self.running_builds:
684 if build.name == name and build.number == number:
685 build.aborted = True
686 build.release()
687 job.sendWorkComplete()
688 return
689 job.sendWorkFail()
690
691 def handleSetDescription(self, job, name):
692 self.log.debug("handle set description")
693 parameters = json.loads(job.arguments)
694 name = parameters['name']
695 number = parameters['number']
696 descr = parameters['html_description']
697 for build in self.running_builds:
698 if build.name == name and build.number == number:
699 build.description = descr
700 job.sendWorkComplete()
701 return
702 for build in self.build_history:
703 if build.name == name and build.number == number:
704 build.description = descr
705 job.sendWorkComplete()
706 return
707 job.sendWorkFail()
708
709 def work(self):
710 while self.running:
711 try:
712 job = self.getJob()
713 except gear.InterruptedError:
714 continue
715 try:
716 self.handleJob(job)
717 except:
718 self.log.exception("Worker exception:")
719
720 def addFailTest(self, name, change):
721 l = self.fail_tests.get(name, [])
722 l.append(change)
723 self.fail_tests[name] = l
724
725 def shouldFailTest(self, name, ref):
726 l = self.fail_tests.get(name, [])
727 for change in l:
728 if self.test.ref_has_change(ref, change):
729 return True
730 return False
731
732 def release(self, regex=None):
733 builds = self.running_builds[:]
734 self.log.debug("releasing build %s (%s)" % (regex,
735 len(self.running_builds)))
736 for build in builds:
737 if not regex or re.match(regex, build.name):
738 self.log.debug("releasing build %s" %
739 (build.parameters['ZUUL_UUID']))
740 build.release()
741 else:
742 self.log.debug("not releasing build %s" %
743 (build.parameters['ZUUL_UUID']))
744 self.log.debug("done releasing builds %s (%s)" %
745 (regex, len(self.running_builds)))
746
747
748class FakeGearmanServer(gear.Server):
749 def __init__(self):
750 self.hold_jobs_in_queue = False
751 super(FakeGearmanServer, self).__init__(0)
752
753 def getJobForConnection(self, connection, peek=False):
754 for queue in [self.high_queue, self.normal_queue, self.low_queue]:
755 for job in queue:
756 if not hasattr(job, 'waiting'):
757 if job.name.startswith('build:'):
758 job.waiting = self.hold_jobs_in_queue
759 else:
760 job.waiting = False
761 if job.waiting:
762 continue
763 if job.name in connection.functions:
764 if not peek:
765 queue.remove(job)
766 connection.related_jobs[job.handle] = job
767 job.worker_connection = connection
768 job.running = True
769 return job
770 return None
771
772 def release(self, regex=None):
773 released = False
774 qlen = (len(self.high_queue) + len(self.normal_queue) +
775 len(self.low_queue))
776 self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
777 for job in self.getQueue():
778 cmd, name = job.name.split(':')
779 if cmd != 'build':
780 continue
781 if not regex or re.match(regex, name):
782 self.log.debug("releasing queued job %s" %
783 job.unique)
784 job.waiting = False
785 released = True
786 else:
787 self.log.debug("not releasing queued job %s" %
788 job.unique)
789 if released:
790 self.wakeConnections()
791 qlen = (len(self.high_queue) + len(self.normal_queue) +
792 len(self.low_queue))
793 self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
794
795
796class FakeSMTP(object):
797 log = logging.getLogger('zuul.FakeSMTP')
798
799 def __init__(self, messages, server, port):
800 self.server = server
801 self.port = port
802 self.messages = messages
803
804 def sendmail(self, from_email, to_email, msg):
805 self.log.info("Sending email from %s, to %s, with msg %s" % (
806 from_email, to_email, msg))
807
808 headers = msg.split('\n\n', 1)[0]
809 body = msg.split('\n\n', 1)[1]
810
811 self.messages.append(dict(
812 from_email=from_email,
813 to_email=to_email,
814 msg=msg,
815 headers=headers,
816 body=body,
817 ))
818
819 return True
820
821 def quit(self):
822 return True
823
824
825class FakeSwiftClientConnection(swiftclient.client.Connection):
826 def post_account(self, headers):
827 # Do nothing
828 pass
829
830 def get_auth(self):
831 # Returns endpoint and (unused) auth token
832 endpoint = os.path.join('https://storage.example.org', 'V1',
833 'AUTH_account')
834 return endpoint, ''
835
836
Maru Newby3fe5f852015-01-13 04:22:14 +0000837class BaseTestCase(testtools.TestCase):
Clark Boylanb640e052014-04-03 16:41:46 -0700838 log = logging.getLogger("zuul.test")
839
840 def setUp(self):
Maru Newby3fe5f852015-01-13 04:22:14 +0000841 super(BaseTestCase, self).setUp()
Clark Boylanb640e052014-04-03 16:41:46 -0700842 test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
843 try:
844 test_timeout = int(test_timeout)
845 except ValueError:
846 # If timeout value is invalid do not set a timeout.
847 test_timeout = 0
848 if test_timeout > 0:
849 self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
850
851 if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
852 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
853 stdout = self.useFixture(fixtures.StringStream('stdout')).stream
854 self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
855 if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
856 os.environ.get('OS_STDERR_CAPTURE') == '1'):
857 stderr = self.useFixture(fixtures.StringStream('stderr')).stream
858 self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
859 if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
860 os.environ.get('OS_LOG_CAPTURE') == '1'):
861 self.useFixture(fixtures.FakeLogger(
862 level=logging.DEBUG,
863 format='%(asctime)s %(name)-32s '
864 '%(levelname)-8s %(message)s'))
Maru Newby3fe5f852015-01-13 04:22:14 +0000865
866
867class ZuulTestCase(BaseTestCase):
James E. Blair83005782015-12-11 14:46:03 -0800868 config_file = 'zuul.conf'
Maru Newby3fe5f852015-01-13 04:22:14 +0000869
870 def setUp(self):
871 super(ZuulTestCase, self).setUp()
James E. Blair97d902e2014-08-21 13:25:56 -0700872 if USE_TEMPDIR:
873 tmp_root = self.useFixture(fixtures.TempDir(
Joshua Hesketh29d99b72014-08-19 16:27:42 +1000874 rootdir=os.environ.get("ZUUL_TEST_ROOT"))
875 ).path
James E. Blair97d902e2014-08-21 13:25:56 -0700876 else:
877 tmp_root = os.environ.get("ZUUL_TEST_ROOT")
Clark Boylanb640e052014-04-03 16:41:46 -0700878 self.test_root = os.path.join(tmp_root, "zuul-test")
879 self.upstream_root = os.path.join(self.test_root, "upstream")
880 self.git_root = os.path.join(self.test_root, "git")
881
882 if os.path.exists(self.test_root):
883 shutil.rmtree(self.test_root)
884 os.makedirs(self.test_root)
885 os.makedirs(self.upstream_root)
Clark Boylanb640e052014-04-03 16:41:46 -0700886
887 # Make per test copy of Configuration.
888 self.setup_config()
James E. Blair59fdbac2015-12-07 17:08:06 -0800889 self.config.set('zuul', 'tenant_config',
Joshua Heskethacccffc2015-03-31 23:38:17 +1100890 os.path.join(FIXTURE_DIR,
James E. Blair59fdbac2015-12-07 17:08:06 -0800891 self.config.get('zuul', 'tenant_config')))
Clark Boylanb640e052014-04-03 16:41:46 -0700892 self.config.set('merger', 'git_dir', self.git_root)
893
894 # For each project in config:
895 self.init_repo("org/project")
896 self.init_repo("org/project1")
897 self.init_repo("org/project2")
898 self.init_repo("org/project3")
James E. Blair97d902e2014-08-21 13:25:56 -0700899 self.init_repo("org/project4")
James E. Blairbce35e12014-08-21 14:31:17 -0700900 self.init_repo("org/project5")
901 self.init_repo("org/project6")
Clark Boylanb640e052014-04-03 16:41:46 -0700902 self.init_repo("org/one-job-project")
903 self.init_repo("org/nonvoting-project")
904 self.init_repo("org/templated-project")
905 self.init_repo("org/layered-project")
906 self.init_repo("org/node-project")
907 self.init_repo("org/conflict-project")
908 self.init_repo("org/noop-project")
909 self.init_repo("org/experimental-project")
Evgeny Antyshevd6e546c2015-06-11 15:13:57 +0000910 self.init_repo("org/no-jobs-project")
Clark Boylanb640e052014-04-03 16:41:46 -0700911
James E. Blair83005782015-12-11 14:46:03 -0800912 self.setup_repos()
913
Clark Boylanb640e052014-04-03 16:41:46 -0700914 self.statsd = FakeStatsd()
Ian Wienandff977bf2015-09-30 15:38:47 +1000915 # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
916 # see: https://github.com/jsocol/pystatsd/issues/61
917 os.environ['STATSD_HOST'] = '127.0.0.1'
Clark Boylanb640e052014-04-03 16:41:46 -0700918 os.environ['STATSD_PORT'] = str(self.statsd.port)
919 self.statsd.start()
920 # the statsd client object is configured in the statsd module import
921 reload(statsd)
922 reload(zuul.scheduler)
923
924 self.gearman_server = FakeGearmanServer()
925
926 self.config.set('gearman', 'port', str(self.gearman_server.port))
927
928 self.worker = FakeWorker('fake_worker', self)
929 self.worker.addServer('127.0.0.1', self.gearman_server.port)
930 self.gearman_server.worker = self.worker
931
Joshua Hesketh352264b2015-08-11 23:42:08 +1000932 zuul.source.gerrit.GerritSource.replication_timeout = 1.5
933 zuul.source.gerrit.GerritSource.replication_retry_interval = 0.5
934 zuul.connection.gerrit.GerritEventConnector.delay = 0.0
Clark Boylanb640e052014-04-03 16:41:46 -0700935
Joshua Hesketh352264b2015-08-11 23:42:08 +1000936 self.sched = zuul.scheduler.Scheduler(self.config)
Clark Boylanb640e052014-04-03 16:41:46 -0700937
938 self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
939 FakeSwiftClientConnection))
940 self.swift = zuul.lib.swift.Swift(self.config)
941
Jan Hruban6b71aff2015-10-22 16:58:08 +0200942 self.event_queues = [
943 self.sched.result_event_queue,
944 self.sched.trigger_event_queue
945 ]
946
James E. Blair83005782015-12-11 14:46:03 -0800947 self.configure_connections(self.sched)
Joshua Hesketh352264b2015-08-11 23:42:08 +1000948 self.sched.registerConnections(self.connections)
Joshua Hesketh352264b2015-08-11 23:42:08 +1000949
Clark Boylanb640e052014-04-03 16:41:46 -0700950 def URLOpenerFactory(*args, **kw):
951 if isinstance(args[0], urllib2.Request):
952 return old_urlopen(*args, **kw)
Clark Boylanb640e052014-04-03 16:41:46 -0700953 return FakeURLOpener(self.upstream_root, *args, **kw)
954
955 old_urlopen = urllib2.urlopen
956 urllib2.urlopen = URLOpenerFactory
957
Joshua Hesketh352264b2015-08-11 23:42:08 +1000958 self.merge_server = zuul.merger.server.MergeServer(self.config,
959 self.connections)
960 self.merge_server.start()
Clark Boylanb640e052014-04-03 16:41:46 -0700961
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100962 self.launcher = zuul.launcher.gearman.Gearman(self.config, self.sched,
963 self.swift)
964 self.merge_client = zuul.merger.client.MergeClient(
965 self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -0700966
967 self.sched.setLauncher(self.launcher)
968 self.sched.setMerger(self.merge_client)
Clark Boylanb640e052014-04-03 16:41:46 -0700969
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100970 self.webapp = zuul.webapp.WebApp(self.sched, port=0)
971 self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
Clark Boylanb640e052014-04-03 16:41:46 -0700972
973 self.sched.start()
974 self.sched.reconfigure(self.config)
975 self.sched.resume()
976 self.webapp.start()
977 self.rpc.start()
978 self.launcher.gearman.waitForServer()
979 self.registerJobs()
980 self.builds = self.worker.running_builds
981 self.history = self.worker.build_history
982
983 self.addCleanup(self.assertFinalState)
984 self.addCleanup(self.shutdown)
985
James E. Blair83005782015-12-11 14:46:03 -0800986 def configure_connections(self, sched):
Joshua Hesketh352264b2015-08-11 23:42:08 +1000987 # Register connections from the config
988 self.smtp_messages = []
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100989
Joshua Hesketh352264b2015-08-11 23:42:08 +1000990 def FakeSMTPFactory(*args, **kw):
991 args = [self.smtp_messages] + list(args)
992 return FakeSMTP(*args, **kw)
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100993
Joshua Hesketh352264b2015-08-11 23:42:08 +1000994 self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
Joshua Hesketh850ccb62014-11-27 11:31:02 +1100995
Joshua Hesketh352264b2015-08-11 23:42:08 +1000996 # Set a changes database so multiple FakeGerrit's can report back to
997 # a virtual canonical database given by the configured hostname
998 self.gerrit_changes_dbs = {}
999 self.gerrit_queues_dbs = {}
James E. Blair83005782015-12-11 14:46:03 -08001000 self.connections = zuul.lib.connections.ConnectionRegistry(sched)
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001001
Joshua Hesketh352264b2015-08-11 23:42:08 +10001002 for section_name in self.config.sections():
1003 con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
1004 section_name, re.I)
1005 if not con_match:
1006 continue
1007 con_name = con_match.group(2)
1008 con_config = dict(self.config.items(section_name))
1009
1010 if 'driver' not in con_config:
1011 raise Exception("No driver specified for connection %s."
1012 % con_name)
1013
1014 con_driver = con_config['driver']
1015
1016 # TODO(jhesketh): load the required class automatically
1017 if con_driver == 'gerrit':
Joshua Heskethacccffc2015-03-31 23:38:17 +11001018 if con_config['server'] not in self.gerrit_changes_dbs.keys():
1019 self.gerrit_changes_dbs[con_config['server']] = {}
1020 if con_config['server'] not in self.gerrit_queues_dbs.keys():
1021 self.gerrit_queues_dbs[con_config['server']] = \
1022 Queue.Queue()
1023 self.event_queues.append(
1024 self.gerrit_queues_dbs[con_config['server']])
James E. Blair83005782015-12-11 14:46:03 -08001025 self.connections.connections[con_name] = FakeGerritConnection(
Joshua Hesketh352264b2015-08-11 23:42:08 +10001026 con_name, con_config,
Joshua Heskethacccffc2015-03-31 23:38:17 +11001027 changes_db=self.gerrit_changes_dbs[con_config['server']],
1028 queues_db=self.gerrit_queues_dbs[con_config['server']],
Jan Hruban6b71aff2015-10-22 16:58:08 +02001029 upstream_root=self.upstream_root
Joshua Hesketh352264b2015-08-11 23:42:08 +10001030 )
James E. Blair83005782015-12-11 14:46:03 -08001031 setattr(self, 'fake_' + con_name,
1032 self.connections.connections[con_name])
Joshua Hesketh352264b2015-08-11 23:42:08 +10001033 elif con_driver == 'smtp':
James E. Blair83005782015-12-11 14:46:03 -08001034 self.connections.connections[con_name] = \
Joshua Hesketh352264b2015-08-11 23:42:08 +10001035 zuul.connection.smtp.SMTPConnection(con_name, con_config)
1036 else:
1037 raise Exception("Unknown driver, %s, for connection %s"
1038 % (con_config['driver'], con_name))
1039
1040 # If the [gerrit] or [smtp] sections still exist, load them in as a
1041 # connection named 'gerrit' or 'smtp' respectfully
1042
1043 if 'gerrit' in self.config.sections():
1044 self.gerrit_changes_dbs['gerrit'] = {}
1045 self.gerrit_queues_dbs['gerrit'] = Queue.Queue()
Jan Hruban6b71aff2015-10-22 16:58:08 +02001046 self.event_queues.append(self.gerrit_queues_dbs['gerrit'])
James E. Blair83005782015-12-11 14:46:03 -08001047 self.connections.connections['gerrit'] = FakeGerritConnection(
Joshua Hesketh352264b2015-08-11 23:42:08 +10001048 '_legacy_gerrit', dict(self.config.items('gerrit')),
1049 changes_db=self.gerrit_changes_dbs['gerrit'],
1050 queues_db=self.gerrit_queues_dbs['gerrit'])
1051
1052 if 'smtp' in self.config.sections():
James E. Blair83005782015-12-11 14:46:03 -08001053 self.connections.connections['smtp'] = \
Joshua Hesketh352264b2015-08-11 23:42:08 +10001054 zuul.connection.smtp.SMTPConnection(
1055 '_legacy_smtp', dict(self.config.items('smtp')))
Joshua Hesketh850ccb62014-11-27 11:31:02 +11001056
James E. Blair83005782015-12-11 14:46:03 -08001057 def setup_config(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001058 """Per test config object. Override to set different config."""
1059 self.config = ConfigParser.ConfigParser()
James E. Blair83005782015-12-11 14:46:03 -08001060 self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
1061
1062 def setup_repos(self):
1063 """Subclasses can override to manipulate repos before tests"""
1064 pass
Clark Boylanb640e052014-04-03 16:41:46 -07001065
1066 def assertFinalState(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001067 # Make sure that git.Repo objects have been garbage collected.
1068 repos = []
1069 gc.collect()
1070 for obj in gc.get_objects():
1071 if isinstance(obj, git.Repo):
1072 repos.append(obj)
1073 self.assertEqual(len(repos), 0)
1074 self.assertEmptyQueues()
James E. Blair83005782015-12-11 14:46:03 -08001075 ipm = zuul.manager.independent.IndependentPipelineManager
James E. Blair59fdbac2015-12-07 17:08:06 -08001076 for tenant in self.sched.abide.tenants.values():
1077 for pipeline in tenant.layout.pipelines.values():
James E. Blair83005782015-12-11 14:46:03 -08001078 if isinstance(pipeline.manager, ipm):
James E. Blair59fdbac2015-12-07 17:08:06 -08001079 self.assertEqual(len(pipeline.queues), 0)
Clark Boylanb640e052014-04-03 16:41:46 -07001080
1081 def shutdown(self):
1082 self.log.debug("Shutting down after tests")
1083 self.launcher.stop()
1084 self.merge_server.stop()
1085 self.merge_server.join()
1086 self.merge_client.stop()
1087 self.worker.shutdown()
Clark Boylanb640e052014-04-03 16:41:46 -07001088 self.sched.stop()
1089 self.sched.join()
1090 self.statsd.stop()
1091 self.statsd.join()
1092 self.webapp.stop()
1093 self.webapp.join()
1094 self.rpc.stop()
1095 self.rpc.join()
1096 self.gearman_server.shutdown()
1097 threads = threading.enumerate()
1098 if len(threads) > 1:
1099 self.log.error("More than one thread is running: %s" % threads)
Clark Boylanb640e052014-04-03 16:41:46 -07001100
1101 def init_repo(self, project):
1102 parts = project.split('/')
1103 path = os.path.join(self.upstream_root, *parts[:-1])
1104 if not os.path.exists(path):
1105 os.makedirs(path)
1106 path = os.path.join(self.upstream_root, project)
1107 repo = git.Repo.init(path)
1108
1109 repo.config_writer().set_value('user', 'email', 'user@example.com')
1110 repo.config_writer().set_value('user', 'name', 'User Name')
1111 repo.config_writer().write()
1112
1113 fn = os.path.join(path, 'README')
1114 f = open(fn, 'w')
1115 f.write("test\n")
1116 f.close()
1117 repo.index.add([fn])
1118 repo.index.commit('initial commit')
1119 master = repo.create_head('master')
1120 repo.create_tag('init')
1121
James E. Blair97d902e2014-08-21 13:25:56 -07001122 repo.head.reference = master
James E. Blair879dafb2015-07-17 14:04:49 -07001123 zuul.merger.merger.reset_repo_to_head(repo)
James E. Blair97d902e2014-08-21 13:25:56 -07001124 repo.git.clean('-x', '-f', '-d')
1125
1126 self.create_branch(project, 'mp')
1127
1128 def create_branch(self, project, branch):
1129 path = os.path.join(self.upstream_root, project)
1130 repo = git.Repo.init(path)
1131 fn = os.path.join(path, 'README')
1132
1133 branch_head = repo.create_head(branch)
1134 repo.head.reference = branch_head
Clark Boylanb640e052014-04-03 16:41:46 -07001135 f = open(fn, 'a')
James E. Blair97d902e2014-08-21 13:25:56 -07001136 f.write("test %s\n" % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001137 f.close()
1138 repo.index.add([fn])
James E. Blair97d902e2014-08-21 13:25:56 -07001139 repo.index.commit('%s commit' % branch)
Clark Boylanb640e052014-04-03 16:41:46 -07001140
James E. Blair97d902e2014-08-21 13:25:56 -07001141 repo.head.reference = repo.heads['master']
James E. Blair879dafb2015-07-17 14:04:49 -07001142 zuul.merger.merger.reset_repo_to_head(repo)
Clark Boylanb640e052014-04-03 16:41:46 -07001143 repo.git.clean('-x', '-f', '-d')
1144
1145 def ref_has_change(self, ref, change):
1146 path = os.path.join(self.git_root, change.project)
1147 repo = git.Repo(path)
Mike Heald8225f522014-11-21 09:52:33 +00001148 try:
1149 for commit in repo.iter_commits(ref):
1150 if commit.message.strip() == ('%s-1' % change.subject):
1151 return True
1152 except GitCommandError:
1153 pass
Clark Boylanb640e052014-04-03 16:41:46 -07001154 return False
1155
1156 def job_has_changes(self, *args):
1157 job = args[0]
1158 commits = args[1:]
1159 if isinstance(job, FakeBuild):
1160 parameters = job.parameters
1161 else:
1162 parameters = json.loads(job.arguments)
1163 project = parameters['ZUUL_PROJECT']
1164 path = os.path.join(self.git_root, project)
1165 repo = git.Repo(path)
1166 ref = parameters['ZUUL_REF']
1167 sha = parameters['ZUUL_COMMIT']
1168 repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
1169 repo_shas = [c.hexsha for c in repo.iter_commits(ref)]
1170 commit_messages = ['%s-1' % commit.subject for commit in commits]
1171 self.log.debug("Checking if job %s has changes; commit_messages %s;"
1172 " repo_messages %s; sha %s" % (job, commit_messages,
1173 repo_messages, sha))
1174 for msg in commit_messages:
1175 if msg not in repo_messages:
1176 self.log.debug(" messages do not match")
1177 return False
1178 if repo_shas[0] != sha:
1179 self.log.debug(" sha does not match")
1180 return False
1181 self.log.debug(" OK")
1182 return True
1183
1184 def registerJobs(self):
1185 count = 0
James E. Blair59fdbac2015-12-07 17:08:06 -08001186 for tenant in self.sched.abide.tenants.values():
1187 for job in tenant.layout.jobs.keys():
1188 self.worker.registerFunction('build:' + job)
1189 count += 1
Clark Boylanb640e052014-04-03 16:41:46 -07001190 self.worker.registerFunction('stop:' + self.worker.worker_id)
1191 count += 1
1192
1193 while len(self.gearman_server.functions) < count:
1194 time.sleep(0)
1195
James E. Blairb8c16472015-05-05 14:55:26 -07001196 def orderedRelease(self):
1197 # Run one build at a time to ensure non-race order:
1198 while len(self.builds):
1199 self.release(self.builds[0])
1200 self.waitUntilSettled()
1201
Clark Boylanb640e052014-04-03 16:41:46 -07001202 def release(self, job):
1203 if isinstance(job, FakeBuild):
1204 job.release()
1205 else:
1206 job.waiting = False
1207 self.log.debug("Queued job %s released" % job.unique)
1208 self.gearman_server.wakeConnections()
1209
1210 def getParameter(self, job, name):
1211 if isinstance(job, FakeBuild):
1212 return job.parameters[name]
1213 else:
1214 parameters = json.loads(job.arguments)
1215 return parameters[name]
1216
1217 def resetGearmanServer(self):
1218 self.worker.setFunctions([])
1219 while True:
1220 done = True
1221 for connection in self.gearman_server.active_connections:
1222 if (connection.functions and
1223 connection.client_id not in ['Zuul RPC Listener',
1224 'Zuul Merger']):
1225 done = False
1226 if done:
1227 break
1228 time.sleep(0)
1229 self.gearman_server.functions = set()
1230 self.rpc.register()
1231 self.merge_server.register()
1232
1233 def haveAllBuildsReported(self):
1234 # See if Zuul is waiting on a meta job to complete
1235 if self.launcher.meta_jobs:
1236 return False
1237 # Find out if every build that the worker has completed has been
1238 # reported back to Zuul. If it hasn't then that means a Gearman
1239 # event is still in transit and the system is not stable.
1240 for build in self.worker.build_history:
1241 zbuild = self.launcher.builds.get(build.uuid)
1242 if not zbuild:
1243 # It has already been reported
1244 continue
1245 # It hasn't been reported yet.
1246 return False
1247 # Make sure that none of the worker connections are in GRAB_WAIT
1248 for connection in self.worker.active_connections:
1249 if connection.state == 'GRAB_WAIT':
1250 return False
1251 return True
1252
1253 def areAllBuildsWaiting(self):
Clark Boylanb640e052014-04-03 16:41:46 -07001254 builds = self.launcher.builds.values()
1255 for build in builds:
1256 client_job = None
1257 for conn in self.launcher.gearman.active_connections:
1258 for j in conn.related_jobs.values():
1259 if j.unique == build.uuid:
1260 client_job = j
1261 break
1262 if not client_job:
1263 self.log.debug("%s is not known to the gearman client" %
1264 build)
James E. Blairf15139b2015-04-02 16:37:15 -07001265 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001266 if not client_job.handle:
1267 self.log.debug("%s has no handle" % client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001268 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001269 server_job = self.gearman_server.jobs.get(client_job.handle)
1270 if not server_job:
1271 self.log.debug("%s is not known to the gearman server" %
1272 client_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001273 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001274 if not hasattr(server_job, 'waiting'):
1275 self.log.debug("%s is being enqueued" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001276 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001277 if server_job.waiting:
1278 continue
1279 worker_job = self.worker.gearman_jobs.get(server_job.unique)
1280 if worker_job:
James E. Blairf15139b2015-04-02 16:37:15 -07001281 if build.number is None:
1282 self.log.debug("%s has not reported start" % worker_job)
1283 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001284 if worker_job.build.isWaiting():
1285 continue
1286 else:
1287 self.log.debug("%s is running" % worker_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001288 return False
Clark Boylanb640e052014-04-03 16:41:46 -07001289 else:
1290 self.log.debug("%s is unassigned" % server_job)
James E. Blairf15139b2015-04-02 16:37:15 -07001291 return False
1292 return True
Clark Boylanb640e052014-04-03 16:41:46 -07001293
Jan Hruban6b71aff2015-10-22 16:58:08 +02001294 def eventQueuesEmpty(self):
1295 for queue in self.event_queues:
1296 yield queue.empty()
1297
1298 def eventQueuesJoin(self):
1299 for queue in self.event_queues:
1300 queue.join()
1301
Clark Boylanb640e052014-04-03 16:41:46 -07001302 def waitUntilSettled(self):
1303 self.log.debug("Waiting until settled...")
1304 start = time.time()
1305 while True:
1306 if time.time() - start > 10:
1307 print 'queue status:',
Jan Hruban6b71aff2015-10-22 16:58:08 +02001308 print ' '.join(self.eventQueuesEmpty())
Clark Boylanb640e052014-04-03 16:41:46 -07001309 print self.areAllBuildsWaiting()
1310 raise Exception("Timeout waiting for Zuul to settle")
1311 # Make sure no new events show up while we're checking
1312 self.worker.lock.acquire()
1313 # have all build states propogated to zuul?
1314 if self.haveAllBuildsReported():
1315 # Join ensures that the queue is empty _and_ events have been
1316 # processed
Jan Hruban6b71aff2015-10-22 16:58:08 +02001317 self.eventQueuesJoin()
Clark Boylanb640e052014-04-03 16:41:46 -07001318 self.sched.run_handler_lock.acquire()
James E. Blair14abdf42015-12-09 16:11:53 -08001319 if (not self.merge_client.jobs and
Jan Hruban6b71aff2015-10-22 16:58:08 +02001320 all(self.eventQueuesEmpty()) and
Clark Boylanb640e052014-04-03 16:41:46 -07001321 self.haveAllBuildsReported() and
1322 self.areAllBuildsWaiting()):
1323 self.sched.run_handler_lock.release()
1324 self.worker.lock.release()
1325 self.log.debug("...settled.")
1326 return
1327 self.sched.run_handler_lock.release()
1328 self.worker.lock.release()
1329 self.sched.wake_event.wait(0.1)
1330
1331 def countJobResults(self, jobs, result):
1332 jobs = filter(lambda x: x.result == result, jobs)
1333 return len(jobs)
1334
1335 def getJobFromHistory(self, name):
1336 history = self.worker.build_history
1337 for job in history:
1338 if job.name == name:
1339 return job
1340 raise Exception("Unable to find job %s in history" % name)
1341
1342 def assertEmptyQueues(self):
1343 # Make sure there are no orphaned jobs
James E. Blair59fdbac2015-12-07 17:08:06 -08001344 for tenant in self.sched.abide.tenants.values():
1345 for pipeline in tenant.layout.pipelines.values():
1346 for queue in pipeline.queues:
1347 if len(queue.queue) != 0:
1348 print 'pipeline %s queue %s contents %s' % (
1349 pipeline.name, queue.name, queue.queue)
1350 self.assertEqual(len(queue.queue), 0,
1351 "Pipelines queues should be empty")
Clark Boylanb640e052014-04-03 16:41:46 -07001352
1353 def assertReportedStat(self, key, value=None, kind=None):
1354 start = time.time()
1355 while time.time() < (start + 5):
1356 for stat in self.statsd.stats:
1357 pprint.pprint(self.statsd.stats)
1358 k, v = stat.split(':')
1359 if key == k:
1360 if value is None and kind is None:
1361 return
1362 elif value:
1363 if value == v:
1364 return
1365 elif kind:
1366 if v.endswith('|' + kind):
1367 return
1368 time.sleep(0.1)
1369
1370 pprint.pprint(self.statsd.stats)
1371 raise Exception("Key %s not found in reported stats" % key)
James E. Blair59fdbac2015-12-07 17:08:06 -08001372
1373 def getPipeline(self, name):
James E. Blairf84026c2015-12-08 16:11:46 -08001374 return self.sched.abide.tenants.values()[0].layout.pipelines.get(name)
1375
1376 def updateConfigLayout(self, path):
1377 root = os.path.join(self.test_root, "config")
1378 os.makedirs(root)
1379 f = tempfile.NamedTemporaryFile(dir=root, delete=False)
1380 f.write("""
1381tenants:
1382 - name: openstack
1383 include:
1384 - %s
1385 """ % os.path.abspath(path))
1386 f.close()
1387 self.config.set('zuul', 'tenant_config', f.name)
James E. Blair14abdf42015-12-09 16:11:53 -08001388
1389 def addCommitToRepo(self, project, message, files, branch='master'):
1390 path = os.path.join(self.upstream_root, project)
1391 repo = git.Repo(path)
1392 repo.head.reference = branch
1393 zuul.merger.merger.reset_repo_to_head(repo)
1394 for fn, content in files.items():
1395 fn = os.path.join(path, fn)
1396 with open(fn, 'w') as f:
1397 f.write(content)
1398 repo.index.add([fn])
1399 commit = repo.index.commit(message)
1400 repo.heads[branch].commit = commit
1401 repo.head.reference = branch
1402 repo.git.clean('-x', '-f', '-d')
1403 repo.heads[branch].checkout()