blob: 75eb3a97bbee72565951843be91151dc32d43689 [file] [log] [blame]
Simon Glassd4144e42014-09-05 19:00:13 -06001#
2# Copyright (c) 2014 Google, Inc
3#
4# SPDX-License-Identifier: GPL-2.0+
5#
6
7import os
8import shutil
9import sys
10import tempfile
11import unittest
12
Simon Glass823e60b2014-09-05 19:00:16 -060013import board
Simon Glass8b985ee2014-09-05 19:00:15 -060014import bsettings
Simon Glassd4144e42014-09-05 19:00:13 -060015import cmdline
16import command
17import control
18import gitutil
19import terminal
20import toolchain
21
Simon Glass8b985ee2014-09-05 19:00:15 -060022settings_data = '''
23# Buildman settings file
24
25[toolchain]
26
27[toolchain-alias]
28
29[make-flags]
30src=/home/sjg/c/src
31chroot=/home/sjg/c/chroot
32vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36'''
37
Simon Glass823e60b2014-09-05 19:00:16 -060038boards = [
39 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0', ''],
40 ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41 ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42 ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
43 ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
44]
45
Simon Glassdfb7e932014-09-05 19:00:20 -060046commit_shortlog = """4aca821 patman: Avoid changing the order of tags
4739403bb patman: Use --no-pager' to stop git from forking a pager
48db6e6f2 patman: Remove the -a option
49f2ccf03 patman: Correct unit tests to run correctly
501d097f9 patman: Fix indentation in terminal.py
51d073747 patman: Support the 'reverse' option for 'git log
52"""
53
54commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
55Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
56Date: Fri Aug 22 19:12:41 2014 +0900
57
58 buildman: refactor help message
59
60 "buildman [options]" is displayed by default.
61
62 Append the rest of help messages to parser.usage
63 instead of replacing it.
64
65 Besides, "-b <branch>" is not mandatory since commit fea5858e.
66 Drop it from the usage.
67
68 Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
69""",
70"""commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
71Author: Simon Glass <sjg@chromium.org>
72Date: Thu Aug 14 16:48:25 2014 -0600
73
74 patman: Support the 'reverse' option for 'git log'
75
76 This option is currently not supported, but needs to be, for buildman to
77 operate as expected.
78
79 Series-changes: 7
80 - Add new patch to fix the 'reverse' bug
81
Simon Glass950a2312014-09-05 19:00:23 -060082 Series-version: 8
Simon Glassdfb7e932014-09-05 19:00:20 -060083
84 Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
85 Reported-by: York Sun <yorksun@freescale.com>
86 Signed-off-by: Simon Glass <sjg@chromium.org>
87
88""",
89"""commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
90Author: Simon Glass <sjg@chromium.org>
91Date: Sat Aug 9 11:44:32 2014 -0600
92
93 patman: Fix indentation in terminal.py
94
95 This code came from a different project with 2-character indentation. Fix
96 it for U-Boot.
97
98 Series-changes: 6
99 - Add new patch to fix indentation in teminal.py
100
101 Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
102 Signed-off-by: Simon Glass <sjg@chromium.org>
103
104""",
105"""commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
106Author: Simon Glass <sjg@chromium.org>
107Date: Sat Aug 9 11:08:24 2014 -0600
108
109 patman: Correct unit tests to run correctly
110
111 It seems that doctest behaves differently now, and some of the unit tests
112 do not run. Adjust the tests to work correctly.
113
114 ./tools/patman/patman --test
115 <unittest.result.TestResult run=10 errors=0 failures=0>
116
117 Series-changes: 6
118 - Add new patch to fix patman unit tests
119
120 Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
121
122""",
123"""commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
124Author: Simon Glass <sjg@chromium.org>
125Date: Sat Aug 9 12:06:02 2014 -0600
126
127 patman: Remove the -a option
128
129 It seems that this is no longer needed, since checkpatch.pl will catch
130 whitespace problems in patches. Also the option is not widely used, so
131 it seems safe to just remove it.
132
133 Series-changes: 6
134 - Add new patch to remove patman's -a option
135
136 Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
137 Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
138
139""",
140"""commit 39403bb4f838153028a6f21ca30bf100f3791133
141Author: Simon Glass <sjg@chromium.org>
142Date: Thu Aug 14 21:50:52 2014 -0600
143
144 patman: Use --no-pager' to stop git from forking a pager
145
146""",
147"""commit 4aca821e27e97925c039e69fd37375b09c6f129c
148Author: Simon Glass <sjg@chromium.org>
149Date: Fri Aug 22 15:57:39 2014 -0600
150
151 patman: Avoid changing the order of tags
152
153 patman collects tags that it sees in the commit and places them nicely
154 sorted at the end of the patch. However, this is not really necessary and
155 in fact is apparently not desirable.
156
157 Series-changes: 9
158 - Add new patch to avoid changing the order of tags
159
Simon Glass950a2312014-09-05 19:00:23 -0600160 Series-version: 9
161
Simon Glassdfb7e932014-09-05 19:00:20 -0600162 Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
163 Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
164"""]
165
166TEST_BRANCH = '__testbranch'
167
Simon Glassd4144e42014-09-05 19:00:13 -0600168class TestFunctional(unittest.TestCase):
169 """Functional test for buildman.
170
171 This aims to test from just below the invocation of buildman (parsing
172 of arguments) to 'make' and 'git' invocation. It is not a true
173 emd-to-end test, as it mocks git, make and the tool chain. But this
174 makes it easier to detect when the builder is doing the wrong thing,
175 since in many cases this test code will fail. For example, only a
176 very limited subset of 'git' arguments is supported - anything
177 unexpected will fail.
178 """
179 def setUp(self):
180 self._base_dir = tempfile.mkdtemp()
181 self._git_dir = os.path.join(self._base_dir, 'src')
182 self._buildman_pathname = sys.argv[0]
183 self._buildman_dir = os.path.dirname(sys.argv[0])
184 command.test_result = self._HandleCommand
Simon Glassdfb7e932014-09-05 19:00:20 -0600185 self.setupToolchains()
186 self._toolchains.Add('arm-gcc', test=False)
187 self._toolchains.Add('powerpc-gcc', test=False)
Simon Glass8b985ee2014-09-05 19:00:15 -0600188 bsettings.Setup(None)
189 bsettings.AddFile(settings_data)
Simon Glass823e60b2014-09-05 19:00:16 -0600190 self._boards = board.Boards()
191 for brd in boards:
192 self._boards.AddBoard(board.Board(*brd))
Simon Glassd4144e42014-09-05 19:00:13 -0600193
Simon Glassdfb7e932014-09-05 19:00:20 -0600194 # Directories where the source been cloned
195 self._clone_dirs = []
196 self._commits = len(commit_shortlog.splitlines()) + 1
197 self._total_builds = self._commits * len(boards)
198
199 # Number of calls to make
200 self._make_calls = 0
201
202 # Map of [board, commit] to error messages
203 self._error = {}
204
Simon Glassf7582ce2014-09-05 19:00:22 -0600205 self._test_branch = TEST_BRANCH
206
Simon Glassdfb7e932014-09-05 19:00:20 -0600207 # Avoid sending any output and clear all terminal output
208 terminal.SetPrintTestMode()
209 terminal.GetPrintTestLines()
210
Simon Glassd4144e42014-09-05 19:00:13 -0600211 def tearDown(self):
212 shutil.rmtree(self._base_dir)
213
Simon Glassdfb7e932014-09-05 19:00:20 -0600214 def setupToolchains(self):
215 self._toolchains = toolchain.Toolchains()
216 self._toolchains.Add('gcc', test=False)
217
Simon Glassd4144e42014-09-05 19:00:13 -0600218 def _RunBuildman(self, *args):
219 return command.RunPipe([[self._buildman_pathname] + list(args)],
220 capture=True, capture_stderr=True)
221
Simon Glassdfb7e932014-09-05 19:00:20 -0600222 def _RunControl(self, *args, **kwargs):
Simon Glassd4144e42014-09-05 19:00:13 -0600223 sys.argv = [sys.argv[0]] + list(args)
224 options, args = cmdline.ParseArgs()
Simon Glassdfb7e932014-09-05 19:00:20 -0600225 result = control.DoBuildman(options, args, toolchains=self._toolchains,
226 make_func=self._HandleMake, boards=self._boards,
227 clean_dir=kwargs.get('clean_dir', True))
228 self._builder = control.builder
229 return result
Simon Glassd4144e42014-09-05 19:00:13 -0600230
231 def testFullHelp(self):
232 command.test_result = None
233 result = self._RunBuildman('-H')
234 help_file = os.path.join(self._buildman_dir, 'README')
235 self.assertEqual(len(result.stdout), os.path.getsize(help_file))
236 self.assertEqual(0, len(result.stderr))
237 self.assertEqual(0, result.return_code)
238
239 def testHelp(self):
240 command.test_result = None
241 result = self._RunBuildman('-h')
242 help_file = os.path.join(self._buildman_dir, 'README')
243 self.assertTrue(len(result.stdout) > 1000)
244 self.assertEqual(0, len(result.stderr))
245 self.assertEqual(0, result.return_code)
246
247 def testGitSetup(self):
248 """Test gitutils.Setup(), from outside the module itself"""
249 command.test_result = command.CommandResult(return_code=1)
250 gitutil.Setup()
251 self.assertEqual(gitutil.use_no_decorate, False)
252
253 command.test_result = command.CommandResult(return_code=0)
254 gitutil.Setup()
255 self.assertEqual(gitutil.use_no_decorate, True)
256
257 def _HandleCommandGitLog(self, args):
258 if '-n0' in args:
259 return command.CommandResult(return_code=0)
Simon Glassf7582ce2014-09-05 19:00:22 -0600260 elif args[-1] == 'upstream/master..%s' % self._test_branch:
Simon Glassdfb7e932014-09-05 19:00:20 -0600261 return command.CommandResult(return_code=0, stdout=commit_shortlog)
262 elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
Simon Glassf7582ce2014-09-05 19:00:22 -0600263 if args[-1] == self._test_branch:
Simon Glassdfb7e932014-09-05 19:00:20 -0600264 count = int(args[3][2:])
265 return command.CommandResult(return_code=0,
266 stdout=''.join(commit_log[:count]))
Simon Glassd4144e42014-09-05 19:00:13 -0600267
268 # Not handled, so abort
269 print 'git log', args
270 sys.exit(1)
271
Simon Glassdfb7e932014-09-05 19:00:20 -0600272 def _HandleCommandGitConfig(self, args):
273 config = args[0]
274 if config == 'sendemail.aliasesfile':
275 return command.CommandResult(return_code=0)
276 elif config.startswith('branch.badbranch'):
277 return command.CommandResult(return_code=1)
Simon Glassf7582ce2014-09-05 19:00:22 -0600278 elif config == 'branch.%s.remote' % self._test_branch:
Simon Glassdfb7e932014-09-05 19:00:20 -0600279 return command.CommandResult(return_code=0, stdout='upstream\n')
Simon Glassf7582ce2014-09-05 19:00:22 -0600280 elif config == 'branch.%s.merge' % self._test_branch:
Simon Glassdfb7e932014-09-05 19:00:20 -0600281 return command.CommandResult(return_code=0,
282 stdout='refs/heads/master\n')
283
284 # Not handled, so abort
285 print 'git config', args
286 sys.exit(1)
287
Simon Glassd4144e42014-09-05 19:00:13 -0600288 def _HandleCommandGit(self, in_args):
289 """Handle execution of a git command
290
291 This uses a hacked-up parser.
292
293 Args:
294 in_args: Arguments after 'git' from the command line
295 """
296 git_args = [] # Top-level arguments to git itself
297 sub_cmd = None # Git sub-command selected
298 args = [] # Arguments to the git sub-command
299 for arg in in_args:
300 if sub_cmd:
301 args.append(arg)
302 elif arg[0] == '-':
303 git_args.append(arg)
304 else:
Simon Glassdfb7e932014-09-05 19:00:20 -0600305 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
306 git_args.append(arg)
307 else:
308 sub_cmd = arg
Simon Glassd4144e42014-09-05 19:00:13 -0600309 if sub_cmd == 'config':
Simon Glassdfb7e932014-09-05 19:00:20 -0600310 return self._HandleCommandGitConfig(args)
Simon Glassd4144e42014-09-05 19:00:13 -0600311 elif sub_cmd == 'log':
312 return self._HandleCommandGitLog(args)
Simon Glassdfb7e932014-09-05 19:00:20 -0600313 elif sub_cmd == 'clone':
314 return command.CommandResult(return_code=0)
315 elif sub_cmd == 'checkout':
316 return command.CommandResult(return_code=0)
Simon Glassd4144e42014-09-05 19:00:13 -0600317
318 # Not handled, so abort
319 print 'git', git_args, sub_cmd, args
320 sys.exit(1)
321
322 def _HandleCommandNm(self, args):
323 return command.CommandResult(return_code=0)
324
325 def _HandleCommandObjdump(self, args):
326 return command.CommandResult(return_code=0)
327
328 def _HandleCommandSize(self, args):
329 return command.CommandResult(return_code=0)
330
331 def _HandleCommand(self, **kwargs):
332 """Handle a command execution.
333
334 The command is in kwargs['pipe-list'], as a list of pipes, each a
335 list of commands. The command should be emulated as required for
336 testing purposes.
337
338 Returns:
339 A CommandResult object
340 """
341 pipe_list = kwargs['pipe_list']
Simon Glassdfb7e932014-09-05 19:00:20 -0600342 wc = False
Simon Glassd4144e42014-09-05 19:00:13 -0600343 if len(pipe_list) != 1:
Simon Glassdfb7e932014-09-05 19:00:20 -0600344 if pipe_list[1] == ['wc', '-l']:
345 wc = True
346 else:
347 print 'invalid pipe', kwargs
348 sys.exit(1)
Simon Glassd4144e42014-09-05 19:00:13 -0600349 cmd = pipe_list[0][0]
350 args = pipe_list[0][1:]
Simon Glassdfb7e932014-09-05 19:00:20 -0600351 result = None
Simon Glassd4144e42014-09-05 19:00:13 -0600352 if cmd == 'git':
Simon Glassdfb7e932014-09-05 19:00:20 -0600353 result = self._HandleCommandGit(args)
Simon Glassd4144e42014-09-05 19:00:13 -0600354 elif cmd == './scripts/show-gnu-make':
355 return command.CommandResult(return_code=0, stdout='make')
Simon Glassdfb7e932014-09-05 19:00:20 -0600356 elif cmd.endswith('nm'):
Simon Glassd4144e42014-09-05 19:00:13 -0600357 return self._HandleCommandNm(args)
Simon Glassdfb7e932014-09-05 19:00:20 -0600358 elif cmd.endswith('objdump'):
Simon Glassd4144e42014-09-05 19:00:13 -0600359 return self._HandleCommandObjdump(args)
Simon Glassdfb7e932014-09-05 19:00:20 -0600360 elif cmd.endswith( 'size'):
Simon Glassd4144e42014-09-05 19:00:13 -0600361 return self._HandleCommandSize(args)
362
Simon Glassdfb7e932014-09-05 19:00:20 -0600363 if not result:
364 # Not handled, so abort
365 print 'unknown command', kwargs
366 sys.exit(1)
367
368 if wc:
369 result.stdout = len(result.stdout.splitlines())
370 return result
Simon Glassd4144e42014-09-05 19:00:13 -0600371
372 def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
373 """Handle execution of 'make'
374
375 Args:
376 commit: Commit object that is being built
377 brd: Board object that is being built
378 stage: Stage that we are at (mrproper, config, build)
379 cwd: Directory where make should be run
380 args: Arguments to pass to make
381 kwargs: Arguments to pass to command.RunPipe()
382 """
Simon Glassdfb7e932014-09-05 19:00:20 -0600383 self._make_calls += 1
Simon Glassd4144e42014-09-05 19:00:13 -0600384 if stage == 'mrproper':
385 return command.CommandResult(return_code=0)
386 elif stage == 'config':
387 return command.CommandResult(return_code=0,
388 combined='Test configuration complete')
389 elif stage == 'build':
Simon Glassdfb7e932014-09-05 19:00:20 -0600390 stderr = ''
391 if type(commit) is not str:
392 stderr = self._error.get((brd.target, commit.sequence))
393 if stderr:
394 return command.CommandResult(return_code=1, stderr=stderr)
Simon Glassd4144e42014-09-05 19:00:13 -0600395 return command.CommandResult(return_code=0)
396
397 # Not handled, so abort
398 print 'make', stage
399 sys.exit(1)
400
Simon Glassdfb7e932014-09-05 19:00:20 -0600401 # Example function to print output lines
402 def print_lines(self, lines):
403 print len(lines)
404 for line in lines:
405 print line
406 #self.print_lines(terminal.GetPrintTestLines())
407
Simon Glass823e60b2014-09-05 19:00:16 -0600408 def testNoBoards(self):
409 """Test that buildman aborts when there are no boards"""
410 self._boards = board.Boards()
411 with self.assertRaises(SystemExit):
412 self._RunControl()
413
Simon Glassd4144e42014-09-05 19:00:13 -0600414 def testCurrentSource(self):
415 """Very simple test to invoke buildman on the current source"""
Simon Glassdfb7e932014-09-05 19:00:20 -0600416 self.setupToolchains();
Simon Glassd4144e42014-09-05 19:00:13 -0600417 self._RunControl()
418 lines = terminal.GetPrintTestLines()
Simon Glassdfb7e932014-09-05 19:00:20 -0600419 self.assertIn('Building current source for %d boards' % len(boards),
420 lines[0].text)
421
422 def testBadBranch(self):
423 """Test that we can detect an invalid branch"""
424 with self.assertRaises(ValueError):
425 self._RunControl('-b', 'badbranch')
426
427 def testBadToolchain(self):
428 """Test that missing toolchains are detected"""
429 self.setupToolchains();
430 ret_code = self._RunControl('-b', TEST_BRANCH)
431 lines = terminal.GetPrintTestLines()
432
433 # Buildman always builds the upstream commit as well
434 self.assertIn('Building %d commits for %d boards' %
435 (self._commits, len(boards)), lines[0].text)
436 self.assertEqual(self._builder.count, self._total_builds)
437
438 # Only sandbox should succeed, the others don't have toolchains
439 self.assertEqual(self._builder.fail,
440 self._total_builds - self._commits)
441 self.assertEqual(ret_code, 128)
442
443 for commit in range(self._commits):
444 for board in self._boards.GetList():
445 if board.arch != 'sandbox':
446 errfile = self._builder.GetErrFile(commit, board.target)
447 fd = open(errfile)
448 self.assertEqual(fd.readlines(),
449 ['No tool chain for %s\n' % board.arch])
450 fd.close()
451
452 def testBranch(self):
453 """Test building a branch with all toolchains present"""
454 self._RunControl('-b', TEST_BRANCH)
455 self.assertEqual(self._builder.count, self._total_builds)
456 self.assertEqual(self._builder.fail, 0)
457
458 def testCount(self):
459 """Test building a specific number of commitst"""
460 self._RunControl('-b', TEST_BRANCH, '-c2')
461 self.assertEqual(self._builder.count, 2 * len(boards))
462 self.assertEqual(self._builder.fail, 0)
463 # Each board has a mrproper, config, and then one make per commit
464 self.assertEqual(self._make_calls, len(boards) * (2 + 2))
465
466 def testIncremental(self):
467 """Test building a branch twice - the second time should do nothing"""
468 self._RunControl('-b', TEST_BRANCH)
469
470 # Each board has a mrproper, config, and then one make per commit
471 self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
472 self._make_calls = 0
473 self._RunControl('-b', TEST_BRANCH, clean_dir=False)
474 self.assertEqual(self._make_calls, 0)
475 self.assertEqual(self._builder.count, self._total_builds)
476 self.assertEqual(self._builder.fail, 0)
477
478 def testForceBuild(self):
479 """The -f flag should force a rebuild"""
480 self._RunControl('-b', TEST_BRANCH)
481 self._make_calls = 0
482 self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
483 # Each board has a mrproper, config, and then one make per commit
484 self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
485
486 def testForceReconfigure(self):
487 """The -f flag should force a rebuild"""
488 self._RunControl('-b', TEST_BRANCH, '-C')
489 # Each commit has a mrproper, config and make
490 self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
491
492 def testErrors(self):
493 """Test handling of build errors"""
494 self._error['board2', 1] = 'fred\n'
495 self._RunControl('-b', TEST_BRANCH)
496 self.assertEqual(self._builder.count, self._total_builds)
497 self.assertEqual(self._builder.fail, 1)
498
499 # Remove the error. This should have no effect since the commit will
500 # not be rebuilt
501 del self._error['board2', 1]
502 self._make_calls = 0
503 self._RunControl('-b', TEST_BRANCH, clean_dir=False)
504 self.assertEqual(self._builder.count, self._total_builds)
505 self.assertEqual(self._make_calls, 0)
506 self.assertEqual(self._builder.fail, 1)
507
508 # Now use the -F flag to force rebuild of the bad commit
509 self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
510 self.assertEqual(self._builder.count, self._total_builds)
511 self.assertEqual(self._builder.fail, 0)
512 self.assertEqual(self._make_calls, 3)
Simon Glassf7582ce2014-09-05 19:00:22 -0600513
514 def testBranchWithSlash(self):
515 """Test building a branch with a '/' in the name"""
516 self._test_branch = '/__dev/__testbranch'
517 self._RunControl('-b', self._test_branch, clean_dir=False)
518 self.assertEqual(self._builder.count, self._total_builds)
519 self.assertEqual(self._builder.fail, 0)