blob: acb0810457e4c9a1a2b8344b6f1639f90d09b131 [file] [log] [blame]
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001# Copyright (c) 2013 The Chromium OS Authors.
2#
3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
4#
Wolfgang Denk1a459662013-07-08 09:37:19 +02005# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006#
7
8import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00009from datetime import datetime, timedelta
10import glob
11import os
12import re
13import Queue
14import shutil
Simon Glass2f256642016-09-18 16:48:37 -060015import signal
Simon Glassfc3fe1c2013-04-03 11:07:16 +000016import string
17import sys
Simon Glassd436e382016-09-18 16:48:35 -060018import threading
Simon Glassfc3fe1c2013-04-03 11:07:16 +000019import time
20
Simon Glass190064b2014-08-09 15:33:00 -060021import builderthread
Simon Glassfc3fe1c2013-04-03 11:07:16 +000022import command
23import gitutil
24import terminal
Simon Glass4653a882014-09-05 19:00:07 -060025from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000026import toolchain
27
28
29"""
30Theory of Operation
31
32Please see README for user documentation, and you should be familiar with
33that before trying to make sense of this.
34
35Buildman works by keeping the machine as busy as possible, building different
36commits for different boards on multiple CPUs at once.
37
38The source repo (self.git_dir) contains all the commits to be built. Each
39thread works on a single board at a time. It checks out the first commit,
40configures it for that board, then builds it. Then it checks out the next
41commit and builds it (typically without re-configuring). When it runs out
42of commits, it gets another job from the builder and starts again with that
43board.
44
45Clearly the builder threads could work either way - they could check out a
46commit and then built it for all boards. Using separate directories for each
47commit/board pair they could leave their build product around afterwards
48also.
49
50The intent behind building a single board for multiple commits, is to make
51use of incremental builds. Since each commit is built incrementally from
52the previous one, builds are faster. Reconfiguring for a different board
53removes all intermediate object files.
54
55Many threads can be working at once, but each has its own working directory.
56When a thread finishes a build, it puts the output files into a result
57directory.
58
59The base directory used by buildman is normally '../<branch>', i.e.
60a directory higher than the source repository and named after the branch
61being built.
62
63Within the base directory, we have one subdirectory for each commit. Within
64that is one subdirectory for each board. Within that is the build output for
65that commit/board combination.
66
67Buildman also create working directories for each thread, in a .bm-work/
68subdirectory in the base dir.
69
70As an example, say we are building branch 'us-net' for boards 'sandbox' and
71'seaboard', and say that us-net has two commits. We will have directories
72like this:
73
74us-net/ base directory
75 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
76 sandbox/
77 u-boot.bin
78 seaboard/
79 u-boot.bin
80 02_of_02_g4ed4ebc_net--Check-tftp-comp/
81 sandbox/
82 u-boot.bin
83 seaboard/
84 u-boot.bin
85 .bm-work/
86 00/ working directory for thread 0 (contains source checkout)
87 build/ build output
88 01/ working directory for thread 1
89 build/ build output
90 ...
91u-boot/ source directory
92 .git/ repository
93"""
94
95# Possible build outcomes
96OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
97
Simon Glass9a6d2e22017-04-12 18:23:26 -060098# Translate a commit subject into a valid filename (and handle unicode)
99trans_valid_chars = string.maketrans('/: ', '---')
100trans_valid_chars = trans_valid_chars.decode('latin-1')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000101
Simon Glassb464f8e2016-11-13 14:25:53 -0700102BASE_CONFIG_FILENAMES = [
103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
104]
105
106EXTRA_CONFIG_FILENAMES = [
Simon Glass843312d2015-02-05 22:06:15 -0700107 '.config', '.config-spl', '.config-tpl',
108 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
109 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glass843312d2015-02-05 22:06:15 -0700110]
111
Simon Glass8270e3c2015-08-25 21:52:14 -0600112class Config:
113 """Holds information about configuration settings for a board."""
Simon Glassb464f8e2016-11-13 14:25:53 -0700114 def __init__(self, config_filename, target):
Simon Glass8270e3c2015-08-25 21:52:14 -0600115 self.target = target
116 self.config = {}
Simon Glassb464f8e2016-11-13 14:25:53 -0700117 for fname in config_filename:
Simon Glass8270e3c2015-08-25 21:52:14 -0600118 self.config[fname] = {}
119
120 def Add(self, fname, key, value):
121 self.config[fname][key] = value
122
123 def __hash__(self):
124 val = 0
125 for fname in self.config:
126 for key, value in self.config[fname].iteritems():
127 print key, value
128 val = val ^ hash(key) & hash(value)
129 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000130
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000131class Builder:
132 """Class for building U-Boot for a particular commit.
133
134 Public members: (many should ->private)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000135 already_done: Number of builds already completed
136 base_dir: Base directory to use for builder
137 checkout: True to check out source, False to skip that step.
138 This is used for testing.
139 col: terminal.Color() object
140 count: Number of commits to build
141 do_make: Method to call to invoke Make
142 fail: Number of builds that failed due to error
143 force_build: Force building even if a build already exists
144 force_config_on_failure: If a commit fails for a board, disable
145 incremental building for the next commit we build for that
146 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600147 force_build_failures: If a previously-built build (i.e. built on
148 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000149 git_dir: Git directory containing source repository
150 last_line_len: Length of the last line we printed (used for erasing
151 it with new progress information)
152 num_jobs: Number of jobs to run at once (passed to make as -j)
153 num_threads: Number of builder threads to run
154 out_queue: Queue of results to process
155 re_make_err: Compiled regular expression for ignore_lines
156 queue: Queue of jobs to run
157 threads: List of active threads
158 toolchains: Toolchains object to use for building
159 upto: Current commit number we are building (0.count-1)
160 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600161 force_reconfig: Reconfigure U-Boot on each comiit. This disables
162 incremental building, where buildman reconfigures on the first
163 commit for a baord, and then just does an incremental build for
164 the following commits. In fact buildman will reconfigure and
165 retry for any failing commits, so generally the only effect of
166 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600167 in_tree: Build U-Boot in-tree instead of specifying an output
168 directory separate from the source code. This option is really
169 only useful for testing in-tree builds.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000170
171 Private members:
172 _base_board_dict: Last-summarised Dict of boards
173 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600174 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000175 _build_period_us: Time taken for a single build (float object).
176 _complete_delay: Expected delay until completion (timedelta)
177 _next_delay_update: Next time we plan to display a progress update
178 (datatime)
179 _show_unknown: Show unknown boards (those not built) in summary
180 _timestamps: List of timestamps for the completion of the last
181 last _timestamp_count builds. Each is a datetime object.
182 _timestamp_count: Number of timestamps to keep in our list.
183 _working_dir: Base working directory containing all threads
184 """
185 class Outcome:
186 """Records a build outcome for a single make invocation
187
188 Public Members:
189 rc: Outcome value (OUTCOME_...)
190 err_lines: List of error lines or [] if none
191 sizes: Dictionary of image size information, keyed by filename
192 - Each value is itself a dictionary containing
193 values for 'text', 'data' and 'bss', being the integer
194 size in bytes of each section.
195 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
196 value is itself a dictionary:
197 key: function name
198 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700199 config: Dictionary keyed by filename - e.g. '.config'. Each
200 value is itself a dictionary:
201 key: config name
202 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000203 """
Simon Glass843312d2015-02-05 22:06:15 -0700204 def __init__(self, rc, err_lines, sizes, func_sizes, config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000205 self.rc = rc
206 self.err_lines = err_lines
207 self.sizes = sizes
208 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700209 self.config = config
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000210
211 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700212 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600213 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glassb50113f2016-11-13 14:25:51 -0700214 incremental=False, per_board_out_dir=False,
Simon Glassb464f8e2016-11-13 14:25:53 -0700215 config_only=False, squash_config_y=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000216 """Create a new Builder object
217
218 Args:
219 toolchains: Toolchains object to use for building
220 base_dir: Base directory to use for builder
221 git_dir: Git directory containing source repository
222 num_threads: Number of builder threads to run
223 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900224 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000225 checkout: True to check out source, False to skip that step.
226 This is used for testing.
227 show_unknown: Show unknown boards (those not built) in summary
228 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700229 no_subdirs: Don't create subdirectories when building current
230 source for a single board
231 full_path: Return the full path in CROSS_COMPILE and don't set
232 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700233 verbose_build: Run build with V=1 and don't use 'make -s'
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600234 incremental: Always perform incremental builds; don't run make
235 mrproper when configuring
236 per_board_out_dir: Build in a separate persistent directory per
237 board rather than a thread-specific directory
Simon Glassb50113f2016-11-13 14:25:51 -0700238 config_only: Only configure each build, don't build it
Simon Glassb464f8e2016-11-13 14:25:53 -0700239 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000240 """
241 self.toolchains = toolchains
242 self.base_dir = base_dir
243 self._working_dir = os.path.join(base_dir, '.bm-work')
244 self.threads = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000245 self.do_make = self.Make
Masahiro Yamada99796922014-07-22 11:19:09 +0900246 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000247 self.checkout = checkout
248 self.num_threads = num_threads
249 self.num_jobs = num_jobs
250 self.already_done = 0
251 self.force_build = False
252 self.git_dir = git_dir
253 self._show_unknown = show_unknown
254 self._timestamp_count = 10
255 self._build_period_us = None
256 self._complete_delay = None
257 self._next_delay_update = datetime.now()
258 self.force_config_on_failure = True
Simon Glass4266dc22014-07-13 12:22:31 -0600259 self.force_build_failures = False
Simon Glass97e91522014-07-14 17:51:02 -0600260 self.force_reconfig = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000261 self._step = step
Simon Glass189a4962014-07-14 17:51:03 -0600262 self.in_tree = False
Simon Glass28370c12014-08-09 15:33:06 -0600263 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700264 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700265 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700266 self.verbose_build = verbose_build
Simon Glassb50113f2016-11-13 14:25:51 -0700267 self.config_only = config_only
Simon Glassb464f8e2016-11-13 14:25:53 -0700268 self.squash_config_y = squash_config_y
269 self.config_filenames = BASE_CONFIG_FILENAMES
270 if not self.squash_config_y:
271 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000272
273 self.col = terminal.Color()
274
Simon Glasse30965d2014-08-28 09:43:44 -0600275 self._re_function = re.compile('(.*): In function.*')
276 self._re_files = re.compile('In file included from.*')
277 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
278 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
279
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000280 self.queue = Queue.Queue()
281 self.out_queue = Queue.Queue()
282 for i in range(self.num_threads):
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600283 t = builderthread.BuilderThread(self, i, incremental,
284 per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000285 t.setDaemon(True)
286 t.start()
287 self.threads.append(t)
288
289 self.last_line_len = 0
Simon Glass190064b2014-08-09 15:33:00 -0600290 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000291 t.setDaemon(True)
292 t.start()
293 self.threads.append(t)
294
295 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
296 self.re_make_err = re.compile('|'.join(ignore_lines))
297
Simon Glass2f256642016-09-18 16:48:37 -0600298 # Handle existing graceful with SIGINT / Ctrl-C
299 signal.signal(signal.SIGINT, self.signal_handler)
300
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000301 def __del__(self):
302 """Get rid of all threads created by the builder"""
303 for t in self.threads:
304 del t
305
Simon Glass2f256642016-09-18 16:48:37 -0600306 def signal_handler(self, signal, frame):
307 sys.exit(1)
308
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600309 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600310 show_detail=False, show_bloat=False,
Simon Glass843312d2015-02-05 22:06:15 -0700311 list_error_boards=False, show_config=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600312 """Setup display options for the builder.
313
314 show_errors: True to show summarised error/warning info
315 show_sizes: Show size deltas
316 show_detail: Show detail for each board
317 show_bloat: Show detail for each function
Simon Glassed966652014-08-28 09:43:43 -0600318 list_error_boards: Show the boards which caused each error/warning
Simon Glass843312d2015-02-05 22:06:15 -0700319 show_config: Show config deltas
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600320 """
321 self._show_errors = show_errors
322 self._show_sizes = show_sizes
323 self._show_detail = show_detail
324 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600325 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700326 self._show_config = show_config
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600327
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000328 def _AddTimestamp(self):
329 """Add a new timestamp to the list and record the build period.
330
331 The build period is the length of time taken to perform a single
332 build (one board, one commit).
333 """
334 now = datetime.now()
335 self._timestamps.append(now)
336 count = len(self._timestamps)
337 delta = self._timestamps[-1] - self._timestamps[0]
338 seconds = delta.total_seconds()
339
340 # If we have enough data, estimate build period (time taken for a
341 # single build) and therefore completion time.
342 if count > 1 and self._next_delay_update < now:
343 self._next_delay_update = now + timedelta(seconds=2)
344 if seconds > 0:
345 self._build_period = float(seconds) / count
346 todo = self.count - self.upto
347 self._complete_delay = timedelta(microseconds=
348 self._build_period * todo * 1000000)
349 # Round it
350 self._complete_delay -= timedelta(
351 microseconds=self._complete_delay.microseconds)
352
353 if seconds > 60:
354 self._timestamps.popleft()
355 count -= 1
356
357 def ClearLine(self, length):
358 """Clear any characters on the current line
359
360 Make way for a new line of length 'length', by outputting enough
361 spaces to clear out the old line. Then remember the new length for
362 next time.
363
364 Args:
365 length: Length of new line, in characters
366 """
367 if length < self.last_line_len:
Simon Glass4653a882014-09-05 19:00:07 -0600368 Print(' ' * (self.last_line_len - length), newline=False)
369 Print('\r', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000370 self.last_line_len = length
371 sys.stdout.flush()
372
373 def SelectCommit(self, commit, checkout=True):
374 """Checkout the selected commit for this build
375 """
376 self.commit = commit
377 if checkout and self.checkout:
378 gitutil.Checkout(commit.hash)
379
380 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
381 """Run make
382
383 Args:
384 commit: Commit object that is being built
385 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200386 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000387 cwd: Directory where make should be run
388 args: Arguments to pass to make
389 kwargs: Arguments to pass to command.RunPipe()
390 """
Masahiro Yamada99796922014-07-22 11:19:09 +0900391 cmd = [self.gnu_make] + list(args)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000392 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
393 cwd=cwd, raise_on_error=False, **kwargs)
Simon Glass40f11fc2015-02-05 22:06:12 -0700394 if self.verbose_build:
395 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
396 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000397 return result
398
399 def ProcessResult(self, result):
400 """Process the result of a build, showing progress information
401
402 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600403 result: A CommandResult object, which indicates the result for
404 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000405 """
406 col = terminal.Color()
407 if result:
408 target = result.brd.target
409
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000410 self.upto += 1
411 if result.return_code != 0:
412 self.fail += 1
413 elif result.stderr:
414 self.warned += 1
415 if result.already_done:
416 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600417 if self._verbose:
Simon Glass4653a882014-09-05 19:00:07 -0600418 Print('\r', newline=False)
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600419 self.ClearLine(0)
420 boards_selected = {target : result.brd}
421 self.ResetResultSummary(boards_selected)
422 self.ProduceResultSummary(result.commit_upto, self.commits,
423 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000424 else:
425 target = '(starting)'
426
427 # Display separate counts for ok, warned and fail
428 ok = self.upto - self.warned - self.fail
429 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
430 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
431 line += self.col.Color(self.col.RED, '%5d' % self.fail)
432
433 name = ' /%-5d ' % self.count
434
435 # Add our current completion time estimate
436 self._AddTimestamp()
437 if self._complete_delay:
438 name += '%s : ' % self._complete_delay
439 # When building all boards for a commit, we can print a commit
440 # progress message.
441 if result and result.commit_upto is None:
442 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
443 self.commit_count)
444
445 name += target
Simon Glass4653a882014-09-05 19:00:07 -0600446 Print(line + name, newline=False)
Simon Glass960421e2016-11-15 15:32:59 -0700447 length = 16 + len(name)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000448 self.ClearLine(length)
449
450 def _GetOutputDir(self, commit_upto):
451 """Get the name of the output directory for a commit number
452
453 The output directory is typically .../<branch>/<commit>.
454
455 Args:
456 commit_upto: Commit number to use (0..self.count-1)
457 """
Simon Glass5971ab52014-12-01 17:33:55 -0700458 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600459 if self.commits:
460 commit = self.commits[commit_upto]
461 subject = commit.subject.translate(trans_valid_chars)
462 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
463 self.commit_count, commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700464 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600465 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700466 if not commit_dir:
467 return self.base_dir
468 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000469
470 def GetBuildDir(self, commit_upto, target):
471 """Get the name of the build directory for a commit number
472
473 The build directory is typically .../<branch>/<commit>/<target>.
474
475 Args:
476 commit_upto: Commit number to use (0..self.count-1)
477 target: Target name
478 """
479 output_dir = self._GetOutputDir(commit_upto)
480 return os.path.join(output_dir, target)
481
482 def GetDoneFile(self, commit_upto, target):
483 """Get the name of the done file for a commit number
484
485 Args:
486 commit_upto: Commit number to use (0..self.count-1)
487 target: Target name
488 """
489 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
490
491 def GetSizesFile(self, commit_upto, target):
492 """Get the name of the sizes file for a commit number
493
494 Args:
495 commit_upto: Commit number to use (0..self.count-1)
496 target: Target name
497 """
498 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
499
500 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
501 """Get the name of the funcsizes file for a commit number and ELF file
502
503 Args:
504 commit_upto: Commit number to use (0..self.count-1)
505 target: Target name
506 elf_fname: Filename of elf image
507 """
508 return os.path.join(self.GetBuildDir(commit_upto, target),
509 '%s.sizes' % elf_fname.replace('/', '-'))
510
511 def GetObjdumpFile(self, commit_upto, target, elf_fname):
512 """Get the name of the objdump file for a commit number and ELF file
513
514 Args:
515 commit_upto: Commit number to use (0..self.count-1)
516 target: Target name
517 elf_fname: Filename of elf image
518 """
519 return os.path.join(self.GetBuildDir(commit_upto, target),
520 '%s.objdump' % elf_fname.replace('/', '-'))
521
522 def GetErrFile(self, commit_upto, target):
523 """Get the name of the err file for a commit number
524
525 Args:
526 commit_upto: Commit number to use (0..self.count-1)
527 target: Target name
528 """
529 output_dir = self.GetBuildDir(commit_upto, target)
530 return os.path.join(output_dir, 'err')
531
532 def FilterErrors(self, lines):
533 """Filter out errors in which we have no interest
534
535 We should probably use map().
536
537 Args:
538 lines: List of error lines, each a string
539 Returns:
540 New list with only interesting lines included
541 """
542 out_lines = []
543 for line in lines:
544 if not self.re_make_err.search(line):
545 out_lines.append(line)
546 return out_lines
547
548 def ReadFuncSizes(self, fname, fd):
549 """Read function sizes from the output of 'nm'
550
551 Args:
552 fd: File containing data to read
553 fname: Filename we are reading from (just for errors)
554
555 Returns:
556 Dictionary containing size of each function in bytes, indexed by
557 function name.
558 """
559 sym = {}
560 for line in fd.readlines():
561 try:
562 size, type, name = line[:-1].split()
563 except:
Simon Glass4653a882014-09-05 19:00:07 -0600564 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000565 continue
566 if type in 'tTdDbB':
567 # function names begin with '.' on 64-bit powerpc
568 if '.' in name[1:]:
569 name = 'static.' + name.split('.')[0]
570 sym[name] = sym.get(name, 0) + int(size, 16)
571 return sym
572
Simon Glass843312d2015-02-05 22:06:15 -0700573 def _ProcessConfig(self, fname):
574 """Read in a .config, autoconf.mk or autoconf.h file
575
576 This function handles all config file types. It ignores comments and
577 any #defines which don't start with CONFIG_.
578
579 Args:
580 fname: Filename to read
581
582 Returns:
583 Dictionary:
584 key: Config name (e.g. CONFIG_DM)
585 value: Config value (e.g. 1)
586 """
587 config = {}
588 if os.path.exists(fname):
589 with open(fname) as fd:
590 for line in fd:
591 line = line.strip()
592 if line.startswith('#define'):
593 values = line[8:].split(' ', 1)
594 if len(values) > 1:
595 key, value = values
596 else:
597 key = values[0]
Simon Glassb464f8e2016-11-13 14:25:53 -0700598 value = '1' if self.squash_config_y else ''
Simon Glass843312d2015-02-05 22:06:15 -0700599 if not key.startswith('CONFIG_'):
600 continue
601 elif not line or line[0] in ['#', '*', '/']:
602 continue
603 else:
604 key, value = line.split('=', 1)
Simon Glassb464f8e2016-11-13 14:25:53 -0700605 if self.squash_config_y and value == 'y':
606 value = '1'
Simon Glass843312d2015-02-05 22:06:15 -0700607 config[key] = value
608 return config
609
610 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
611 read_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000612 """Work out the outcome of a build.
613
614 Args:
615 commit_upto: Commit number to check (0..n-1)
616 target: Target board to check
617 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700618 read_config: True to read .config and autoconf.h files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000619
620 Returns:
621 Outcome object
622 """
623 done_file = self.GetDoneFile(commit_upto, target)
624 sizes_file = self.GetSizesFile(commit_upto, target)
625 sizes = {}
626 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700627 config = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000628 if os.path.exists(done_file):
629 with open(done_file, 'r') as fd:
630 return_code = int(fd.readline())
631 err_lines = []
632 err_file = self.GetErrFile(commit_upto, target)
633 if os.path.exists(err_file):
634 with open(err_file, 'r') as fd:
635 err_lines = self.FilterErrors(fd.readlines())
636
637 # Decide whether the build was ok, failed or created warnings
638 if return_code:
639 rc = OUTCOME_ERROR
640 elif len(err_lines):
641 rc = OUTCOME_WARNING
642 else:
643 rc = OUTCOME_OK
644
645 # Convert size information to our simple format
646 if os.path.exists(sizes_file):
647 with open(sizes_file, 'r') as fd:
648 for line in fd.readlines():
649 values = line.split()
650 rodata = 0
651 if len(values) > 6:
652 rodata = int(values[6], 16)
653 size_dict = {
654 'all' : int(values[0]) + int(values[1]) +
655 int(values[2]),
656 'text' : int(values[0]) - rodata,
657 'data' : int(values[1]),
658 'bss' : int(values[2]),
659 'rodata' : rodata,
660 }
661 sizes[values[5]] = size_dict
662
663 if read_func_sizes:
664 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
665 for fname in glob.glob(pattern):
666 with open(fname, 'r') as fd:
667 dict_name = os.path.basename(fname).replace('.sizes',
668 '')
669 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
670
Simon Glass843312d2015-02-05 22:06:15 -0700671 if read_config:
672 output_dir = self.GetBuildDir(commit_upto, target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700673 for name in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700674 fname = os.path.join(output_dir, name)
675 config[name] = self._ProcessConfig(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000676
Simon Glass843312d2015-02-05 22:06:15 -0700677 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000678
Simon Glass843312d2015-02-05 22:06:15 -0700679 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {})
680
681 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
682 read_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000683 """Calculate a summary of the results of building a commit.
684
685 Args:
686 board_selected: Dict containing boards to summarise
687 commit_upto: Commit number to summarize (0..self.count-1)
688 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700689 read_config: True to read .config and autoconf.h files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000690
691 Returns:
692 Tuple:
693 Dict containing boards which passed building this commit.
694 keyed by board.target
Simon Glasse30965d2014-08-28 09:43:44 -0600695 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600696 Dict keyed by error line, containing a list of the Board
697 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600698 List containing a summary of warning lines
699 Dict keyed by error line, containing a list of the Board
700 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600701 Dictionary keyed by board.target. Each value is a dictionary:
702 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700703 value is itself a dictionary:
704 key: config name
705 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000706 """
Simon Glasse30965d2014-08-28 09:43:44 -0600707 def AddLine(lines_summary, lines_boards, line, board):
708 line = line.rstrip()
709 if line in lines_boards:
710 lines_boards[line].append(board)
711 else:
712 lines_boards[line] = [board]
713 lines_summary.append(line)
714
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000715 board_dict = {}
716 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600717 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600718 warn_lines_summary = []
719 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700720 config = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000721
722 for board in boards_selected.itervalues():
723 outcome = self.GetBuildOutcome(commit_upto, board.target,
Simon Glass843312d2015-02-05 22:06:15 -0700724 read_func_sizes, read_config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000725 board_dict[board.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600726 last_func = None
727 last_was_warning = False
728 for line in outcome.err_lines:
729 if line:
730 if (self._re_function.match(line) or
731 self._re_files.match(line)):
732 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600733 else:
Simon Glasse30965d2014-08-28 09:43:44 -0600734 is_warning = self._re_warning.match(line)
735 is_note = self._re_note.match(line)
736 if is_warning or (last_was_warning and is_note):
737 if last_func:
738 AddLine(warn_lines_summary, warn_lines_boards,
739 last_func, board)
740 AddLine(warn_lines_summary, warn_lines_boards,
741 line, board)
742 else:
743 if last_func:
744 AddLine(err_lines_summary, err_lines_boards,
745 last_func, board)
746 AddLine(err_lines_summary, err_lines_boards,
747 line, board)
748 last_was_warning = is_warning
749 last_func = None
Simon Glassb464f8e2016-11-13 14:25:53 -0700750 tconfig = Config(self.config_filenames, board.target)
751 for fname in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700752 if outcome.config:
753 for key, value in outcome.config[fname].iteritems():
Simon Glass8270e3c2015-08-25 21:52:14 -0600754 tconfig.Add(fname, key, value)
755 config[board.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700756
Simon Glasse30965d2014-08-28 09:43:44 -0600757 return (board_dict, err_lines_summary, err_lines_boards,
Simon Glass843312d2015-02-05 22:06:15 -0700758 warn_lines_summary, warn_lines_boards, config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000759
760 def AddOutcome(self, board_dict, arch_list, changes, char, color):
761 """Add an output to our list of outcomes for each architecture
762
763 This simple function adds failing boards (changes) to the
764 relevant architecture string, so we can print the results out
765 sorted by architecture.
766
767 Args:
768 board_dict: Dict containing all boards
769 arch_list: Dict keyed by arch name. Value is a string containing
770 a list of board names which failed for that arch.
771 changes: List of boards to add to arch_list
772 color: terminal.Colour object
773 """
774 done_arch = {}
775 for target in changes:
776 if target in board_dict:
777 arch = board_dict[target].arch
778 else:
779 arch = 'unknown'
780 str = self.col.Color(color, ' ' + target)
781 if not arch in done_arch:
Simon Glass63c619e2015-02-05 22:06:11 -0700782 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000783 done_arch[arch] = True
784 if not arch in arch_list:
785 arch_list[arch] = str
786 else:
787 arch_list[arch] += str
788
789
790 def ColourNum(self, num):
791 color = self.col.RED if num > 0 else self.col.GREEN
792 if num == 0:
793 return '0'
794 return self.col.Color(color, str(num))
795
796 def ResetResultSummary(self, board_selected):
797 """Reset the results summary ready for use.
798
799 Set up the base board list to be all those selected, and set the
800 error lines to empty.
801
802 Following this, calls to PrintResultSummary() will use this
803 information to work out what has changed.
804
805 Args:
806 board_selected: Dict containing boards to summarise, keyed by
807 board.target
808 """
809 self._base_board_dict = {}
810 for board in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -0700811 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000812 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600813 self._base_warn_lines = []
814 self._base_err_line_boards = {}
815 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -0600816 self._base_config = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000817
818 def PrintFuncSizeDetail(self, fname, old, new):
819 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
820 delta, common = [], {}
821
822 for a in old:
823 if a in new:
824 common[a] = 1
825
826 for name in old:
827 if name not in common:
828 remove += 1
829 down += old[name]
830 delta.append([-old[name], name])
831
832 for name in new:
833 if name not in common:
834 add += 1
835 up += new[name]
836 delta.append([new[name], name])
837
838 for name in common:
839 diff = new.get(name, 0) - old.get(name, 0)
840 if diff > 0:
841 grow, up = grow + 1, up + diff
842 elif diff < 0:
843 shrink, down = shrink + 1, down - diff
844 delta.append([diff, name])
845
846 delta.sort()
847 delta.reverse()
848
849 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rinid5686a62017-05-22 13:48:52 -0400850 if max(args) == 0 and min(args) == 0:
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000851 return
852 args = [self.ColourNum(x) for x in args]
853 indent = ' ' * 15
Simon Glass4653a882014-09-05 19:00:07 -0600854 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
855 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
856 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
857 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000858 for diff, name in delta:
859 if diff:
860 color = self.col.RED if diff > 0 else self.col.GREEN
861 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
862 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4653a882014-09-05 19:00:07 -0600863 Print(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000864
865
866 def PrintSizeDetail(self, target_list, show_bloat):
867 """Show details size information for each board
868
869 Args:
870 target_list: List of targets, each a dict containing:
871 'target': Target name
872 'total_diff': Total difference in bytes across all areas
873 <part_name>: Difference for that part
874 show_bloat: Show detail for each function
875 """
876 targets_by_diff = sorted(target_list, reverse=True,
877 key=lambda x: x['_total_diff'])
878 for result in targets_by_diff:
879 printed_target = False
880 for name in sorted(result):
881 diff = result[name]
882 if name.startswith('_'):
883 continue
884 if diff != 0:
885 color = self.col.RED if diff > 0 else self.col.GREEN
886 msg = ' %s %+d' % (name, diff)
887 if not printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600888 Print('%10s %-15s:' % ('', result['_target']),
889 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000890 printed_target = True
Simon Glass4653a882014-09-05 19:00:07 -0600891 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000892 if printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600893 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000894 if show_bloat:
895 target = result['_target']
896 outcome = result['_outcome']
897 base_outcome = self._base_board_dict[target]
898 for fname in outcome.func_sizes:
899 self.PrintFuncSizeDetail(fname,
900 base_outcome.func_sizes[fname],
901 outcome.func_sizes[fname])
902
903
904 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
905 show_bloat):
906 """Print a summary of image sizes broken down by section.
907
908 The summary takes the form of one line per architecture. The
909 line contains deltas for each of the sections (+ means the section
910 got bigger, - means smaller). The nunmbers are the average number
911 of bytes that a board in this section increased by.
912
913 For example:
914 powerpc: (622 boards) text -0.0
915 arm: (285 boards) text -0.0
916 nds32: (3 boards) text -8.0
917
918 Args:
919 board_selected: Dict containing boards to summarise, keyed by
920 board.target
921 board_dict: Dict containing boards for which we built this
922 commit, keyed by board.target. The value is an Outcome object.
923 show_detail: Show detail for each board
924 show_bloat: Show detail for each function
925 """
926 arch_list = {}
927 arch_count = {}
928
929 # Calculate changes in size for different image parts
930 # The previous sizes are in Board.sizes, for each board
931 for target in board_dict:
932 if target not in board_selected:
933 continue
934 base_sizes = self._base_board_dict[target].sizes
935 outcome = board_dict[target]
936 sizes = outcome.sizes
937
938 # Loop through the list of images, creating a dict of size
939 # changes for each image/part. We end up with something like
940 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
941 # which means that U-Boot data increased by 5 bytes and SPL
942 # text decreased by 4.
943 err = {'_target' : target}
944 for image in sizes:
945 if image in base_sizes:
946 base_image = base_sizes[image]
947 # Loop through the text, data, bss parts
948 for part in sorted(sizes[image]):
949 diff = sizes[image][part] - base_image[part]
950 col = None
951 if diff:
952 if image == 'u-boot':
953 name = part
954 else:
955 name = image + ':' + part
956 err[name] = diff
957 arch = board_selected[target].arch
958 if not arch in arch_count:
959 arch_count[arch] = 1
960 else:
961 arch_count[arch] += 1
962 if not sizes:
963 pass # Only add to our list when we have some stats
964 elif not arch in arch_list:
965 arch_list[arch] = [err]
966 else:
967 arch_list[arch].append(err)
968
969 # We now have a list of image size changes sorted by arch
970 # Print out a summary of these
971 for arch, target_list in arch_list.iteritems():
972 # Get total difference for each type
973 totals = {}
974 for result in target_list:
975 total = 0
976 for name, diff in result.iteritems():
977 if name.startswith('_'):
978 continue
979 total += diff
980 if name in totals:
981 totals[name] += diff
982 else:
983 totals[name] = diff
984 result['_total_diff'] = total
985 result['_outcome'] = board_dict[result['_target']]
986
987 count = len(target_list)
988 printed_arch = False
989 for name in sorted(totals):
990 diff = totals[name]
991 if diff:
992 # Display the average difference in this name for this
993 # architecture
994 avg_diff = float(diff) / count
995 color = self.col.RED if avg_diff > 0 else self.col.GREEN
996 msg = ' %s %+1.1f' % (name, avg_diff)
997 if not printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -0600998 Print('%10s: (for %d/%d boards)' % (arch, count,
999 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001000 printed_arch = True
Simon Glass4653a882014-09-05 19:00:07 -06001001 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001002
1003 if printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001004 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001005 if show_detail:
1006 self.PrintSizeDetail(target_list, show_bloat)
1007
1008
1009 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001010 err_line_boards, warn_lines, warn_line_boards,
Simon Glass843312d2015-02-05 22:06:15 -07001011 config, show_sizes, show_detail, show_bloat,
1012 show_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001013 """Compare results with the base results and display delta.
1014
1015 Only boards mentioned in board_selected will be considered. This
1016 function is intended to be called repeatedly with the results of
1017 each commit. It therefore shows a 'diff' between what it saw in
1018 the last call and what it sees now.
1019
1020 Args:
1021 board_selected: Dict containing boards to summarise, keyed by
1022 board.target
1023 board_dict: Dict containing boards for which we built this
1024 commit, keyed by board.target. The value is an Outcome object.
1025 err_lines: A list of errors for this commit, or [] if there is
1026 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001027 err_line_boards: Dict keyed by error line, containing a list of
1028 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001029 warn_lines: A list of warnings for this commit, or [] if there is
1030 none, or we don't want to print errors
1031 warn_line_boards: Dict keyed by warning line, containing a list of
1032 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001033 config: Dictionary keyed by filename - e.g. '.config'. Each
1034 value is itself a dictionary:
1035 key: config name
1036 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001037 show_sizes: Show image size deltas
1038 show_detail: Show detail for each board
1039 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001040 show_config: Show config changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001041 """
Simon Glasse30965d2014-08-28 09:43:44 -06001042 def _BoardList(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001043 """Helper function to get a line of boards containing a line
1044
1045 Args:
1046 line: Error line to search for
1047 Return:
1048 String containing a list of boards with that error line, or
1049 '' if the user has not requested such a list
1050 """
1051 if self._list_error_boards:
1052 names = []
Simon Glasse30965d2014-08-28 09:43:44 -06001053 for board in line_boards[line]:
Simon Glassf66153b2014-10-16 01:05:55 -06001054 if not board.target in names:
1055 names.append(board.target)
Simon Glassed966652014-08-28 09:43:43 -06001056 names_str = '(%s) ' % ','.join(names)
1057 else:
1058 names_str = ''
1059 return names_str
1060
Simon Glasse30965d2014-08-28 09:43:44 -06001061 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1062 char):
1063 better_lines = []
1064 worse_lines = []
1065 for line in lines:
1066 if line not in base_lines:
1067 worse_lines.append(char + '+' +
1068 _BoardList(line, line_boards) + line)
1069 for line in base_lines:
1070 if line not in lines:
1071 better_lines.append(char + '-' +
1072 _BoardList(line, base_line_boards) + line)
1073 return better_lines, worse_lines
1074
Simon Glass843312d2015-02-05 22:06:15 -07001075 def _CalcConfig(delta, name, config):
1076 """Calculate configuration changes
1077
1078 Args:
1079 delta: Type of the delta, e.g. '+'
1080 name: name of the file which changed (e.g. .config)
1081 config: configuration change dictionary
1082 key: config name
1083 value: config value
1084 Returns:
1085 String containing the configuration changes which can be
1086 printed
1087 """
1088 out = ''
1089 for key in sorted(config.keys()):
1090 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001091 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001092
Simon Glass8270e3c2015-08-25 21:52:14 -06001093 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1094 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001095
1096 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001097 lines: list to add to
1098 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001099 config_plus: configurations added, dictionary
1100 key: config name
1101 value: config value
1102 config_minus: configurations removed, dictionary
1103 key: config name
1104 value: config value
1105 config_change: configurations changed, dictionary
1106 key: config name
1107 value: config value
1108 """
1109 if config_plus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001110 lines.append(_CalcConfig('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001111 if config_minus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001112 lines.append(_CalcConfig('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001113 if config_change:
Simon Glass8270e3c2015-08-25 21:52:14 -06001114 lines.append(_CalcConfig('c', name, config_change))
1115
1116 def _OutputConfigInfo(lines):
1117 for line in lines:
1118 if not line:
1119 continue
1120 if line[0] == '+':
1121 col = self.col.GREEN
1122 elif line[0] == '-':
1123 col = self.col.RED
1124 elif line[0] == 'c':
1125 col = self.col.YELLOW
1126 Print(' ' + line, newline=True, colour=col)
1127
Simon Glass843312d2015-02-05 22:06:15 -07001128
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001129 better = [] # List of boards fixed since last commit
1130 worse = [] # List of new broken boards since last commit
1131 new = [] # List of boards that didn't exist last time
1132 unknown = [] # List of boards that were not built
1133
1134 for target in board_dict:
1135 if target not in board_selected:
1136 continue
1137
1138 # If the board was built last time, add its outcome to a list
1139 if target in self._base_board_dict:
1140 base_outcome = self._base_board_dict[target].rc
1141 outcome = board_dict[target]
1142 if outcome.rc == OUTCOME_UNKNOWN:
1143 unknown.append(target)
1144 elif outcome.rc < base_outcome:
1145 better.append(target)
1146 elif outcome.rc > base_outcome:
1147 worse.append(target)
1148 else:
1149 new.append(target)
1150
1151 # Get a list of errors that have appeared, and disappeared
Simon Glasse30965d2014-08-28 09:43:44 -06001152 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1153 self._base_err_line_boards, err_lines, err_line_boards, '')
1154 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1155 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001156
1157 # Display results by arch
Simon Glasse30965d2014-08-28 09:43:44 -06001158 if (better or worse or unknown or new or worse_err or better_err
1159 or worse_warn or better_warn):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001160 arch_list = {}
1161 self.AddOutcome(board_selected, arch_list, better, '',
1162 self.col.GREEN)
1163 self.AddOutcome(board_selected, arch_list, worse, '+',
1164 self.col.RED)
1165 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
1166 if self._show_unknown:
1167 self.AddOutcome(board_selected, arch_list, unknown, '?',
1168 self.col.MAGENTA)
1169 for arch, target_list in arch_list.iteritems():
Simon Glass4653a882014-09-05 19:00:07 -06001170 Print('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001171 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001172 if better_err:
Simon Glass4653a882014-09-05 19:00:07 -06001173 Print('\n'.join(better_err), colour=self.col.GREEN)
Simon Glass28370c12014-08-09 15:33:06 -06001174 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001175 if worse_err:
Simon Glass4653a882014-09-05 19:00:07 -06001176 Print('\n'.join(worse_err), colour=self.col.RED)
Simon Glass28370c12014-08-09 15:33:06 -06001177 self._error_lines += 1
Simon Glasse30965d2014-08-28 09:43:44 -06001178 if better_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001179 Print('\n'.join(better_warn), colour=self.col.CYAN)
Simon Glasse30965d2014-08-28 09:43:44 -06001180 self._error_lines += 1
1181 if worse_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001182 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
Simon Glasse30965d2014-08-28 09:43:44 -06001183 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001184
1185 if show_sizes:
1186 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1187 show_bloat)
1188
Simon Glass8270e3c2015-08-25 21:52:14 -06001189 if show_config and self._base_config:
1190 summary = {}
1191 arch_config_plus = {}
1192 arch_config_minus = {}
1193 arch_config_change = {}
1194 arch_list = []
1195
1196 for target in board_dict:
1197 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001198 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001199 arch = board_selected[target].arch
1200 if arch not in arch_list:
1201 arch_list.append(arch)
1202
1203 for arch in arch_list:
1204 arch_config_plus[arch] = {}
1205 arch_config_minus[arch] = {}
1206 arch_config_change[arch] = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001207 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001208 arch_config_plus[arch][name] = {}
1209 arch_config_minus[arch][name] = {}
1210 arch_config_change[arch][name] = {}
1211
1212 for target in board_dict:
1213 if target not in board_selected:
1214 continue
1215
1216 arch = board_selected[target].arch
1217
1218 all_config_plus = {}
1219 all_config_minus = {}
1220 all_config_change = {}
1221 tbase = self._base_config[target]
1222 tconfig = config[target]
1223 lines = []
Simon Glassb464f8e2016-11-13 14:25:53 -07001224 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001225 if not tconfig.config[name]:
1226 continue
1227 config_plus = {}
1228 config_minus = {}
1229 config_change = {}
1230 base = tbase.config[name]
1231 for key, value in tconfig.config[name].iteritems():
1232 if key not in base:
1233 config_plus[key] = value
1234 all_config_plus[key] = value
1235 for key, value in base.iteritems():
1236 if key not in tconfig.config[name]:
1237 config_minus[key] = value
1238 all_config_minus[key] = value
1239 for key, value in base.iteritems():
1240 new_value = tconfig.config.get(key)
1241 if new_value and value != new_value:
1242 desc = '%s -> %s' % (value, new_value)
1243 config_change[key] = desc
1244 all_config_change[key] = desc
1245
1246 arch_config_plus[arch][name].update(config_plus)
1247 arch_config_minus[arch][name].update(config_minus)
1248 arch_config_change[arch][name].update(config_change)
1249
1250 _AddConfig(lines, name, config_plus, config_minus,
1251 config_change)
1252 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1253 all_config_change)
1254 summary[target] = '\n'.join(lines)
1255
1256 lines_by_target = {}
1257 for target, lines in summary.iteritems():
1258 if lines in lines_by_target:
1259 lines_by_target[lines].append(target)
1260 else:
1261 lines_by_target[lines] = [target]
1262
1263 for arch in arch_list:
1264 lines = []
1265 all_plus = {}
1266 all_minus = {}
1267 all_change = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001268 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001269 all_plus.update(arch_config_plus[arch][name])
1270 all_minus.update(arch_config_minus[arch][name])
1271 all_change.update(arch_config_change[arch][name])
1272 _AddConfig(lines, name, arch_config_plus[arch][name],
1273 arch_config_minus[arch][name],
1274 arch_config_change[arch][name])
1275 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1276 #arch_summary[target] = '\n'.join(lines)
1277 if lines:
1278 Print('%s:' % arch)
1279 _OutputConfigInfo(lines)
1280
1281 for lines, targets in lines_by_target.iteritems():
1282 if not lines:
1283 continue
1284 Print('%s :' % ' '.join(sorted(targets)))
1285 _OutputConfigInfo(lines.split('\n'))
1286
Simon Glass843312d2015-02-05 22:06:15 -07001287
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001288 # Save our updated information for the next call to this function
1289 self._base_board_dict = board_dict
1290 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001291 self._base_warn_lines = warn_lines
1292 self._base_err_line_boards = err_line_boards
1293 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001294 self._base_config = config
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001295
1296 # Get a list of boards that did not get built, if needed
1297 not_built = []
1298 for board in board_selected:
1299 if not board in board_dict:
1300 not_built.append(board)
1301 if not_built:
Simon Glass4653a882014-09-05 19:00:07 -06001302 Print("Boards not built (%d): %s" % (len(not_built),
1303 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001304
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001305 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001306 (board_dict, err_lines, err_line_boards, warn_lines,
Simon Glass843312d2015-02-05 22:06:15 -07001307 warn_line_boards, config) = self.GetResultSummary(
Simon Glassed966652014-08-28 09:43:43 -06001308 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001309 read_func_sizes=self._show_bloat,
1310 read_config=self._show_config)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001311 if commits:
1312 msg = '%02d: %s' % (commit_upto + 1,
1313 commits[commit_upto].subject)
Simon Glass4653a882014-09-05 19:00:07 -06001314 Print(msg, colour=self.col.BLUE)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001315 self.PrintResultSummary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001316 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001317 warn_lines if self._show_errors else [], warn_line_boards,
Simon Glass843312d2015-02-05 22:06:15 -07001318 config, self._show_sizes, self._show_detail,
1319 self._show_bloat, self._show_config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001320
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001321 def ShowSummary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001322 """Show a build summary for U-Boot for a given board list.
1323
1324 Reset the result summary, then repeatedly call GetResultSummary on
1325 each commit's results, then display the differences we see.
1326
1327 Args:
1328 commit: Commit objects to summarise
1329 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001330 """
Simon Glassfea58582014-08-09 15:32:59 -06001331 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001332 self.commits = commits
1333 self.ResetResultSummary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001334 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001335
1336 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001337 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001338 if not self._error_lines:
Simon Glass4653a882014-09-05 19:00:07 -06001339 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001340
1341
1342 def SetupBuild(self, board_selected, commits):
1343 """Set up ready to start a build.
1344
1345 Args:
1346 board_selected: Selected boards to build
1347 commits: Selected commits to build
1348 """
1349 # First work out how many commits we will build
Simon Glassfea58582014-08-09 15:32:59 -06001350 count = (self.commit_count + self._step - 1) / self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001351 self.count = len(board_selected) * count
1352 self.upto = self.warned = self.fail = 0
1353 self._timestamps = collections.deque()
1354
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001355 def GetThreadDir(self, thread_num):
1356 """Get the directory path to the working dir for a thread.
1357
1358 Args:
1359 thread_num: Number of thread to check.
1360 """
1361 return os.path.join(self._working_dir, '%02d' % thread_num)
1362
Simon Glassfea58582014-08-09 15:32:59 -06001363 def _PrepareThread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001364 """Prepare the working directory for a thread.
1365
1366 This clones or fetches the repo into the thread's work directory.
1367
1368 Args:
1369 thread_num: Thread number (0, 1, ...)
Simon Glassfea58582014-08-09 15:32:59 -06001370 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001371 """
1372 thread_dir = self.GetThreadDir(thread_num)
Simon Glass190064b2014-08-09 15:33:00 -06001373 builderthread.Mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001374 git_dir = os.path.join(thread_dir, '.git')
1375
1376 # Clone the repo if it doesn't already exist
1377 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1378 # we have a private index but uses the origin repo's contents?
Simon Glassfea58582014-08-09 15:32:59 -06001379 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001380 src_dir = os.path.abspath(self.git_dir)
1381 if os.path.exists(git_dir):
1382 gitutil.Fetch(git_dir, thread_dir)
1383 else:
Simon Glass21f0eb32016-09-18 16:48:31 -06001384 Print('\rCloning repo for thread %d' % thread_num,
1385 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001386 gitutil.Clone(src_dir, thread_dir)
Simon Glass21f0eb32016-09-18 16:48:31 -06001387 Print('\r%s\r' % (' ' * 30), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001388
Simon Glassfea58582014-08-09 15:32:59 -06001389 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001390 """Prepare the working directory for use.
1391
1392 Set up the git repo for each thread.
1393
1394 Args:
1395 max_threads: Maximum number of threads we expect to need.
Simon Glassfea58582014-08-09 15:32:59 -06001396 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001397 """
Simon Glass190064b2014-08-09 15:33:00 -06001398 builderthread.Mkdir(self._working_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001399 for thread in range(max_threads):
Simon Glassfea58582014-08-09 15:32:59 -06001400 self._PrepareThread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001401
1402 def _PrepareOutputSpace(self):
1403 """Get the output directories ready to receive files.
1404
1405 We delete any output directories which look like ones we need to
1406 create. Having left over directories is confusing when the user wants
1407 to check the output manually.
1408 """
Simon Glass1a915672014-12-01 17:33:53 -07001409 if not self.commits:
1410 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001411 dir_list = []
1412 for commit_upto in range(self.commit_count):
1413 dir_list.append(self._GetOutputDir(commit_upto))
1414
Simon Glassb222abe2016-09-18 16:48:32 -06001415 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001416 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1417 if dirname not in dir_list:
Simon Glassb222abe2016-09-18 16:48:32 -06001418 to_remove.append(dirname)
1419 if to_remove:
1420 Print('Removing %d old build directories' % len(to_remove),
1421 newline=False)
1422 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001423 shutil.rmtree(dirname)
1424
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001425 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001426 """Build all commits for a list of boards
1427
1428 Args:
1429 commits: List of commits to be build, each a Commit object
1430 boards_selected: Dict of selected boards, key is target name,
1431 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001432 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001433 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001434 Returns:
1435 Tuple containing:
1436 - number of boards that failed to build
1437 - number of boards that issued warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001438 """
Simon Glassfea58582014-08-09 15:32:59 -06001439 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001440 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001441 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001442
1443 self.ResetResultSummary(board_selected)
Thierry Redingf3d015c2014-08-19 10:22:39 +02001444 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassfea58582014-08-09 15:32:59 -06001445 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1446 commits is not None)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001447 self._PrepareOutputSpace()
Simon Glass745b3952016-09-18 16:48:33 -06001448 Print('\rStarting build...', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001449 self.SetupBuild(board_selected, commits)
1450 self.ProcessResult(None)
1451
1452 # Create jobs to build all commits for each board
1453 for brd in board_selected.itervalues():
Simon Glass190064b2014-08-09 15:33:00 -06001454 job = builderthread.BuilderJob()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001455 job.board = brd
1456 job.commits = commits
1457 job.keep_outputs = keep_outputs
1458 job.step = self._step
1459 self.queue.put(job)
1460
Simon Glassd436e382016-09-18 16:48:35 -06001461 term = threading.Thread(target=self.queue.join)
1462 term.setDaemon(True)
1463 term.start()
1464 while term.isAlive():
1465 term.join(100)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001466
1467 # Wait until we have processed all output
1468 self.out_queue.join()
Simon Glass4653a882014-09-05 19:00:07 -06001469 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001470 self.ClearLine(0)
Simon Glass2c3deb92014-08-28 09:43:39 -06001471 return (self.fail, self.warned)