blob: bc0819784f86147e2cecb2f3be474ee1ad8d3400 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00004
5import multiprocessing
6import os
Simon Glass883a3212014-09-05 19:00:18 -06007import shutil
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008import sys
9
10import board
11import bsettings
12from builder import Builder
13import gitutil
14import patchstream
15import terminal
Simon Glassd4144e42014-09-05 19:00:13 -060016from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000017import toolchain
Masahiro Yamada99796922014-07-22 11:19:09 +090018import command
Masahiro Yamada73f30b92014-07-30 14:08:22 +090019import subprocess
Simon Glassfc3fe1c2013-04-03 11:07:16 +000020
21def GetPlural(count):
22 """Returns a plural 's' if count is not 1"""
23 return 's' if count != 1 else ''
24
Simon Glassfea58582014-08-09 15:32:59 -060025def GetActionSummary(is_summary, commits, selected, options):
Simon Glassfc3fe1c2013-04-03 11:07:16 +000026 """Return a string summarising the intended action.
27
28 Returns:
29 Summary string.
30 """
Simon Glassfea58582014-08-09 15:32:59 -060031 if commits:
32 count = len(commits)
33 count = (count + options.step - 1) / options.step
34 commit_str = '%d commit%s' % (count, GetPlural(count))
35 else:
36 commit_str = 'current source'
37 str = '%s %s for %d boards' % (
38 'Summary of' if is_summary else 'Building', commit_str,
Simon Glassfc3fe1c2013-04-03 11:07:16 +000039 len(selected))
40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
42 return str
43
44def ShowActions(series, why_selected, boards_selected, builder, options):
45 """Display a list of actions that we would take, if not a dry run.
46
47 Args:
48 series: Series object
49 why_selected: Dictionary where each key is a buildman argument
Simon Glass8d7523c2017-01-23 05:38:56 -070050 provided by the user, and the value is the list of boards
51 brought in by that argument. For example, 'arm' might bring
52 in 400 boards, so in this case the key would be 'arm' and
Simon Glassfc3fe1c2013-04-03 11:07:16 +000053 the value would be a list of board names.
54 boards_selected: Dict of selected boards, key is target name,
55 value is Board object
56 builder: The builder that will be used to build the commits
57 options: Command line options object
58 """
59 col = terminal.Color()
60 print 'Dry run, so not doing much. But I would do this:'
61 print
Simon Glassfea58582014-08-09 15:32:59 -060062 if series:
63 commits = series.commits
64 else:
65 commits = None
66 print GetActionSummary(False, commits, boards_selected,
Simon Glassfc3fe1c2013-04-03 11:07:16 +000067 options)
68 print 'Build directory: %s' % builder.base_dir
Simon Glassfea58582014-08-09 15:32:59 -060069 if commits:
70 for upto in range(0, len(series.commits), options.step):
71 commit = series.commits[upto]
Simon Glass1ddda1b2014-10-15 02:27:00 -060072 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
Simon Glassfea58582014-08-09 15:32:59 -060073 print commit.subject
Simon Glassfc3fe1c2013-04-03 11:07:16 +000074 print
75 for arg in why_selected:
76 if arg != 'all':
Simon Glass8d7523c2017-01-23 05:38:56 -070077 print arg, ': %d boards' % len(why_selected[arg])
78 if options.verbose:
79 print ' %s' % ' '.join(why_selected[arg])
Simon Glassfc3fe1c2013-04-03 11:07:16 +000080 print ('Total boards to build for each commit: %d\n' %
Simon Glass8d7523c2017-01-23 05:38:56 -070081 len(why_selected['all']))
Simon Glassfc3fe1c2013-04-03 11:07:16 +000082
Lothar Waßmann409fc022018-04-08 05:14:11 -060083def CheckOutputDir(output_dir):
84 """Make sure that the output directory is not within the current directory
85
86 If we try to use an output directory which is within the current directory
87 (which is assumed to hold the U-Boot source) we may end up deleting the
88 U-Boot source code. Detect this and print an error in this case.
89
90 Args:
91 output_dir: Output directory path to check
92 """
93 path = os.path.realpath(output_dir)
94 cwd_path = os.path.realpath('.')
95 while True:
96 if os.path.realpath(path) == cwd_path:
97 Print("Cannot use output directory '%s' since it is within the current directtory '%s'" %
98 (path, cwd_path))
99 sys.exit(1)
100 parent = os.path.dirname(path)
101 if parent == path:
102 break
103 path = parent
104
Simon Glass883a3212014-09-05 19:00:18 -0600105def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
106 clean_dir=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000107 """The main control code for buildman
108
109 Args:
110 options: Command line options object
111 args: Command line arguments (list of strings)
Simon Glassd4144e42014-09-05 19:00:13 -0600112 toolchains: Toolchains to use - this should be a Toolchains()
113 object. If None, then it will be created and scanned
114 make_func: Make function to use for the builder. This is called
115 to execute 'make'. If this is None, the normal function
116 will be used, which calls the 'make' tool with suitable
117 arguments. This setting is useful for tests.
Simon Glass823e60b2014-09-05 19:00:16 -0600118 board: Boards() object to use, containing a list of available
119 boards. If this is None it will be created and scanned.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000120 """
Simon Glass883a3212014-09-05 19:00:18 -0600121 global builder
122
Simon Glass48ba5852014-09-05 19:00:11 -0600123 if options.full_help:
124 pager = os.getenv('PAGER')
125 if not pager:
126 pager = 'more'
Simon Glass2bdeade2016-03-06 19:45:34 -0700127 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
128 'README')
Simon Glass48ba5852014-09-05 19:00:11 -0600129 command.Run(pager, fname)
130 return 0
131
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000132 gitutil.Setup()
Simon Glass713bea32016-07-27 20:33:02 -0600133 col = terminal.Color()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000134
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000135 options.git_dir = os.path.join(options.git, '.git')
136
Simon Glass7e92e462016-07-27 20:33:04 -0600137 no_toolchains = toolchains is None
138 if no_toolchains:
Simon Glassd4144e42014-09-05 19:00:13 -0600139 toolchains = toolchain.Toolchains()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000140
Simon Glass827e37b2014-12-01 17:34:06 -0700141 if options.fetch_arch:
142 if options.fetch_arch == 'list':
143 sorted_list = toolchains.ListArchs()
Simon Glass713bea32016-07-27 20:33:02 -0600144 print col.Color(col.BLUE, 'Available architectures: %s\n' %
145 ' '.join(sorted_list))
Simon Glass827e37b2014-12-01 17:34:06 -0700146 return 0
147 else:
148 fetch_arch = options.fetch_arch
149 if fetch_arch == 'all':
150 fetch_arch = ','.join(toolchains.ListArchs())
Simon Glass713bea32016-07-27 20:33:02 -0600151 print col.Color(col.CYAN, '\nDownloading toolchains: %s' %
152 fetch_arch)
Simon Glass827e37b2014-12-01 17:34:06 -0700153 for arch in fetch_arch.split(','):
Simon Glass713bea32016-07-27 20:33:02 -0600154 print
Simon Glass827e37b2014-12-01 17:34:06 -0700155 ret = toolchains.FetchAndInstall(arch)
156 if ret:
157 return ret
158 return 0
159
Simon Glass7e92e462016-07-27 20:33:04 -0600160 if no_toolchains:
161 toolchains.GetSettings()
162 toolchains.Scan(options.list_tool_chains)
163 if options.list_tool_chains:
164 toolchains.List()
165 print
166 return 0
167
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000168 # Work out how many commits to build. We want to build everything on the
169 # branch. We also build the upstream commit as a control so we can see
170 # problems introduced by the first commit on the branch.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000171 count = options.count
Simon Glass5abab202014-12-01 17:33:57 -0700172 has_range = options.branch and '..' in options.branch
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000173 if count == -1:
174 if not options.branch:
Simon Glassfea58582014-08-09 15:32:59 -0600175 count = 1
176 else:
Simon Glass5abab202014-12-01 17:33:57 -0700177 if has_range:
178 count, msg = gitutil.CountCommitsInRange(options.git_dir,
179 options.branch)
180 else:
181 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
182 options.branch)
Simon Glassfea58582014-08-09 15:32:59 -0600183 if count is None:
Simon Glass2a9e2c62014-12-01 17:33:54 -0700184 sys.exit(col.Color(col.RED, msg))
Simon Glass5abab202014-12-01 17:33:57 -0700185 elif count == 0:
186 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
187 options.branch))
Simon Glass2a9e2c62014-12-01 17:33:54 -0700188 if msg:
189 print col.Color(col.YELLOW, msg)
Simon Glassfea58582014-08-09 15:32:59 -0600190 count += 1 # Build upstream commit also
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000191
192 if not count:
193 str = ("No commits found to process in branch '%s': "
194 "set branch's upstream or use -c flag" % options.branch)
Masahiro Yamada31e21412014-08-16 00:59:26 +0900195 sys.exit(col.Color(col.RED, str))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000196
197 # Work out what subset of the boards we are building
Simon Glass823e60b2014-09-05 19:00:16 -0600198 if not boards:
199 board_file = os.path.join(options.git, 'boards.cfg')
200 status = subprocess.call([os.path.join(options.git,
201 'tools/genboardscfg.py')])
202 if status != 0:
203 sys.exit("Failed to generate boards.cfg")
Masahiro Yamada73f30b92014-07-30 14:08:22 +0900204
Simon Glass823e60b2014-09-05 19:00:16 -0600205 boards = board.Boards()
206 boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
Simon Glass3cf4ae62014-08-28 09:43:41 -0600207
208 exclude = []
209 if options.exclude:
210 for arg in options.exclude:
211 exclude += arg.split(',')
212
213 why_selected = boards.SelectBoards(args, exclude)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000214 selected = boards.GetSelected()
215 if not len(selected):
Masahiro Yamada31e21412014-08-16 00:59:26 +0900216 sys.exit(col.Color(col.RED, 'No matching boards found'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000217
218 # Read the metadata from the commits. First look at the upstream commit,
219 # then the ones in the branch. We would like to do something like
220 # upstream/master~..branch but that isn't possible if upstream/master is
221 # a merge commit (it will list all the commits that form part of the
222 # merge)
Simon Glass950a2312014-09-05 19:00:23 -0600223 # Conflicting tags are not a problem for buildman, since it does not use
224 # them. For example, Series-version is not useful for buildman. On the
225 # other hand conflicting tags will cause an error. So allow later tags
226 # to overwrite earlier ones by setting allow_overwrite=True
Simon Glassfea58582014-08-09 15:32:59 -0600227 if options.branch:
Simon Glass3b74ba52014-08-09 15:33:09 -0600228 if count == -1:
Simon Glass5abab202014-12-01 17:33:57 -0700229 if has_range:
230 range_expr = options.branch
231 else:
232 range_expr = gitutil.GetRangeInBranch(options.git_dir,
233 options.branch)
Simon Glass3b74ba52014-08-09 15:33:09 -0600234 upstream_commit = gitutil.GetUpstream(options.git_dir,
235 options.branch)
236 series = patchstream.GetMetaDataForList(upstream_commit,
Simon Glass950a2312014-09-05 19:00:23 -0600237 options.git_dir, 1, series=None, allow_overwrite=True)
Simon Glassfea58582014-08-09 15:32:59 -0600238
Simon Glass3b74ba52014-08-09 15:33:09 -0600239 series = patchstream.GetMetaDataForList(range_expr,
Simon Glass950a2312014-09-05 19:00:23 -0600240 options.git_dir, None, series, allow_overwrite=True)
Simon Glass3b74ba52014-08-09 15:33:09 -0600241 else:
242 # Honour the count
243 series = patchstream.GetMetaDataForList(options.branch,
Simon Glass950a2312014-09-05 19:00:23 -0600244 options.git_dir, count, series=None, allow_overwrite=True)
Simon Glassfea58582014-08-09 15:32:59 -0600245 else:
246 series = None
Simon Glass8d7523c2017-01-23 05:38:56 -0700247 if not options.dry_run:
248 options.verbose = True
249 if not options.summary:
250 options.show_errors = True
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000251
252 # By default we have one thread per CPU. But if there are not enough jobs
253 # we can have fewer threads and use a high '-j' value for make.
254 if not options.threads:
255 options.threads = min(multiprocessing.cpu_count(), len(selected))
256 if not options.jobs:
257 options.jobs = max(1, (multiprocessing.cpu_count() +
258 len(selected) - 1) / len(selected))
259
260 if not options.step:
261 options.step = len(series.commits) - 1
262
Masahiro Yamada99796922014-07-22 11:19:09 +0900263 gnu_make = command.Output(os.path.join(options.git,
Simon Glass785f1542016-07-25 18:59:00 -0600264 'scripts/show-gnu-make'), raise_on_error=False).rstrip()
Masahiro Yamada99796922014-07-22 11:19:09 +0900265 if not gnu_make:
Masahiro Yamada31e21412014-08-16 00:59:26 +0900266 sys.exit('GNU Make not found')
Masahiro Yamada99796922014-07-22 11:19:09 +0900267
Simon Glass05c96b12014-12-01 17:33:52 -0700268 # Create a new builder with the selected options.
269 output_dir = options.output_dir
Simon Glassfea58582014-08-09 15:32:59 -0600270 if options.branch:
Simon Glassf7582ce2014-09-05 19:00:22 -0600271 dirname = options.branch.replace('/', '_')
Simon Glass5971ab52014-12-01 17:33:55 -0700272 # As a special case allow the board directory to be placed in the
273 # output directory itself rather than any subdirectory.
274 if not options.no_subdirs:
275 output_dir = os.path.join(options.output_dir, dirname)
Lothar Waßmann409fc022018-04-08 05:14:11 -0600276 if clean_dir and os.path.exists(output_dir):
277 shutil.rmtree(output_dir)
278 CheckOutputDir(output_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000279 builder = Builder(toolchains, output_dir, options.git_dir,
Masahiro Yamada99796922014-07-22 11:19:09 +0900280 options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
Simon Glass5971ab52014-12-01 17:33:55 -0700281 show_unknown=options.show_unknown, step=options.step,
Simon Glassd2ce6582014-12-01 17:34:07 -0700282 no_subdirs=options.no_subdirs, full_path=options.full_path,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600283 verbose_build=options.verbose_build,
284 incremental=options.incremental,
Simon Glassb50113f2016-11-13 14:25:51 -0700285 per_board_out_dir=options.per_board_out_dir,
Simon Glassb464f8e2016-11-13 14:25:53 -0700286 config_only=options.config_only,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100287 squash_config_y=not options.preserve_config_y,
288 warnings_as_errors=options.warnings_as_errors)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000289 builder.force_config_on_failure = not options.quick
Simon Glassd4144e42014-09-05 19:00:13 -0600290 if make_func:
291 builder.do_make = make_func
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000292
293 # For a dry run, just show our actions as a sanity check
294 if options.dry_run:
295 ShowActions(series, why_selected, selected, builder, options)
296 else:
297 builder.force_build = options.force_build
Simon Glass4266dc22014-07-13 12:22:31 -0600298 builder.force_build_failures = options.force_build_failures
Simon Glass97e91522014-07-14 17:51:02 -0600299 builder.force_reconfig = options.force_reconfig
Simon Glass189a4962014-07-14 17:51:03 -0600300 builder.in_tree = options.in_tree
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000301
302 # Work out which boards to build
303 board_selected = boards.GetSelectedDict()
304
Simon Glassfea58582014-08-09 15:32:59 -0600305 if series:
306 commits = series.commits
Simon Glass883a3212014-09-05 19:00:18 -0600307 # Number the commits for test purposes
308 for commit in range(len(commits)):
309 commits[commit].sequence = commit
Simon Glassfea58582014-08-09 15:32:59 -0600310 else:
311 commits = None
312
Simon Glassd4144e42014-09-05 19:00:13 -0600313 Print(GetActionSummary(options.summary, commits, board_selected,
314 options))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000315
Simon Glass7798e222014-09-14 20:23:16 -0600316 # We can't show function sizes without board details at present
317 if options.show_bloat:
318 options.show_detail = True
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600319 builder.SetDisplayOptions(options.show_errors, options.show_sizes,
Simon Glassed966652014-08-28 09:43:43 -0600320 options.show_detail, options.show_bloat,
Simon Glass843312d2015-02-05 22:06:15 -0700321 options.list_error_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000322 options.show_config,
323 options.show_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000324 if options.summary:
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600325 builder.ShowSummary(commits, board_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000326 else:
Simon Glass2c3deb92014-08-28 09:43:39 -0600327 fail, warned = builder.BuildBoards(commits, board_selected,
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600328 options.keep_outputs, options.verbose)
Simon Glass2c3deb92014-08-28 09:43:39 -0600329 if fail:
330 return 128
331 elif warned:
332 return 129
333 return 0