Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. |
| 2 | # |
Wolfgang Denk | 1a45966 | 2013-07-08 09:37:19 +0200 | [diff] [blame] | 3 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 4 | # |
| 5 | |
| 6 | import multiprocessing |
| 7 | import os |
| 8 | import sys |
| 9 | |
| 10 | import board |
| 11 | import bsettings |
| 12 | from builder import Builder |
| 13 | import gitutil |
| 14 | import patchstream |
| 15 | import terminal |
| 16 | import toolchain |
Masahiro Yamada | 9979692 | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 17 | import command |
Masahiro Yamada | 73f30b9 | 2014-07-30 14:08:22 +0900 | [diff] [blame] | 18 | import subprocess |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 19 | |
| 20 | def GetPlural(count): |
| 21 | """Returns a plural 's' if count is not 1""" |
| 22 | return 's' if count != 1 else '' |
| 23 | |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 24 | def GetActionSummary(is_summary, commits, selected, options): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 25 | """Return a string summarising the intended action. |
| 26 | |
| 27 | Returns: |
| 28 | Summary string. |
| 29 | """ |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 30 | if commits: |
| 31 | count = len(commits) |
| 32 | count = (count + options.step - 1) / options.step |
| 33 | commit_str = '%d commit%s' % (count, GetPlural(count)) |
| 34 | else: |
| 35 | commit_str = 'current source' |
| 36 | str = '%s %s for %d boards' % ( |
| 37 | 'Summary of' if is_summary else 'Building', commit_str, |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 38 | len(selected)) |
| 39 | str += ' (%d thread%s, %d job%s per thread)' % (options.threads, |
| 40 | GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) |
| 41 | return str |
| 42 | |
| 43 | def ShowActions(series, why_selected, boards_selected, builder, options): |
| 44 | """Display a list of actions that we would take, if not a dry run. |
| 45 | |
| 46 | Args: |
| 47 | series: Series object |
| 48 | why_selected: Dictionary where each key is a buildman argument |
| 49 | provided by the user, and the value is the boards brought |
| 50 | in by that argument. For example, 'arm' might bring in |
| 51 | 400 boards, so in this case the key would be 'arm' and |
| 52 | the value would be a list of board names. |
| 53 | boards_selected: Dict of selected boards, key is target name, |
| 54 | value is Board object |
| 55 | builder: The builder that will be used to build the commits |
| 56 | options: Command line options object |
| 57 | """ |
| 58 | col = terminal.Color() |
| 59 | print 'Dry run, so not doing much. But I would do this:' |
| 60 | print |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 61 | if series: |
| 62 | commits = series.commits |
| 63 | else: |
| 64 | commits = None |
| 65 | print GetActionSummary(False, commits, boards_selected, |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 66 | options) |
| 67 | print 'Build directory: %s' % builder.base_dir |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 68 | if commits: |
| 69 | for upto in range(0, len(series.commits), options.step): |
| 70 | commit = series.commits[upto] |
| 71 | print ' ', col.Color(col.YELLOW, commit.hash, bright=False), |
| 72 | print commit.subject |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 73 | print |
| 74 | for arg in why_selected: |
| 75 | if arg != 'all': |
| 76 | print arg, ': %d boards' % why_selected[arg] |
| 77 | print ('Total boards to build for each commit: %d\n' % |
| 78 | why_selected['all']) |
| 79 | |
| 80 | def DoBuildman(options, args): |
| 81 | """The main control code for buildman |
| 82 | |
| 83 | Args: |
| 84 | options: Command line options object |
| 85 | args: Command line arguments (list of strings) |
| 86 | """ |
Simon Glass | 48ba585 | 2014-09-05 19:00:11 -0600 | [diff] [blame^] | 87 | if options.full_help: |
| 88 | pager = os.getenv('PAGER') |
| 89 | if not pager: |
| 90 | pager = 'more' |
| 91 | fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') |
| 92 | command.Run(pager, fname) |
| 93 | return 0 |
| 94 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 95 | gitutil.Setup() |
| 96 | |
Simon Glass | 0f7c9dd | 2014-08-09 15:33:05 -0600 | [diff] [blame] | 97 | bsettings.Setup(options.config_file) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 98 | options.git_dir = os.path.join(options.git, '.git') |
| 99 | |
| 100 | toolchains = toolchain.Toolchains() |
| 101 | toolchains.Scan(options.list_tool_chains) |
| 102 | if options.list_tool_chains: |
| 103 | toolchains.List() |
| 104 | print |
Simon Glass | 2c3deb9 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 105 | return 0 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 106 | |
| 107 | # Work out how many commits to build. We want to build everything on the |
| 108 | # branch. We also build the upstream commit as a control so we can see |
| 109 | # problems introduced by the first commit on the branch. |
| 110 | col = terminal.Color() |
| 111 | count = options.count |
| 112 | if count == -1: |
| 113 | if not options.branch: |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 114 | count = 1 |
| 115 | else: |
| 116 | count = gitutil.CountCommitsInBranch(options.git_dir, |
| 117 | options.branch) |
| 118 | if count is None: |
| 119 | str = ("Branch '%s' not found or has no upstream" % |
| 120 | options.branch) |
Masahiro Yamada | 31e2141 | 2014-08-16 00:59:26 +0900 | [diff] [blame] | 121 | sys.exit(col.Color(col.RED, str)) |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 122 | count += 1 # Build upstream commit also |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 123 | |
| 124 | if not count: |
| 125 | str = ("No commits found to process in branch '%s': " |
| 126 | "set branch's upstream or use -c flag" % options.branch) |
Masahiro Yamada | 31e2141 | 2014-08-16 00:59:26 +0900 | [diff] [blame] | 127 | sys.exit(col.Color(col.RED, str)) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 128 | |
| 129 | # Work out what subset of the boards we are building |
Masahiro Yamada | 73f30b9 | 2014-07-30 14:08:22 +0900 | [diff] [blame] | 130 | board_file = os.path.join(options.git, 'boards.cfg') |
Masahiro Yamada | 390f703 | 2014-08-22 14:33:41 +0900 | [diff] [blame] | 131 | status = subprocess.call([os.path.join(options.git, |
| 132 | 'tools/genboardscfg.py')]) |
| 133 | if status != 0: |
| 134 | sys.exit("Failed to generate boards.cfg") |
Masahiro Yamada | 73f30b9 | 2014-07-30 14:08:22 +0900 | [diff] [blame] | 135 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 136 | boards = board.Boards() |
| 137 | boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) |
Simon Glass | 3cf4ae6 | 2014-08-28 09:43:41 -0600 | [diff] [blame] | 138 | |
| 139 | exclude = [] |
| 140 | if options.exclude: |
| 141 | for arg in options.exclude: |
| 142 | exclude += arg.split(',') |
| 143 | |
| 144 | why_selected = boards.SelectBoards(args, exclude) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 145 | selected = boards.GetSelected() |
| 146 | if not len(selected): |
Masahiro Yamada | 31e2141 | 2014-08-16 00:59:26 +0900 | [diff] [blame] | 147 | sys.exit(col.Color(col.RED, 'No matching boards found')) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 148 | |
| 149 | # Read the metadata from the commits. First look at the upstream commit, |
| 150 | # then the ones in the branch. We would like to do something like |
| 151 | # upstream/master~..branch but that isn't possible if upstream/master is |
| 152 | # a merge commit (it will list all the commits that form part of the |
| 153 | # merge) |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 154 | if options.branch: |
Simon Glass | 3b74ba5 | 2014-08-09 15:33:09 -0600 | [diff] [blame] | 155 | if count == -1: |
| 156 | range_expr = gitutil.GetRangeInBranch(options.git_dir, |
| 157 | options.branch) |
| 158 | upstream_commit = gitutil.GetUpstream(options.git_dir, |
| 159 | options.branch) |
| 160 | series = patchstream.GetMetaDataForList(upstream_commit, |
| 161 | options.git_dir, 1) |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 162 | |
Simon Glass | 3b74ba5 | 2014-08-09 15:33:09 -0600 | [diff] [blame] | 163 | # Conflicting tags are not a problem for buildman, since it does |
| 164 | # not use them. For example, Series-version is not useful for |
| 165 | # buildman. On the other hand conflicting tags will cause an |
| 166 | # error. So allow later tags to overwrite earlier ones. |
| 167 | series.allow_overwrite = True |
| 168 | series = patchstream.GetMetaDataForList(range_expr, |
| 169 | options.git_dir, None, series) |
| 170 | else: |
| 171 | # Honour the count |
| 172 | series = patchstream.GetMetaDataForList(options.branch, |
| 173 | options.git_dir, count) |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 174 | else: |
| 175 | series = None |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 176 | options.verbose = True |
| 177 | options.show_errors = True |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 178 | |
| 179 | # By default we have one thread per CPU. But if there are not enough jobs |
| 180 | # we can have fewer threads and use a high '-j' value for make. |
| 181 | if not options.threads: |
| 182 | options.threads = min(multiprocessing.cpu_count(), len(selected)) |
| 183 | if not options.jobs: |
| 184 | options.jobs = max(1, (multiprocessing.cpu_count() + |
| 185 | len(selected) - 1) / len(selected)) |
| 186 | |
| 187 | if not options.step: |
| 188 | options.step = len(series.commits) - 1 |
| 189 | |
Masahiro Yamada | 9979692 | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 190 | gnu_make = command.Output(os.path.join(options.git, |
| 191 | 'scripts/show-gnu-make')).rstrip() |
| 192 | if not gnu_make: |
Masahiro Yamada | 31e2141 | 2014-08-16 00:59:26 +0900 | [diff] [blame] | 193 | sys.exit('GNU Make not found') |
Masahiro Yamada | 9979692 | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 194 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 195 | # Create a new builder with the selected options |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 196 | if options.branch: |
| 197 | dirname = options.branch |
| 198 | else: |
| 199 | dirname = 'current' |
| 200 | output_dir = os.path.join(options.output_dir, dirname) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 201 | builder = Builder(toolchains, output_dir, options.git_dir, |
Masahiro Yamada | 9979692 | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 202 | options.threads, options.jobs, gnu_make=gnu_make, checkout=True, |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 203 | show_unknown=options.show_unknown, step=options.step) |
| 204 | builder.force_config_on_failure = not options.quick |
| 205 | |
| 206 | # For a dry run, just show our actions as a sanity check |
| 207 | if options.dry_run: |
| 208 | ShowActions(series, why_selected, selected, builder, options) |
| 209 | else: |
| 210 | builder.force_build = options.force_build |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 211 | builder.force_build_failures = options.force_build_failures |
Simon Glass | 97e9152 | 2014-07-14 17:51:02 -0600 | [diff] [blame] | 212 | builder.force_reconfig = options.force_reconfig |
Simon Glass | 189a496 | 2014-07-14 17:51:03 -0600 | [diff] [blame] | 213 | builder.in_tree = options.in_tree |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 214 | |
| 215 | # Work out which boards to build |
| 216 | board_selected = boards.GetSelectedDict() |
| 217 | |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 218 | if series: |
| 219 | commits = series.commits |
| 220 | else: |
| 221 | commits = None |
| 222 | |
| 223 | print GetActionSummary(options.summary, commits, board_selected, |
| 224 | options) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 225 | |
Simon Glass | b2ea7ab | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 226 | builder.SetDisplayOptions(options.show_errors, options.show_sizes, |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 227 | options.show_detail, options.show_bloat, |
| 228 | options.list_error_boards) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 229 | if options.summary: |
| 230 | # We can't show function sizes without board details at present |
| 231 | if options.show_bloat: |
| 232 | options.show_detail = True |
Simon Glass | b2ea7ab | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 233 | builder.ShowSummary(commits, board_selected) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 234 | else: |
Simon Glass | 2c3deb9 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 235 | fail, warned = builder.BuildBoards(commits, board_selected, |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 236 | options.keep_outputs, options.verbose) |
Simon Glass | 2c3deb9 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 237 | if fail: |
| 238 | return 128 |
| 239 | elif warned: |
| 240 | return 129 |
| 241 | return 0 |