Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 1 | # Copyright 2013 Rackspace Australia |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 | # not use this file except in compliance with the License. You may obtain |
| 5 | # a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations |
| 13 | # under the License. |
| 14 | |
| 15 | |
| 16 | import copy |
| 17 | import json |
| 18 | import logging |
| 19 | import os |
| 20 | |
Joshua Hesketh | 6224554 | 2014-01-16 18:00:56 +1100 | [diff] [blame] | 21 | from turbo_hipster.lib import common |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 22 | from turbo_hipster.lib import utils |
| 23 | |
| 24 | |
| 25 | class Task(object): |
Joshua Hesketh | 9cd2f93 | 2014-03-05 16:49:22 +1100 | [diff] [blame] | 26 | """ A base object for running a job (aka Task) """ |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 27 | log = logging.getLogger("lib.models.Task") |
| 28 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 29 | def __init__(self, worker_server, plugin_config, job_name): |
| 30 | self.worker_server = worker_server |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 31 | self.plugin_config = plugin_config |
| 32 | self.job_name = job_name |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 33 | self._reset() |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 34 | |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 35 | # Define the number of steps we will do to determine our progress. |
| 36 | self.total_steps = 0 |
| 37 | |
| 38 | def _reset(self): |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 39 | self.job = None |
| 40 | self.job_arguments = None |
| 41 | self.work_data = None |
| 42 | self.cancelled = False |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 43 | self.success = True |
| 44 | self.messages = [] |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 45 | self.current_step = 0 |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 46 | |
| 47 | def start_job(self, job): |
| 48 | self._reset() |
| 49 | self.job = job |
| 50 | |
| 51 | if self.job is not None: |
| 52 | try: |
| 53 | self.job_arguments = \ |
| 54 | json.loads(self.job.arguments.decode('utf-8')) |
| 55 | self.log.debug("Got job from ZUUL %s" % self.job_arguments) |
| 56 | |
| 57 | # Send an initial WORK_DATA and WORK_STATUS packets |
| 58 | self._send_work_data() |
| 59 | |
| 60 | # Execute the job_steps |
| 61 | self.do_job_steps() |
| 62 | |
| 63 | # Finally, send updated work data and completed packets |
| 64 | self._send_final_results() |
| 65 | |
| 66 | except Exception as e: |
| 67 | self.log.exception('Exception handling log event.') |
| 68 | if not self.cancelled: |
Joshua Hesketh | 6e20b16 | 2014-04-09 13:09:19 +1000 | [diff] [blame] | 69 | self.success = False |
| 70 | self.messages.append('Exception: %s' % e) |
| 71 | self._send_work_data() |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 72 | self.job.sendWorkException(str(e).encode('utf-8')) |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 73 | |
Joshua Hesketh | 38a1718 | 2014-03-05 14:19:38 +1100 | [diff] [blame] | 74 | def stop_working(self, number=None): |
| 75 | # Check the number is for this job instance (None will cancel all) |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 76 | # (makes it possible to run multiple workers with this task |
| 77 | # on this server) |
Joshua Hesketh | 38a1718 | 2014-03-05 14:19:38 +1100 | [diff] [blame] | 78 | if number is None or number == self.job.unique: |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 79 | self.log.debug("We've been asked to stop by our gearman manager") |
| 80 | self.cancelled = True |
| 81 | # TODO: Work out how to kill current step |
| 82 | |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 83 | def _get_work_data(self): |
| 84 | if self.work_data is None: |
| 85 | hostname = os.uname()[1] |
| 86 | self.work_data = dict( |
| 87 | name=self.job_name, |
| 88 | number=self.job.unique, |
| 89 | manager='turbo-hipster-manager-%s' % hostname, |
| 90 | url='http://localhost', |
| 91 | ) |
| 92 | return self.work_data |
| 93 | |
| 94 | def _send_work_data(self): |
| 95 | """ Send the WORK DATA in json format for job """ |
| 96 | self.log.debug("Send the work data response: %s" % |
| 97 | json.dumps(self._get_work_data())) |
Joshua Hesketh | 3cda79f | 2014-01-31 13:10:29 +1100 | [diff] [blame] | 98 | if self.success: |
| 99 | self.work_data['result'] = 'SUCCESS' |
| 100 | else: |
| 101 | self.work_data['result'] = '\n'.join(self.messages) |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 102 | self.job.sendWorkData(json.dumps(self._get_work_data())) |
| 103 | |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 104 | def _send_final_results(self): |
| 105 | self._send_work_data() |
| 106 | |
Joshua Hesketh | b5f99b6 | 2014-01-30 16:03:19 +1100 | [diff] [blame] | 107 | if self.success: |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 108 | self.job.sendWorkComplete( |
| 109 | json.dumps(self._get_work_data())) |
| 110 | else: |
| 111 | self.job.sendWorkFail() |
| 112 | |
Joshua Hesketh | e76a0dd | 2014-01-16 17:57:45 +1100 | [diff] [blame] | 113 | def _do_next_step(self): |
| 114 | """ Send a WORK_STATUS command to the gearman server. |
| 115 | This can provide a progress bar. """ |
| 116 | |
| 117 | # Each opportunity we should check if we need to stop |
| 118 | if self.cancelled: |
| 119 | self.work_data['result'] = "Failed: Job cancelled" |
| 120 | self.job.sendWorkStatus(self.current_step, self.total_steps) |
| 121 | self.job.sendWorkFail() |
| 122 | raise Exception('Job cancelled') |
| 123 | |
| 124 | self.current_step += 1 |
| 125 | self.job.sendWorkStatus(self.current_step, self.total_steps) |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 126 | |
| 127 | |
| 128 | class ShellTask(Task): |
| 129 | log = logging.getLogger("lib.models.ShellTask") |
| 130 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 131 | def __init__(self, worker_server, plugin_config, job_name): |
| 132 | super(ShellTask, self).__init__(worker_server, plugin_config, job_name) |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 133 | # Define the number of steps we will do to determine our progress. |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 134 | self.total_steps = 6 |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 135 | |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 136 | def _reset(self): |
| 137 | super(ShellTask, self)._reset() |
| 138 | self.git_path = None |
Joshua Hesketh | c73328c | 2014-01-18 16:09:54 +1100 | [diff] [blame] | 139 | self.job_working_dir = None |
| 140 | self.shell_output_log = None |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 141 | |
Joshua Hesketh | 235b13d | 2014-01-30 15:14:04 +1100 | [diff] [blame] | 142 | def do_job_steps(self): |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 143 | self.log.info('Step 1: Prep job working dir') |
Joshua Hesketh | c73328c | 2014-01-18 16:09:54 +1100 | [diff] [blame] | 144 | self._prep_working_dir() |
| 145 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 146 | self.log.info('Step 2: Checkout updates from git') |
Joshua Hesketh | 1f2d1a2 | 2014-01-30 15:41:21 +1100 | [diff] [blame] | 147 | self._grab_patchset(self.job_arguments) |
| 148 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 149 | self.log.info('Step 3: Run shell script') |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 150 | self._execute_script() |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 151 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 152 | self.log.info('Step 4: Analyse logs for errors') |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 153 | self._parse_and_check_results() |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 154 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 155 | self.log.info('Step 5: handle the results (and upload etc)') |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 156 | self._handle_results() |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 157 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 158 | self.log.info('Step 6: Handle extra actions such as shutting down') |
| 159 | self._handle_cleanup() |
| 160 | |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 161 | @common.task_step |
Joshua Hesketh | 1f2d1a2 | 2014-01-30 15:41:21 +1100 | [diff] [blame] | 162 | def _prep_working_dir(self): |
Joshua Hesketh | 9900554 | 2014-01-30 16:34:36 +1100 | [diff] [blame] | 163 | self.job_identifier = utils.determine_job_identifier( |
| 164 | self.job_arguments, |
| 165 | self.plugin_config['function'], |
| 166 | self.job.unique |
| 167 | ) |
Joshua Hesketh | 1f2d1a2 | 2014-01-30 15:41:21 +1100 | [diff] [blame] | 168 | self.job_working_dir = os.path.join( |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 169 | self.worker_server.config['jobs_working_dir'], |
Joshua Hesketh | 9900554 | 2014-01-30 16:34:36 +1100 | [diff] [blame] | 170 | self.job_identifier |
Joshua Hesketh | 1f2d1a2 | 2014-01-30 15:41:21 +1100 | [diff] [blame] | 171 | ) |
| 172 | self.shell_output_log = os.path.join( |
| 173 | self.job_working_dir, |
| 174 | 'shell_output.log' |
| 175 | ) |
| 176 | |
| 177 | if not os.path.isdir(os.path.dirname(self.shell_output_log)): |
| 178 | os.makedirs(os.path.dirname(self.shell_output_log)) |
| 179 | |
| 180 | @common.task_step |
| 181 | def _grab_patchset(self, job_args): |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 182 | """ Checkout the reference into config['git_working_dir'] """ |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 183 | |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 184 | self.log.debug("Grab the patchset we want to test against") |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 185 | local_path = os.path.join(self.worker_server.config['git_working_dir'], |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 186 | self.job_name, job_args['ZUUL_PROJECT']) |
| 187 | if not os.path.exists(local_path): |
| 188 | os.makedirs(local_path) |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 189 | |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 190 | git_args = copy.deepcopy(job_args) |
Joshua Hesketh | 9177876 | 2014-01-16 18:24:46 +1100 | [diff] [blame] | 191 | |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 192 | cmd = os.path.join(os.path.join(os.path.dirname(__file__), |
| 193 | 'gerrit-git-prep.sh')) |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 194 | cmd += ' ' + self.worker_server.config['zuul_server']['gerrit_site'] |
| 195 | cmd += ' ' + self.worker_server.config['zuul_server']['zuul_site'] |
Joshua Hesketh | 1f2d1a2 | 2014-01-30 15:41:21 +1100 | [diff] [blame] | 196 | utils.execute_to_log(cmd, self.shell_output_log, env=git_args, |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 197 | cwd=local_path) |
| 198 | self.git_path = local_path |
| 199 | return local_path |
| 200 | |
| 201 | @common.task_step |
| 202 | def _execute_script(self): |
| 203 | # Run script |
Joshua Hesketh | c73328c | 2014-01-18 16:09:54 +1100 | [diff] [blame] | 204 | cmd = self.plugin_config['shell_script'] |
| 205 | cmd += ( |
| 206 | (' %(git_path)s %(job_working_dir)s %(unique_id)s') |
| 207 | % { |
| 208 | 'git_path': self.git_path, |
| 209 | 'job_working_dir': self.job_working_dir, |
| 210 | 'unique_id': self.job.unique |
| 211 | } |
| 212 | ) |
| 213 | self.script_return_code = utils.execute_to_log( |
| 214 | cmd, |
| 215 | self.shell_output_log |
| 216 | ) |
Joshua Hesketh | 81f87ed | 2014-01-18 15:24:48 +1100 | [diff] [blame] | 217 | |
| 218 | @common.task_step |
| 219 | def _parse_and_check_results(self): |
| 220 | if self.script_return_code > 0: |
| 221 | self.success = False |
| 222 | self.messages.append('Return code from test script was non-zero ' |
| 223 | '(%d)' % self.script_return_code) |
| 224 | |
| 225 | @common.task_step |
| 226 | def _handle_results(self): |
Joshua Hesketh | 6055f31 | 2014-01-22 15:04:54 +1100 | [diff] [blame] | 227 | """Upload the contents of the working dir either using the instructions |
| 228 | provided by zuul and/or our configuration""" |
Joshua Hesketh | 221ae74 | 2014-01-22 16:09:58 +1100 | [diff] [blame] | 229 | |
Joshua Hesketh | 6055f31 | 2014-01-22 15:04:54 +1100 | [diff] [blame] | 230 | self.log.debug("Process the resulting files (upload/push)") |
Joshua Hesketh | 221ae74 | 2014-01-22 16:09:58 +1100 | [diff] [blame] | 231 | |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 232 | if 'publish_logs' in self.worker_server.config: |
| 233 | index_url = utils.push_file( |
| 234 | self.job_identifier, self.shell_output_log, |
| 235 | self.worker_server.config['publish_logs']) |
Joshua Hesketh | 221ae74 | 2014-01-22 16:09:58 +1100 | [diff] [blame] | 236 | self.log.debug("Index URL found at %s" % index_url) |
| 237 | self.work_data['url'] = index_url |
| 238 | |
| 239 | if 'ZUUL_EXTRA_SWIFT_URL' in self.job_arguments: |
| 240 | # Upload to zuul's url as instructed |
| 241 | utils.zuul_swift_upload(self.job_working_dir, self.job_arguments) |
Joshua Hesketh | 9900554 | 2014-01-30 16:34:36 +1100 | [diff] [blame] | 242 | self.work_data['url'] = self.job_identifier |
Joshua Hesketh | 96adb28 | 2014-03-25 16:26:45 +1100 | [diff] [blame] | 243 | |
| 244 | @common.task_step |
| 245 | def _handle_cleanup(self): |
| 246 | """Handle and cleanup functions. Shutdown if requested to so that no |
| 247 | further jobs are ran if the environment is dirty.""" |
| 248 | if ('shutdown-th' in self.plugin_config and |
| 249 | self.plugin_config['shutdown-th']): |
| 250 | self.worker_server.shutdown_gracefully() |