JJB runner POC
The initial work to support running job instructions pulled in from
jenkins-job-builder config.
Because the XML is tightly coupled with JJB it's easier to use
xmltodict at this point. Ideally a new interpreter for JJB formatted
files to turbo-hipster instructions could be made.
At the moment we're ignoring JJB instructions from items we aren't
interested in (for example, publishers and build wrappers). Some
level of support should be added later for these or the job
instructions should be updated to not use them.
Change-Id: I0560d8e0a7e33548bacee3aa98bd45a5907bec21
diff --git a/requirements.txt b/requirements.txt
index 26e13b2..5405f9d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,3 +14,6 @@
requests
PyYAML>=3.1.0,<4.0.0
+
+jenkins-job-builder
+xmltodict
\ No newline at end of file
diff --git a/tests/etc/jjb-config.yaml b/tests/etc/jjb-config.yaml
new file mode 100644
index 0000000..bcc341c
--- /dev/null
+++ b/tests/etc/jjb-config.yaml
@@ -0,0 +1,21 @@
+zuul_server:
+ gerrit_site: http://review.openstack.org
+ zuul_site: http://119.9.13.90
+ git_origin: git://git.openstack.org/
+ gearman_host: localhost
+ gearman_port: 0
+
+debug_log: /var/log/turbo-hipster/debug.log
+jobs_working_dir: /var/lib/turbo-hipster/jobs
+git_working_dir: /var/lib/turbo-hipster/git
+pip_download_cache: /var/cache/pip
+
+plugins:
+ - name: jjb_runner
+ function: build:gate-turbo-hipster-pep8
+ jjb_config: modules/openstack_project/files/jenkins_job_builder/config
+
+publish_logs:
+ type: local
+ path: /var/lib/turbo_hipster/logs
+ prepend_url: http://mylogserver/
\ No newline at end of file
diff --git a/tests/fakes.py b/tests/fakes.py
index 1b377cd..07e5eeb 100644
--- a/tests/fakes.py
+++ b/tests/fakes.py
@@ -38,15 +38,16 @@
self.job = None
def make_zuul_data(self, data={}):
+ job_uuid = str(uuid.uuid1())
defaults = {
- 'ZUUL_UUID': str(uuid.uuid1()),
+ 'ZUUL_UUID': job_uuid,
'ZUUL_REF': 'a',
'ZUUL_COMMIT': 'a',
'ZUUL_PROJECT': 'a',
'ZUUL_PIPELINE': 'a',
'ZUUL_URL': 'http://localhost',
'BASE_LOG_PATH': '56/123456/8',
- 'LOG_PATH': '56/123456/8/check/job_name/uuid123'
+ 'LOG_PATH': '56/123456/8/check/job_name/%s' % job_uuid
}
defaults.update(data)
return defaults
diff --git a/tests/test_jjb_runner.py b/tests/test_jjb_runner.py
new file mode 100644
index 0000000..fd60eb7
--- /dev/null
+++ b/tests/test_jjb_runner.py
@@ -0,0 +1,66 @@
+# Copyright 2014 Rackspace Australia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import base
+import fakes
+import fixtures
+import json
+import logging
+import os
+import uuid
+
+from turbo_hipster.lib import utils
+
+
+class TestTaskRunner(base.TestWithGearman):
+ log = logging.getLogger("TestTaskRunner")
+
+ def setUp(self):
+ super(TestTaskRunner, self).setUp()
+ # Grab a copy of JJB's config
+ temp_path = self.useFixture(fixtures.TempDir()).path
+ cmd = 'git clone git://git.openstack.org/openstack-infra/config'
+ utils.execute_to_log(cmd, '/dev/null', cwd=temp_path)
+ self.jjb_config_dir = os.path.join(
+ temp_path, 'config',
+ 'modules/openstack_project/files/jenkins_job_builder/config'
+ )
+
+ def test_job_can_shutdown_th(self):
+ self._load_config_fixture('jjb-config.yaml')
+ # set jjb_config to pulled in config
+ self.config['plugins'][0]['jjb_config'] = self.jjb_config_dir
+
+ self.start_server()
+ zuul = fakes.FakeZuul(self.config['zuul_server']['gearman_host'],
+ self.config['zuul_server']['gearman_port'])
+
+ job_uuid = str(uuid.uuid1())[:8]
+ data_req = {
+ 'ZUUL_UUID': job_uuid,
+ 'ZUUL_PROJECT': 'stackforge/turbo-hipster',
+ 'ZUUL_PIPELINE': 'check',
+ 'ZUUL_URL': 'git://git.openstack.org/',
+ 'BRANCH': 'master',
+ 'BASE_LOG_PATH': '56/123456/8',
+ 'LOG_PATH': '56/123456/8/check/job_name/%s' % job_uuid
+ }
+
+ zuul.submit_job('build:gate-turbo-hipster-pep8', data_req)
+ zuul.wait_for_completion()
+
+ self.assertTrue(zuul.job.complete)
+ last_data = json.loads(zuul.job.data[-1])
+ self.log.debug(last_data)
+ self.assertEqual("SUCCESS", last_data['result'])
diff --git a/tests/test_utils.py b/tests/test_utils.py
index dcaec8f..4965c03 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -48,7 +48,7 @@
print d
self.assertNotEqual('', d)
- self.assertEqual(3, len(d.split('\n')))
+ self.assertEqual(4, len(d.split('\n')))
self.assertNotEqual(-1, d.find('yay'))
self.assertNotEqual(-1, d.find('[script exit code = 0]'))
diff --git a/turbo_hipster/lib/utils.py b/turbo_hipster/lib/utils.py
index f0603af..5487765 100644
--- a/turbo_hipster/lib/utils.py
+++ b/turbo_hipster/lib/utils.py
@@ -91,14 +91,8 @@
self.repo = git.Repo(self.local_path)
-def execute_to_log(cmd, logfile, timeout=-1,
- watch_logs=[
- ('[syslog]', '/var/log/syslog'),
- ('[sqlslo]', '/var/log/mysql/slow-queries.log'),
- ('[sqlerr]', '/var/log/mysql/error.log')
- ],
- heartbeat=True, env=None, cwd=None
- ):
+def execute_to_log(cmd, logfile, timeout=-1, watch_logs=[], heartbeat=30,
+ env=None, cwd=None):
""" Executes a command and logs the STDOUT/STDERR and output of any
supplied watch_logs from logs into a new logfile
@@ -132,6 +126,7 @@
% (watch_file[1], e))
cmd += ' 2>&1'
+ logger.info("[running %s]" % cmd)
start_time = time.time()
p = subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
@@ -174,7 +169,7 @@
for fd, flag in poll_obj.poll(0):
process(fd)
- if time.time() - last_heartbeat > 30:
+ if heartbeat and (time.time() - last_heartbeat > heartbeat):
# Append to logfile
logger.info("[heartbeat]")
last_heartbeat = time.time()
diff --git a/turbo_hipster/task_plugins/jjb_runner/__init__.py b/turbo_hipster/task_plugins/jjb_runner/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/turbo_hipster/task_plugins/jjb_runner/__init__.py
diff --git a/turbo_hipster/task_plugins/jjb_runner/task.py b/turbo_hipster/task_plugins/jjb_runner/task.py
new file mode 100644
index 0000000..5156c8b
--- /dev/null
+++ b/turbo_hipster/task_plugins/jjb_runner/task.py
@@ -0,0 +1,178 @@
+# Copyright 2014 Rackspace Australia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import copy
+import logging
+import os
+import xmltodict
+
+import jenkins_jobs.builder
+
+from turbo_hipster.lib import common
+from turbo_hipster.lib import models
+from turbo_hipster.lib import utils
+
+
+class UnimplementedJJBFunction(Exception):
+ pass
+
+
+class Runner(models.ShellTask):
+
+ """A plugin to run jobs defined by JJB.
+ Based on models.ShellTask the steps can be overwritten."""
+
+ log = logging.getLogger("task_plugins.jjb_runner.task.Runner")
+
+ def __init__(self, worker_server, plugin_config, job_name):
+ super(Runner, self).__init__(worker_server, plugin_config, job_name)
+ self.total_steps = 6
+ self.jjb_instructions = {}
+ self.script_return_codes = []
+
+ def do_job_steps(self):
+ self.log.info('Step 1: Prep job working dir')
+ self._prep_working_dir()
+
+ self.log.info('Step 2: Grab instructions from jjb')
+ self._grab_jjb_instructions()
+
+ self.log.info('Step 3: Follow JJB Instructions')
+ self._execute_instructions()
+
+ self.log.info('Step 4: Analyse logs for errors')
+ self._parse_and_check_results()
+
+ self.log.info('Step 5: handle the results (and upload etc)')
+ self._handle_results()
+
+ self.log.info('Step 6: Handle extra actions such as shutting down')
+ self._handle_cleanup()
+
+ @common.task_step
+ def _grab_jjb_instructions(self):
+ """ Use JJB to interpret instructions into a dictionary. """
+
+ # For the moment we're just using xmltodict as the xml is very tightly
+ # coupled to JJB. In the future we could have an interpreter for JJB
+ # files.
+
+ # Set up a builder with fake jenkins creds
+ jjb = jenkins_jobs.builder.Builder('http://', '', '')
+ jjb.load_files(self.plugin_config['jjb_config'])
+ jjb.parser.generateXML([self.plugin_config['function']
+ .replace('build:', '')])
+ if len(jjb.parser.jobs) == 1:
+ # got the right job
+ self.jjb_instructions = xmltodict.parse(
+ jjb.parser.jobs[0].output())
+
+ @common.task_step
+ def _execute_instructions(self):
+ self.log.debug(self.plugin_config['function'].replace('build:', ''))
+ self.log.debug(self.jjb_instructions.keys())
+ self.log.debug(self.jjb_instructions)
+
+ # Look at all of the items in the jenkins project and raise errors
+ # for unimplemented functionality
+ for key, value in self.jjb_instructions['project'].items():
+ self.log.debug(key)
+ self.log.debug(value)
+
+ if key in ['actions', 'properties']:
+ # Not sure how to handle these when they have values
+ if value is None:
+ continue
+ else:
+ raise UnimplementedJJBFunction(
+ "Not sure how to handle values for %s (yet)" % key)
+ elif key in ['description', 'keepDependencies',
+ 'blockBuildWhenDownstreamBuilding',
+ 'blockBuildWhenUpstreamBuilding', 'concurrentBuild',
+ 'assignedNode', 'canRoam', 'logRotator', 'scm']:
+ # Ignore all of these directives as they don't apply to
+ # turbo-hipster/zuul
+ continue
+ elif key == 'builders':
+ # Loop over builders
+ self._handle_builders(value)
+ elif key == 'publishers':
+ # Ignore publishers for the moment
+ continue
+ elif key == 'buildWrappers':
+ # Ignore buildWrappers for the moment but probably should
+ # duplicate functionality for timeout reasons
+ continue
+ else:
+ raise UnimplementedJJBFunction(
+ "We don't know what to do with '%s' (yet)"
+ % key)
+
+ def _handle_builders(self, builders):
+ for key, value in builders.items():
+ self.log.debug('--builder')
+ self.log.debug(key)
+ self.log.debug(value)
+ if key == 'hudson.tasks.Shell':
+ self._handle_shell_items(value)
+ else:
+ raise UnimplementedJJBFunction(
+ "We don't know how to handle the builder '%s' (yet)"
+ % key)
+
+ def _handle_shell_items(self, shell_tasks):
+ for shell_task in shell_tasks:
+ for key, value in shell_task.items():
+ self.log.debug('--Shell')
+ self.log.debug(key)
+ self.log.debug(value)
+ if key == 'command':
+ self._handle_command(value)
+ else:
+ raise UnimplementedJJBFunction(
+ "We don't know how to handle the command '%s' (yet)"
+ % key)
+
+ def _handle_command(self, command):
+ # Cd to working dir
+ # export job_params as env
+ self.log.debug("EXECUTING COMMAND")
+ cwd = os.path.join(self.job_working_dir, 'working/')
+ if not os.path.isdir(os.path.dirname(cwd)):
+ self.log.debug('making dir, %s' % cwd)
+ os.makedirs(os.path.dirname(cwd))
+
+ env = copy.deepcopy(self.job_arguments)
+ env['PATH'] = os.environ['PATH']
+
+ self.script_return_codes.append(utils.execute_to_log(
+ command, self.shell_output_log,
+ env=env,
+ cwd=cwd
+ ))
+
+ @common.task_step
+ def _parse_and_check_results(self):
+ for return_code in self.script_return_codes:
+ if return_code > 0:
+ self.success = False
+ self.messages.append('Return code from test script was '
+ 'non-zero (%d)' % return_code)
+
+ @common.task_step
+ def _handle_results(self):
+ """Upload the contents of the working dir either using the instructions
+ provided by zuul and/or our configuration"""
+
+ self.log.debug("Process the resulting files (upload/push)")