Allow jobs to shutdown turbo-hipster
This lets jobs turn off and exit turbo-hipster once they are done.
This is useful for when using nodepool or when a job leaves the
environment dirty and we can't run more jobs on this worker.
Change-Id: I823be4196a5bf9ca92a14d9caf26163398a9434c
diff --git a/tests/etc/default-config.yaml b/tests/etc/default-config.yaml
new file mode 100644
index 0000000..0412e01
--- /dev/null
+++ b/tests/etc/default-config.yaml
@@ -0,0 +1,33 @@
+zuul_server:
+ gerrit_site: http://review.openstack.org
+ zuul_site: http://119.9.13.90
+ git_origin: git://git.openstack.org/
+ gearman_host: localhost
+ gearman_port: 0
+
+debug_log: /var/log/turbo-hipster/debug.log
+jobs_working_dir: /var/lib/turbo-hipster/jobs
+git_working_dir: /var/lib/turbo-hipster/git
+pip_download_cache: /var/cache/pip
+
+plugins:
+ - name: gate_real_db_upgrade
+ datasets_dir: /var/lib/turbo-hipster/datasets_devstack_131007
+ function: build:gate-real-db-upgrade_nova_mysql_devstack_131007
+
+ - name: gate_real_db_upgrade
+ datasets_dir: /var/lib/turbo-hipster/datasets_user_001
+ function: build:gate-real-db-upgrade_nova_mysql_user_001
+
+ - name: shell_script
+ function: build:do_something_shelly
+
+publish_logs:
+ type: swift
+ authurl: https://identity.api.rackspacecloud.com/v2.0/
+ tenant: XXXX
+ user: XXXXXX
+ password: XXXXXX
+ container: XXXXXX
+ region: SYD
+ prepend_url: http://www.rcbops.com/turbo_hipster/results/
diff --git a/tests/etc/shutdown-config.yaml b/tests/etc/shutdown-config.yaml
new file mode 100644
index 0000000..9175873
--- /dev/null
+++ b/tests/etc/shutdown-config.yaml
@@ -0,0 +1,25 @@
+zuul_server:
+ gerrit_site: http://review.openstack.org
+ zuul_site: http://119.9.13.90
+ git_origin: git://git.openstack.org/
+ gearman_host: localhost
+ gearman_port: 0
+
+debug_log: /var/log/turbo-hipster/debug.log
+jobs_working_dir: /var/lib/turbo-hipster/jobs
+git_working_dir: /var/lib/turbo-hipster/git
+pip_download_cache: /var/cache/pip
+
+plugins:
+ - name: shell_script
+ function: build:demo_job_clean
+ shell_script: /dev/null
+ - name: shell_script
+ function: build:demo_job_dirty
+ shell_script: /dev/null
+ shutdown-th: true
+
+publish_logs:
+ type: local
+ path: /var/lib/turbo_hipster/logs
+ prepend_url: http://mylogserver/
\ No newline at end of file
diff --git a/tests/fakes.py b/tests/fakes.py
index 5f78fbf..1b377cd 100644
--- a/tests/fakes.py
+++ b/tests/fakes.py
@@ -14,6 +14,11 @@
# License for the specific language governing permissions and limitations
# under the License.
+import gear
+import json
+import time
+import uuid
+
class FakeJob(object):
def __init__(self):
@@ -21,3 +26,43 @@
def sendWorkStatus(self, *args, **kwargs):
pass
+
+
+class FakeZuul(object):
+ """A fake zuul/gearman client to request work from gearman and check
+ results"""
+ def __init__(self, server, port):
+ self.gearman = gear.Client('FakeZuul')
+ self.gearman.addServer(server, port)
+ self.gearman.waitForServer()
+ self.job = None
+
+ def make_zuul_data(self, data={}):
+ defaults = {
+ 'ZUUL_UUID': str(uuid.uuid1()),
+ 'ZUUL_REF': 'a',
+ 'ZUUL_COMMIT': 'a',
+ 'ZUUL_PROJECT': 'a',
+ 'ZUUL_PIPELINE': 'a',
+ 'ZUUL_URL': 'http://localhost',
+ 'BASE_LOG_PATH': '56/123456/8',
+ 'LOG_PATH': '56/123456/8/check/job_name/uuid123'
+ }
+ defaults.update(data)
+ return defaults
+
+ def submit_job(self, name, data):
+ if not self.job:
+ self.job = gear.Job(name,
+ json.dumps(data),
+ unique=str(time.time()))
+ self.gearman.submitJob(self.job)
+ else:
+ raise Exception('A job already exists in self.job')
+
+ return self.job
+
+ def wait_for_completion(self):
+ if self.job:
+ while not self.job.complete:
+ time.sleep(0.1)
diff --git a/tests/fixtures/default-config.json b/tests/fixtures/default-config.json
deleted file mode 100644
index e33ea42..0000000
--- a/tests/fixtures/default-config.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "zuul_server": {
- "gerrit_site": "http://review.openstack.org",
- "zuul_site": "http://localhost",
- "git_origin": "git://git.openstack.org/",
- "gearman_host": "localhost",
- "gearman_port": 0
- },
- "debug_log": "/home/josh/var/log/turbo-hipster/debug.log",
- "jobs_working_dir": "/home/josh/var/lib/turbo-hipster/jobs",
- "git_working_dir": "/home/josh/var/lib/turbo-hipster/git",
- "pip_download_cache": "/home/josh/var/cache/pip",
- "plugins": [
- {
- "name": "gate_real_db_upgrade",
- "datasets_dir": "/var/lib/turbo-hipster/datasets_devstack_131007",
- "function": "build:gate-real-db-upgrade_nova_mysql_devstack_131007"
- },
- {
- "name": "gate_real_db_upgrade",
- "datasets_dir": "/var/lib/turbo-hipster/datasets_user_001",
- "function": "build:gate-real-db-upgrade_nova_mysql_user_001"
- },
- {
- "name": "shell_script",
- "function": "build:do_something_shelly"
- }
- ],
- "publish_logs":
- {
- "type": "local",
- "path": "/home/josh/var/www/results/",
- "prepend_url": "http://localhost/results/"
- }
-}
diff --git a/tests/test_worker_manager.py b/tests/test_worker_manager.py
index 1f3de5e..ff0733f 100644
--- a/tests/test_worker_manager.py
+++ b/tests/test_worker_manager.py
@@ -15,6 +15,7 @@
# under the License.
+import fixtures
import gear
import logging
import os
@@ -25,6 +26,8 @@
import turbo_hipster.task_plugins.gate_real_db_upgrade.task
import turbo_hipster.worker_server
+import fakes
+
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-32s '
'%(levelname)-8s %(message)s')
@@ -36,12 +39,13 @@
def setUp(self):
super(TestWithGearman, self).setUp()
-
- self.config = []
- self._load_config_fixture()
-
+ self.config = None
+ self.worker_server = None
self.gearman_server = gear.Server(0)
+ def start_server(self):
+ if not self.config:
+ self._load_config_fixture()
# Grab the port so the clients can connect to it
self.config['zuul_server']['gearman_port'] = self.gearman_server.port
@@ -57,20 +61,39 @@
self.fail("Failed to start worker_service services")
def tearDown(self):
- self.worker_server.stop()
+ if self.worker_server and not self.worker_server.stopped():
+ self.worker_server.shutdown()
self.gearman_server.shutdown()
super(TestWithGearman, self).tearDown()
- def _load_config_fixture(self, config_name='default-config.json'):
- config_dir = os.path.join(os.path.dirname(__file__), 'fixtures')
+ def _load_config_fixture(self, config_name='default-config.yaml'):
+ config_dir = os.path.join(os.path.dirname(__file__), 'etc')
with open(os.path.join(config_dir, config_name), 'r') as config_stream:
self.config = yaml.safe_load(config_stream)
+ # Set all of the working dirs etc to a writeable temp dir
+ temp_path = self.useFixture(fixtures.TempDir()).path
+ for config_dir in ['debug_log', 'jobs_working_dir', 'git_working_dir',
+ 'pip_download_cache']:
+ if config_dir in self.config:
+ if self.config[config_dir][0] == '/':
+ self.config[config_dir] = self.config[config_dir][1:]
+ self.config[config_dir] = os.path.join(temp_path,
+ self.config[config_dir])
+ if self.config['publish_logs']['type'] == 'local':
+ if self.config['publish_logs']['path'][0] == '/':
+ self.config['publish_logs']['path'] = \
+ self.config['publish_logs']['path'][1:]
+ self.config['publish_logs']['path'] = os.path.join(
+ temp_path, self.config[config_dir])
+
class TestWorkerServer(TestWithGearman):
def test_plugins_load(self):
"Test the configured plugins are loaded"
+ self.start_server()
+
self.assertFalse(self.worker_server.stopped())
self.assertEqual(3, len(self.worker_server.plugins))
@@ -112,10 +135,12 @@
def test_zuul_client_started(self):
"Test the zuul client has been started"
+ self.start_server()
self.assertFalse(self.worker_server.zuul_client.stopped())
def test_zuul_manager_started(self):
"Test the zuul manager has been started"
+ self.start_server()
self.assertFalse(self.worker_server.zuul_manager.stopped())
@@ -126,6 +151,9 @@
def test_registered_functions(self):
"Test the correct functions are registered with gearman"
+
+ self.start_server()
+
# The client should have all of the functions defined in the config
# registered with gearman
@@ -160,10 +188,36 @@
"Test sending a stop signal to the client exists correctly"
pass
+ def test_job_can_shutdown_th(self):
+ self._load_config_fixture('shutdown-config.yaml')
+ self.start_server()
+ zuul = fakes.FakeZuul(self.config['zuul_server']['gearman_host'],
+ self.config['zuul_server']['gearman_port'])
+
+ # First check we can run a job that /doesn't/ shut down turbo-hipster
+ data_req = zuul.make_zuul_data()
+ zuul.submit_job('build:demo_job_clean', data_req)
+ zuul.wait_for_completion()
+ self.assertTrue(zuul.job.complete)
+ self.assertFalse(self.worker_server.stopped())
+
+ # Now run a job that leaves the environment dirty and /should/ shut
+ # down turbo-hipster
+ zuul.job = None
+ zuul.submit_job('build:demo_job_dirty', data_req)
+ zuul.wait_for_completion()
+ self.assertTrue(zuul.job.complete)
+ # Give the server a second to shutdown
+ time.sleep(1)
+ self.assertTrue(self.worker_server.stopped())
+
class TestZuulManager(TestWithGearman):
def test_registered_functions(self):
"Test the correct functions are registered with gearman"
+
+ self.start_server()
+
# We need to wait for all the functions to register with the server..
# We'll give it up to 10seconds to do so
t0 = time.time()