refactor how datasets are stored
diff --git a/turbo_hipster/task_plugins/gate_real_db_upgrade/datasets/some_dataset_example/config.json b/datasets/some_dataset_example/config.json
similarity index 78%
rename from turbo_hipster/task_plugins/gate_real_db_upgrade/datasets/some_dataset_example/config.json
rename to datasets/some_dataset_example/config.json
index 40d4ba9..06eb64e 100644
--- a/turbo_hipster/task_plugins/gate_real_db_upgrade/datasets/some_dataset_example/config.json
+++ b/datasets/some_dataset_example/config.json
@@ -1,7 +1,6 @@
{
"project": "openstack/nova",
"type": "mysql",
- "gate": "build:gate-real-db-upgrade_nova_mysql",
"db_user": "nova",
"db_pass": "tester",
"database": "nova",
diff --git a/turbo_hipster/task_plugins/gate_real_db_upgrade/datasets/some_dataset_example/logging.conf b/datasets/some_dataset_example/logging.conf
similarity index 100%
rename from turbo_hipster/task_plugins/gate_real_db_upgrade/datasets/some_dataset_example/logging.conf
rename to datasets/some_dataset_example/logging.conf
diff --git a/doc/source/index.rst b/doc/source/index.rst
index cded138..25c0f30 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -13,6 +13,9 @@
intro
structure
+ installation
+ configuration
+ running
Indices and tables
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
new file mode 100644
index 0000000..def7cc6
--- /dev/null
+++ b/doc/source/installation.rst
@@ -0,0 +1,9 @@
+Installation
+============
+
+1. Download
+2. Copy config
+3. Edit config
+4. Make dirs
+5. Chmod them
+6. Start using daemon
\ No newline at end of file
diff --git a/etc/turbo-hipster/config.json b/etc/turbo-hipster/config.json
index 2a783f0..ca43fe9 100644
--- a/etc/turbo-hipster/config.json
+++ b/etc/turbo-hipster/config.json
@@ -8,9 +8,14 @@
"jobs_working_dir": "/var/lib/turbo-hipster/jobs",
"git_working_dir": "/var/lib/turbo-hipster/git",
"pip_download_cache": "/var/cache/pip",
- "plugins": ["gate_real_db_upgrade"],
- "publish_logs":
- {
+ "plugins": [
+ {
+ "name": "gate_real_db_upgrade",
+ "datasets_dir": "/var/lib/turbo-hipster/datasets",
+ "gate": "build:gate-real-db-upgrade_nova_mysql",
+ }
+ ],
+ "publish_logs": {
"type": "local",
"path": "/var/www/results/",
"prepend_url": "http://localhost/results/"
diff --git a/setup.cfg b/setup.cfg
index 6ab31b0..96e9c99 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -25,10 +25,6 @@
[pbr]
warnerrors = True
-[global]
-setup-hooks =
- pbr.hooks.setup_hook
-
[entry_points]
console_scripts =
turbo-hipster = turbo_hipster.worker_server:main
diff --git a/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py b/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py
index 5e96806..89a3c07 100644
--- a/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py
+++ b/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py
@@ -40,10 +40,11 @@
log = logging.getLogger("task_plugins.gate_real_db_upgrade.task.Runner")
- def __init__(self, config):
+ def __init__(self, global_config, plugin_config):
super(Runner, self).__init__()
self._stop = threading.Event()
- self.config = config
+ self.global_config = global_config
+ self.plugin_config = plugin_config
# Set up the runner worker
self.gearman_worker = None
@@ -65,16 +66,13 @@
self.log.debug("Set up real_db gearman worker")
self.gearman_worker = gear.Worker(__worker_name__)
self.gearman_worker.addServer(
- self.config['zuul_server']['gearman_host'],
- self.config['zuul_server']['gearman_port']
+ self.global_config['zuul_server']['gearman_host'],
+ self.global_config['zuul_server']['gearman_port']
)
self.register_functions()
def register_functions(self):
- """ Determine which functions to register based off available
- datasets """
- for dataset in self._get_datasets():
- self.gearman_worker.registerFunction(dataset['config']['gate'])
+ self.gearman_worker.registerFunction(self.plugin_config['gate'])
def stop(self):
self._stop.set()
@@ -160,7 +158,7 @@
index_url = handle_results.generate_push_results(
self.job_datasets,
self.job.unique,
- self.config['publish_logs']
+ self.global_config['publish_logs']
)
self.log.debug("Index URL found at %s" % index_url)
self.work_data['url'] = index_url
@@ -186,8 +184,7 @@
if len(self.datasets) > 0:
return self.datasets
- datasets_path = os.path.join(os.path.dirname(__file__),
- 'datasets')
+ datasets_path = self.global_config['datasets_dir']
for ent in os.listdir(datasets_path):
dataset_dir = os.path.join(datasets_path, ent)
if (os.path.isdir(dataset_dir) and os.path.isfile(
@@ -217,7 +214,7 @@
dataset['config']['project'] and
self._get_project_command(dataset['config']['type'])):
dataset['log_file_path'] = os.path.join(
- self.config['jobs_working_dir'],
+ self.global_config['jobs_working_dir'],
self.job.unique,
dataset['name'] + '.log'
)
@@ -262,7 +259,7 @@
% {
'unique_id': self.job.unique,
'job_working_dir': os.path.join(
- self.config['jobs_working_dir'],
+ self.global_config['jobs_working_dir'],
self.job.unique
),
'git_path': git_path,
@@ -277,7 +274,7 @@
dataset['dataset_dir'],
dataset['config']['logging_conf']
),
- 'pip_cache_dir': self.config['pip_download_cache']
+ 'pip_cache_dir': self.global_config['pip_download_cache']
}
)
@@ -285,13 +282,13 @@
syslog = '/var/log/syslog'
sqlslo = '/var/log/mysql/slow-queries.log'
sqlerr = '/var/log/mysql/error.log'
- if 'logs' in self.config:
- if 'syslog' in self.config['logs']:
- syslog = self.config['logs']['syslog']
- if 'sqlslo' in self.config['logs']:
- sqlslo = self.config['logs']['sqlslo']
- if 'sqlerr' in self.config['logs']:
- sqlerr = self.config['logs']['sqlerr']
+ if 'logs' in self.global_config:
+ if 'syslog' in self.global_config['logs']:
+ syslog = self.global_config['logs']['syslog']
+ if 'sqlslo' in self.global_config['logs']:
+ sqlslo = self.global_config['logs']['sqlslo']
+ if 'sqlerr' in self.global_config['logs']:
+ sqlerr = self.global_config['logs']['sqlerr']
utils.execute_to_log(
cmd,
@@ -309,9 +306,9 @@
self.log.debug("Grab the patchset we want to test against")
repo = utils.GitRepository(
- self.config['zuul_server']['git_url'] + project_name + '/.git',
+ self.global_config['zuul_server']['git_url'] + project_name + '/.git',
os.path.join(
- self.config['git_working_dir'],
+ self.global_config['git_working_dir'],
__worker_name__,
project_name
)
diff --git a/turbo_hipster/worker_server.py b/turbo_hipster/worker_server.py
index 5469773..325c4ac 100755
--- a/turbo_hipster/worker_server.py
+++ b/turbo_hipster/worker_server.py
@@ -67,19 +67,21 @@
""" Load the available plugins from task_plugins """
# Load plugins
for plugin in self.config['plugins']:
- print
- plugin_info = imp.find_module('task',
- [(os.path.dirname(
- os.path.realpath(__file__)) +
- '/task_plugins/' + plugin)])
- self.plugins.append(imp.load_module('task', *plugin_info))
+ self.plugins.append({
+ 'module': __import__('turbo_hipster.task_plugins.' +
+ plugin['name'],
+ fromlist='turbo_hipster.task_plugins'),
+ 'plugin_config': self.plugin
+ })
def run_tasks(self):
""" Run the tasks """
for plugin in self.plugins:
- self.tasks[plugin.__worker_name__] = plugin.Runner(self.config)
- self.tasks[plugin.__worker_name__].daemon = True
- self.tasks[plugin.__worker_name__].start()
+ module = plugin['module']
+ self.tasks[module.__worker_name__] = module.Runner(self.config,
+ plugin['plugin_config'])
+ self.tasks[module.__worker_name__].daemon = True
+ self.tasks[module.__worker_name__].start()
self.manager = worker_manager.GearmanManager(self.config, self.tasks)
self.manager.daemon = True