diff --git a/etc/status/public_html/app.js b/etc/status/public_html/app.js
deleted file mode 100644
index 8616201..0000000
--- a/etc/status/public_html/app.js
+++ /dev/null
@@ -1,917 +0,0 @@
-// Client script for Zuul status page
-//
-// Copyright 2012 OpenStack Foundation
-// Copyright 2013 Timo Tijhof
-// Copyright 2013 Wikimedia Foundation
-// Copyright 2014 Rackspace Australia
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License. You may obtain
-// a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations
-// under the License.
-'use strict';
-
-(function ($) {
-    var $container, $msg, $indicator, $queueInfo, $queueEventsNum,
-        $queueResultsNum, $pipelines, $jq;
-    var xhr, zuul,
-        zuul_graph_update_count = 0,
-        zuul_sparkline_urls = {},
-        current_filter = '',
-        demo = location.search.match(/[?&]demo=([^?&]*)/),
-        source_url = location.search.match(/[?&]source_url=([^?&]*)/),
-        source = demo ?
-            './status-' + (demo[1] || 'basic') + '.json-sample' :
-            'status.json';
-    source = source_url ? source_url[1] : source;
-
-    function set_cookie(name, value) {
-        document.cookie = name + '=' + value + '; path=/';
-    }
-
-    function read_cookie(name, default_value) {
-        var nameEQ = name + '=';
-        var ca = document.cookie.split(';');
-        for(var i=0;i < ca.length;i++) {
-            var c = ca[i];
-            while (c.charAt(0) === ' ') {
-                c = c.substring(1, c.length);
-            }
-            if (c.indexOf(nameEQ) === 0) {
-                return c.substring(nameEQ.length, c.length);
-            }
-        }
-        return default_value;
-    }
-
-
-    zuul = {
-        enabled: true,
-        graphite_url: '',
-        collapsed_exceptions: [],
-
-        schedule: function () {
-            if (!zuul.enabled) {
-                setTimeout(zuul.schedule, 5000);
-                return;
-            }
-            zuul.update().complete(function () {
-                setTimeout(zuul.schedule, 5000);
-            });
-
-            /* Only update graphs every minute */
-            if (zuul_graph_update_count > 11) {
-                zuul_graph_update_count = 0;
-                zuul.update_sparklines();
-            }
-        },
-
-        /** @return {jQuery.Promise} */
-        update: function () {
-            // Cancel the previous update if it hasn't completed yet.
-            if (xhr) {
-                xhr.abort();
-            }
-
-            zuul.emit('update-start');
-
-            xhr = $.getJSON(source)
-                .done(function (data) {
-                    if ('message' in data) {
-                        $msg.removeClass('alert-danger').addClass('alert-info');
-                        $msg.text(data.message);
-                        $msg.show();
-                    } else {
-                        $msg.empty();
-                        $msg.hide();
-                    }
-
-                    if ('zuul_version' in data) {
-                        $('#zuul-version-span').text(data.zuul_version);
-                    }
-                    if ('last_reconfigured' in data) {
-                        var last_reconfigured =
-                            new Date(data.last_reconfigured);
-                        $('#last-reconfigured-span').text(
-                            last_reconfigured.toString());
-                    }
-
-                    $pipelines.html('');
-                    $.each(data.pipelines, function (i, pipeline) {
-                        $pipelines.append(zuul.format.pipeline(pipeline));
-                    });
-
-                    $queueEventsNum.text(
-                        data.trigger_event_queue ?
-                            data.trigger_event_queue.length : '0'
-                    );
-                    $queueResultsNum.text(
-                        data.result_event_queue ?
-                            data.result_event_queue.length : '0'
-                    );
-                })
-                .fail(function (err, jqXHR, errMsg) {
-                    $msg.text(source + ': ' + errMsg).show();
-                    $msg.removeClass('zuul-msg-wrap-off');
-                })
-                .complete(function () {
-                    xhr = undefined;
-                    zuul.emit('update-end');
-                });
-
-            return xhr;
-        },
-
-        update_sparklines: function() {
-            $.each(zuul_sparkline_urls, function(name, url) {
-                var newimg = new Image();
-                var parts = url.split('#');
-                newimg.src = parts[0] + '#' + new Date().getTime();
-                $(newimg).load(function (x) {
-                    zuul_sparkline_urls[name] = newimg.src;
-                });
-            });
-        },
-
-        format: {
-            job: function(job) {
-                var $job_line = $('<span />');
-
-                if (job.url !== null) {
-                    $job_line.append(
-                        $('<a />')
-                            .addClass('zuul-job-name')
-                            .attr('href', job.url)
-                            .text(job.name)
-                    );
-                }
-                else {
-                    $job_line.append(
-                        $('<span />')
-                            .addClass('zuul-job-name')
-                            .text(job.name)
-                    );
-                }
-
-                $job_line.append(zuul.format.job_status(job));
-
-                if (job.voting === false) {
-                    $job_line.append(
-                        $(' <small />')
-                            .addClass('zuul-non-voting-desc')
-                            .text(' (non-voting)')
-                    );
-                }
-
-                return $job_line;
-            },
-
-            job_status: function(job) {
-                var result = job.result ? job.result.toLowerCase() : null;
-                if (result === null) {
-                    result = job.url ? 'in progress' : 'queued';
-                }
-
-                if (result === 'in progress') {
-                    return zuul.format.job_progress_bar(job.elapsed_time,
-                                                        job.remaining_time);
-                }
-                else {
-                    return zuul.format.status_label(result);
-                }
-            },
-
-            status_label: function(result) {
-                var $status = $('<span />');
-                $status.addClass('zuul-job-result label');
-
-                switch (result) {
-                    case 'success':
-                        $status.addClass('label-success');
-                        break;
-                    case 'failure':
-                        $status.addClass('label-danger');
-                        break;
-                    case 'unstable':
-                        $status.addClass('label-warning');
-                        break;
-                    case 'in progress':
-                    case 'queued':
-                    case 'lost':
-                        $status.addClass('label-default');
-                        break;
-                }
-                $status.text(result);
-                return $status;
-            },
-
-            job_progress_bar: function(elapsed_time, remaining_time) {
-                var progress_percent = 100 * (elapsed_time / (elapsed_time +
-                                                              remaining_time));
-                var $bar_inner = $('<div />')
-                    .addClass('progress-bar')
-                    .attr('role', 'progressbar')
-                    .attr('aria-valuenow', 'progressbar')
-                    .attr('aria-valuemin', progress_percent)
-                    .attr('aria-valuemin', '0')
-                    .attr('aria-valuemax', '100')
-                    .css('width', progress_percent + '%');
-
-                var $bar_outter = $('<div />')
-                    .addClass('progress zuul-job-result')
-                    .append($bar_inner);
-
-                return $bar_outter;
-            },
-
-            enqueue_time: function(ms) {
-                // Special format case for enqueue time to add style
-                var hours = 60 * 60 * 1000;
-                var now = Date.now();
-                var delta = now - ms;
-                var status = 'text-success';
-                var text = zuul.format.time(delta, true);
-                if (delta > (4 * hours)) {
-                    status = 'text-danger';
-                } else if (delta > (2 * hours)) {
-                    status = 'text-warning';
-                }
-                return '<span class="' + status + '">' + text + '</span>';
-            },
-
-            time: function(ms, words) {
-                if (typeof(words) === 'undefined') {
-                    words = false;
-                }
-                var seconds = (+ms)/1000;
-                var minutes = Math.floor(seconds/60);
-                var hours = Math.floor(minutes/60);
-                seconds = Math.floor(seconds % 60);
-                minutes = Math.floor(minutes % 60);
-                var r = '';
-                if (words) {
-                    if (hours) {
-                        r += hours;
-                        r += ' hr ';
-                    }
-                    r += minutes + ' min';
-                } else {
-                    if (hours < 10) {
-                        r += '0';
-                    }
-                    r += hours + ':';
-                    if (minutes < 10) {
-                        r += '0';
-                    }
-                    r += minutes + ':';
-                    if (seconds < 10) {
-                        r += '0';
-                    }
-                    r += seconds;
-                }
-                return r;
-            },
-
-            change_total_progress_bar: function(change) {
-                var job_percent = Math.floor(100 / change.jobs.length);
-                var $bar_outter = $('<div />')
-                    .addClass('progress zuul-change-total-result');
-
-                $.each(change.jobs, function (i, job) {
-                    var result = job.result ? job.result.toLowerCase() : null;
-                    if (result === null) {
-                        result = job.url ? 'in progress' : 'queued';
-                    }
-
-                    if (result !== 'queued') {
-                        var $bar_inner = $('<div />')
-                            .addClass('progress-bar');
-
-                        switch (result) {
-                            case 'success':
-                                $bar_inner.addClass('progress-bar-success');
-                                break;
-                            case 'lost':
-                            case 'failure':
-                                $bar_inner.addClass('progress-bar-danger');
-                                break;
-                            case 'unstable':
-                                $bar_inner.addClass('progress-bar-warning');
-                                break;
-                            case 'in progress':
-                            case 'queued':
-                                break;
-                        }
-                        $bar_inner.attr('title', job.name)
-                            .css('width', job_percent + '%');
-                        $bar_outter.append($bar_inner);
-                    }
-                });
-                return $bar_outter;
-            },
-
-            change_header: function(change) {
-                var change_id = change.id || 'NA';
-                if (change_id.length === 40) {
-                    change_id = change_id.substr(0, 7);
-                }
-
-                var $change_link = $('<small />');
-                if (change.url !== null) {
-                    $change_link.append(
-                        $('<a />').attr('href', change.url).text(change.id)
-                    );
-                }
-                else {
-                    $change_link.text(change_id);
-                }
-
-                var $change_progress_row_left = $('<div />')
-                    .addClass('col-xs-3')
-                    .append($change_link);
-                var $change_progress_row_right = $('<div />')
-                    .addClass('col-xs-9')
-                    .append(zuul.format.change_total_progress_bar(change));
-
-                var $change_progress_row = $('<div />')
-                    .addClass('row')
-                    .append($change_progress_row_left)
-                    .append($change_progress_row_right);
-
-                var $project_span = $('<span />')
-                    .addClass('change_project')
-                    .text(change.project);
-
-                var $left = $('<div />')
-                    .addClass('col-xs-8')
-                    .append($project_span, $change_progress_row);
-
-                var remaining_time = zuul.format.time(
-                        change.remaining_time, true);
-                var enqueue_time = zuul.format.enqueue_time(
-                        change.enqueue_time);
-                var $remaining_time = $('<small />').addClass('time')
-                    .attr('title', 'Remaining Time').html(remaining_time);
-                var $enqueue_time = $('<small />').addClass('time')
-                    .attr('title', 'Elapsed Time').html(enqueue_time);
-
-                var $right = $('<div />')
-                    .addClass('col-xs-4 text-right')
-                    .append($remaining_time, $('<br />'), $enqueue_time);
-
-                var $header = $('<div />')
-                    .addClass('row')
-                    .append($left, $right);
-                return $header;
-            },
-
-            change_list: function(jobs) {
-                var $list = $('<ul />')
-                    .addClass('list-group zuul-patchset-body');
-
-                $.each(jobs, function (i, job) {
-                    var $item = $('<li />')
-                        .addClass('list-group-item')
-                        .addClass('zuul-change-job')
-                        .append(zuul.format.job(job));
-                    $list.append($item);
-                });
-
-                return $list;
-            },
-
-            change_panel: function (change) {
-                var $header = $('<div />')
-                    .addClass('panel-heading zuul-patchset-header')
-                    .append(zuul.format.change_header(change));
-
-                var panel_id = change.id ? change.id.replace(',', '_')
-                                         : change.project.replace('/', '_') +
-                                           '-' + change.enqueue_time;
-                var $panel = $('<div />')
-                    .attr('id', panel_id)
-                    .addClass('panel panel-default zuul-change')
-                    .append($header)
-                    .append(zuul.format.change_list(change.jobs));
-
-                $header.click(zuul.toggle_patchset);
-                return $panel;
-            },
-
-            change_status_icon: function(change) {
-                var icon_name = 'green.png';
-                var icon_title = 'Succeeding';
-
-                if (change.active !== true) {
-                    // Grey icon
-                    icon_name = 'grey.png';
-                    icon_title = 'Waiting until closer to head of queue to' +
-                        ' start jobs';
-                }
-                else if (change.failing_reasons &&
-                         change.failing_reasons.length > 0) {
-                    var reason = change.failing_reasons.join(', ');
-                    icon_title = 'Failing because ' + reason;
-                    if (reason.match(/merge conflict/)) {
-                        // Black icon
-                        icon_name = 'black.png';
-                    }
-                    else {
-                        // Red icon
-                        icon_name = 'red.png';
-                    }
-                }
-
-                var $icon = $('<img />')
-                    .attr('src', 'images/' + icon_name)
-                    .attr('title', icon_title)
-                    .css('margin-top', '-6px');
-
-                return $icon;
-            },
-
-            change_with_status_tree: function(change, change_queue) {
-                var $change_row = $('<tr />');
-
-                for (var i = 0; i < change_queue._tree_columns; i++) {
-                    var $tree_cell  = $('<td />')
-                        .css('height', '100%')
-                        .css('padding', '0 0 10px 0')
-                        .css('margin', '0')
-                        .css('width', '16px')
-                        .css('min-width', '16px')
-                        .css('overflow', 'hidden')
-                        .css('vertical-align', 'top');
-
-                    if (i < change._tree.length && change._tree[i] !== null) {
-                        $tree_cell.css('background-image',
-                                       'url(\'images/line.png\')')
-                            .css('background-repeat', 'repeat-y');
-                    }
-
-                    if (i === change._tree_index) {
-                        $tree_cell.append(
-                            zuul.format.change_status_icon(change));
-                    }
-                    if (change._tree_branches.indexOf(i) !== -1) {
-                        var $image = $('<img />')
-                            .css('vertical-align', 'baseline');
-                        if (change._tree_branches.indexOf(i) ===
-                            change._tree_branches.length - 1) {
-                            // Angle line
-                            $image.attr('src', 'images/line-angle.png');
-                        }
-                        else {
-                            // T line
-                            $image.attr('src', 'images/line-t.png');
-                        }
-                        $tree_cell.append($image);
-                    }
-                    $change_row.append($tree_cell);
-                }
-
-                var change_width = 360 - 16*change_queue._tree_columns;
-                var $change_column = $('<td />')
-                    .css('width', change_width + 'px')
-                    .addClass('zuul-change-cell')
-                    .append(zuul.format.change_panel(change));
-
-                $change_row.append($change_column);
-
-                var $change_table = $('<table />')
-                    .addClass('zuul-change-box')
-                    .css('-moz-box-sizing', 'content-box')
-                    .css('box-sizing', 'content-box')
-                    .append($change_row);
-
-                return $change_table;
-            },
-
-            pipeline_sparkline: function(pipeline_name) {
-                if (zuul.graphite_url !== '') {
-                    var $sparkline = $('<img />')
-                        .addClass('pull-right')
-                        .attr('src', zuul.get_sparkline_url(pipeline_name));
-                    return $sparkline;
-                }
-                return false;
-            },
-
-            pipeline_header: function(pipeline, count) {
-                // Format the pipeline name, sparkline and description
-                var $header_div = $('<div />')
-                    .addClass('zuul-pipeline-header');
-
-                var $heading = $('<h3 />')
-                    .css('vertical-align', 'middle')
-                    .text(pipeline.name)
-                    .append(
-                        $('<span />')
-                            .addClass('badge pull-right')
-                            .css('vertical-align', 'middle')
-                            .css('margin-top', '0.5em')
-                            .text(count)
-                    )
-                    .append(zuul.format.pipeline_sparkline(pipeline.name));
-
-                $header_div.append($heading);
-
-                if (typeof pipeline.description === 'string') {
-                    $header_div.append(
-                        $('<p />').append(
-                            $('<small />').text(pipeline.description)
-                        )
-                    );
-                }
-                return $header_div;
-            },
-
-            pipeline: function (pipeline) {
-                var count = zuul.create_tree(pipeline);
-                var $html = $('<div />')
-                    .addClass('zuul-pipeline col-md-4')
-                    .append(zuul.format.pipeline_header(pipeline, count));
-
-                $.each(pipeline.change_queues,
-                       function (queue_i, change_queue) {
-                    $.each(change_queue.heads, function (head_i, changes) {
-                        if (pipeline.change_queues.length > 1 &&
-                            head_i === 0) {
-                            var name = change_queue.name;
-                            var short_name = name;
-                            if (short_name.length > 32) {
-                                short_name = short_name.substr(0, 32) + '...';
-                            }
-                            $html.append(
-                                $('<p />')
-                                    .text('Queue: ')
-                                    .append(
-                                        $('<abbr />')
-                                            .attr('title', name)
-                                            .text(short_name)
-                                    )
-                            );
-                        }
-
-                        $.each(changes, function (change_i, change) {
-                            var $change_box =
-                                zuul.format.change_with_status_tree(
-                                    change, change_queue);
-                            $html.append($change_box);
-                            zuul.display_patchset($change_box);
-                        });
-                    });
-                });
-                return $html;
-            },
-
-            filter_form_group: function() {
-                // Update the filter form with a clear button if required
-
-                var $label = $('<label />')
-                    .addClass('control-label')
-                    .attr('for', 'filter_string')
-                    .text('Filters')
-                    .css('padding-right', '0.5em');
-
-                var $input = $('<input />')
-                    .attr('type', 'text')
-                    .attr('id', 'filter_string')
-                    .addClass('form-control')
-                    .attr('title',
-                          'project(s), pipeline(s) or review(s) comma ' +
-                          'separated')
-                    .attr('value', current_filter);
-
-                $input.change(zuul.handle_filter_change);
-
-                var $clear_icon = $('<span />')
-                    .addClass('form-control-feedback')
-                    .addClass('glyphicon glyphicon-remove-circle')
-                    .attr('id', 'filter_form_clear_box')
-                    .attr('title', 'clear filter')
-                    .css('cursor', 'pointer');
-
-                $clear_icon.click(function() {
-                    $('#filter_string').val('').change();
-                });
-
-                if (current_filter === '') {
-                    $clear_icon.hide();
-                }
-
-                var $form_group = $('<div />')
-                    .addClass('form-group has-feedback')
-                    .append($label, $input, $clear_icon);
-                return $form_group;
-            },
-
-            expand_form_group: function() {
-                var expand_by_default = (
-                    read_cookie('zuul_expand_by_default', false) === 'true');
-
-                var $checkbox = $('<input />')
-                    .attr('type', 'checkbox')
-                    .attr('id', 'expand_by_default')
-                    .prop('checked', expand_by_default)
-                    .change(zuul.handle_expand_by_default);
-
-                var $label = $('<label />')
-                    .css('padding-left', '1em')
-                    .html('Expand by default: ')
-                    .append($checkbox);
-
-                var $form_group = $('<div />')
-                    .addClass('checkbox')
-                    .append($label);
-                return $form_group;
-            },
-
-            control_form: function() {
-                // Build the filter form filling anything from cookies
-
-                var $control_form = $('<form />')
-                    .attr('role', 'form')
-                    .addClass('form-inline')
-                    .submit(zuul.handle_filter_change);
-
-                $control_form
-                    .append(zuul.format.filter_form_group())
-                    .append(zuul.format.expand_form_group());
-
-                return $control_form;
-            },
-        },
-
-        emit: function () {
-            $jq.trigger.apply($jq, arguments);
-            return this;
-        },
-        on: function () {
-            $jq.on.apply($jq, arguments);
-            return this;
-        },
-        one: function () {
-            $jq.one.apply($jq, arguments);
-            return this;
-        },
-
-        toggle_patchset: function(e) {
-            // Toggle showing/hiding the patchset when the header is clicked
-            // Grab the patchset panel
-            var $panel = $(e.target).parents('.zuul-change');
-            var $body = $panel.children('.zuul-patchset-body');
-            $body.toggle(200);
-            var collapsed_index = zuul.collapsed_exceptions.indexOf(
-                $panel.attr('id'));
-            if (collapsed_index === -1 ) {
-                // Currently not an exception, add it to list
-                zuul.collapsed_exceptions.push($panel.attr('id'));
-            }
-            else {
-                // Currently an except, remove from exceptions
-                zuul.collapsed_exceptions.splice(collapsed_index, 1);
-            }
-        },
-
-        display_patchset: function($change_box, animate) {
-            // Determine if to show or hide the patchset and/or the results
-            // when loaded
-
-            // See if we should hide the body/results
-            var $panel = $change_box.find('.zuul-change');
-            var panel_change = $panel.attr('id');
-            var $body = $panel.children('.zuul-patchset-body');
-            var expand_by_default = $('#expand_by_default').prop('checked');
-
-            var collapsed_index = zuul.collapsed_exceptions.indexOf(panel_change);
-
-            if (expand_by_default && collapsed_index === -1 ||
-                !expand_by_default && collapsed_index !== -1) {
-                // Expand by default, or is an exception
-                $body.show(animate);
-            }
-            else {
-                $body.hide(animate);
-            }
-
-            // Check if we should hide the whole panel
-            var panel_project = $panel.find('.change_project').text()
-                .toLowerCase();
-
-
-            var panel_pipeline = $change_box
-                .parents('.zuul-pipeline')
-                .find('.zuul-pipeline-header > h3')
-                .html()
-                .toLowerCase();
-
-            if (current_filter !== '') {
-                var show_panel = false;
-                var filter = current_filter.trim().split(/[\s,]+/);
-                $.each(filter, function(index, f_val) {
-                    if (f_val !== '') {
-                        f_val = f_val.toLowerCase();
-                        if (panel_project.indexOf(f_val) !== -1 ||
-                            panel_pipeline.indexOf(f_val) !== -1 ||
-                            panel_change.indexOf(f_val) !== -1) {
-                            show_panel = true;
-                        }
-                    }
-                });
-                if (show_panel === true) {
-                    $change_box.show(animate);
-                }
-                else {
-                    $change_box.hide(animate);
-                }
-            }
-            else {
-                $change_box.show(animate);
-            }
-        },
-
-        handle_filter_change: function() {
-            // Update the filter and save it to a cookie
-            current_filter = $('#filter_string').val();
-            set_cookie('zuul_filter_string', current_filter);
-            if (current_filter === '') {
-                $('#filter_form_clear_box').hide();
-            }
-            else {
-                $('#filter_form_clear_box').show();
-            }
-
-            $('.zuul-change-box').each(function(index, obj) {
-                var $change_box = $(obj);
-                zuul.display_patchset($change_box, 200);
-            });
-            return false;
-        },
-
-        handle_expand_by_default: function(e) {
-            // Handle toggling expand by default
-            set_cookie('zuul_expand_by_default', e.target.checked);
-            zuul.collapsed_exceptions = [];
-            $('.zuul-change-box').each(function(index, obj) {
-                var $change_box = $(obj);
-                zuul.display_patchset($change_box, 200);
-            });
-        },
-
-        create_tree: function(pipeline) {
-            var count = 0;
-            var pipeline_max_tree_columns = 1;
-            $.each(pipeline.change_queues, function(change_queue_i,
-                                                       change_queue) {
-                var tree = [];
-                var max_tree_columns = 1;
-                var changes = [];
-                var last_tree_length = 0;
-                $.each(change_queue.heads, function(head_i, head) {
-                    $.each(head, function(change_i, change) {
-                        changes[change.id] = change;
-                        change._tree_position = change_i;
-                    });
-                });
-                $.each(change_queue.heads, function(head_i, head) {
-                    $.each(head, function(change_i, change) {
-                        count += 1;
-                        var idx = tree.indexOf(change.id);
-                        if (idx > -1) {
-                            change._tree_index = idx;
-                            // remove...
-                            tree[idx] = null;
-                            while (tree[tree.length - 1] === null) {
-                                tree.pop();
-                            }
-                        } else {
-                            change._tree_index = 0;
-                        }
-                        change._tree_branches = [];
-                        change._tree = [];
-                        if (typeof(change.items_behind) === 'undefined') {
-                            change.items_behind = [];
-                        }
-                        change.items_behind.sort(function(a, b) {
-                            return (changes[b]._tree_position -
-                                    changes[a]._tree_position);
-                        });
-                        $.each(change.items_behind, function(i, id) {
-                            tree.push(id);
-                            if (tree.length>last_tree_length &&
-                                last_tree_length > 0) {
-                                change._tree_branches.push(
-                                    tree.length - 1);
-                            }
-                        });
-                        if (tree.length > max_tree_columns) {
-                            max_tree_columns = tree.length;
-                        }
-                        if (tree.length > pipeline_max_tree_columns) {
-                            pipeline_max_tree_columns = tree.length;
-                        }
-                        change._tree = tree.slice(0);  // make a copy
-                        last_tree_length = tree.length;
-                    });
-                });
-                change_queue._tree_columns = max_tree_columns;
-            });
-            pipeline._tree_columns = pipeline_max_tree_columns;
-            return count;
-        },
-
-        get_sparkline_url: function(pipeline_name) {
-            if (zuul.graphite_url !== '') {
-                if (!(pipeline_name in zuul_sparkline_urls)) {
-                    zuul_sparkline_urls[pipeline_name] = $.fn.graphite.geturl({
-                        url: zuul.graphite_url,
-                        from: "-8hours",
-                        width: 100,
-                        height: 26,
-                        margin: 0,
-                        hideLegend: true,
-                        hideAxes: true,
-                        hideGrid: true,
-                        target: [
-                            "color(stats.gauges.zuul.pipeline."+pipeline_name+".current_changes, '6b8182')"
-                        ]
-                    });
-                }
-                return zuul_sparkline_urls[pipeline_name];
-            }
-            return false;
-        },
-    };
-
-    current_filter = read_cookie('zuul_filter_string', current_filter);
-
-    $jq = $(zuul);
-
-    $jq.on('update-start', function () {
-        $container.addClass('zuul-container-loading');
-        $indicator.addClass('zuul-spinner-on');
-    });
-
-    $jq.on('update-end', function () {
-        $container.removeClass('zuul-container-loading');
-        setTimeout(function () {
-            $indicator.removeClass('zuul-spinner-on');
-        }, 500);
-    });
-
-    $jq.one('update-end', function () {
-        // Do this asynchronous so that if the first update adds a message, it
-        // will not animate while we fade in the content. Instead it simply
-        // appears with the rest of the content.
-        setTimeout(function () {
-            // Fade in the content
-            $container.addClass('zuul-container-ready');
-        });
-    });
-
-    $(function ($) {
-        $msg = $('<div />').addClass('alert').hide();
-        $indicator = $('<button class="btn pull-right zuul-spinner">' +
-                       'updating ' +
-                       '<span class="glyphicon glyphicon-refresh"></span>' +
-                       '</button>');
-        $queueInfo = $('<p>Queue lengths: <span>0</span> events, ' +
-                       '<span>0</span> results.</p>');
-        $queueEventsNum = $queueInfo.find('span').eq(0);
-        $queueResultsNum = $queueEventsNum.next();
-
-        var $control_form = zuul.format.control_form();
-
-        $pipelines = $('<div class="row"></div>');
-        var $zuulVersion = $('<p>Zuul version: <span id="zuul-version-span">' +
-                         '</span></p>');
-        var $lastReconf = $('<p>Last reconfigured: ' +
-                        '<span id="last-reconfigured-span"></span></p>');
-
-        $container = $('#zuul-container').append($msg, $indicator,
-                                                 $queueInfo, $control_form,
-                                                 $pipelines, $zuulVersion,
-                                                 $lastReconf);
-
-        //zuul.graphite_url = 'http://graphite.openstack.org/render/'
-        zuul.schedule();
-
-        $(document).on({
-            'show.visibility': function () {
-                zuul.enabled = true;
-                zuul.update();
-            },
-            'hide.visibility': function () {
-                zuul.enabled = false;
-            }
-        });
-    });
-}(jQuery));
diff --git a/etc/status/public_html/index.html b/etc/status/public_html/index.html
index bea1a79..d77470b 100644
--- a/etc/status/public_html/index.html
+++ b/etc/status/public_html/index.html
@@ -24,15 +24,17 @@
     <link rel="stylesheet" href="styles/zuul.css" />
 </head>
 <body>
-    <div class="container">
-        <h1>Zuul Status</h1>
-        <p>Real-time status monitor of Zuul, the pipeline manager between Gerrit and Workers.</p>
 
-        <div class="zuul-container" id="zuul-container"></div>
-    </div>
+    <div id="zuul_container"></div>
+
     <script src="jquery.min.js"></script>
     <script src="jquery-visibility.min.js"></script>
     <script src="jquery.graphite.js"></script>
-    <script src="app.js"></script>
+    <script src="jquery.zuul.js"></script>
+    <script src="zuul.app.js"></script>
+    <script>
+        zuul_build_dom(jQuery, '#zuul_container');
+        zuul_start(jQuery);
+    </script>
 </body>
 </html>
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
new file mode 100644
index 0000000..9a3c1fa
--- /dev/null
+++ b/etc/status/public_html/jquery.zuul.js
@@ -0,0 +1,877 @@
+// jquery plugin for Zuul status page
+//
+// Copyright 2012 OpenStack Foundation
+// Copyright 2013 Timo Tijhof
+// Copyright 2013 Wikimedia Foundation
+// Copyright 2014 Rackspace Australia
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+'use strict';
+
+(function ($) {
+    function set_cookie(name, value) {
+        document.cookie = name + '=' + value + '; path=/';
+    }
+
+    function read_cookie(name, default_value) {
+        var nameEQ = name + '=';
+        var ca = document.cookie.split(';');
+        for(var i=0;i < ca.length;i++) {
+            var c = ca[i];
+            while (c.charAt(0) === ' ') {
+                c = c.substring(1, c.length);
+            }
+            if (c.indexOf(nameEQ) === 0) {
+                return c.substring(nameEQ.length, c.length);
+            }
+        }
+        return default_value;
+    }
+
+    $.zuul = function(options) {
+        var options = $.extend({
+            'enabled': true,
+            'graphite_url': '',
+            'source': 'status.json',
+            'msg_id': '#zuul_msg',
+            'pipelines_id': '#zuul_pipelines',
+            'queue_events_num': '#zuul_queue_events_num',
+            'queue_results_num': '#zuul_queue_results_num',
+        }, options);
+
+        var collapsed_exceptions = [];
+        var current_filter = read_cookie('zuul_filter_string', current_filter);
+        var $jq;
+
+        var xhr,
+            zuul_graph_update_count = 0,
+            zuul_sparkline_urls = {};
+
+        function get_sparkline_url(pipeline_name) {
+            if (options.graphite_url !== '') {
+                if (!(pipeline_name in zuul_sparkline_urls)) {
+                    zuul_sparkline_urls[pipeline_name] = $.fn.graphite
+                        .geturl({
+                        url: options.graphite_url,
+                        from: "-8hours",
+                        width: 100,
+                        height: 26,
+                        margin: 0,
+                        hideLegend: true,
+                        hideAxes: true,
+                        hideGrid: true,
+                        target: [
+                            "color(stats.gauges.zuul.pipeline." + pipeline_name
+                            + ".current_changes, '6b8182')"
+                        ]
+                    });
+                }
+                return zuul_sparkline_urls[pipeline_name];
+            }
+            return false;
+        }
+
+        var format = {
+            job: function(job) {
+                var $job_line = $('<span />');
+
+                if (job.url !== null) {
+                    $job_line.append(
+                        $('<a />')
+                            .addClass('zuul-job-name')
+                            .attr('href', job.url)
+                            .text(job.name)
+                    );
+                }
+                else {
+                    $job_line.append(
+                        $('<span />')
+                            .addClass('zuul-job-name')
+                            .text(job.name)
+                    );
+                }
+
+                $job_line.append(this.job_status(job));
+
+                if (job.voting === false) {
+                    $job_line.append(
+                        $(' <small />')
+                            .addClass('zuul-non-voting-desc')
+                            .text(' (non-voting)')
+                    );
+                }
+
+                return $job_line;
+            },
+
+            job_status: function(job) {
+                var result = job.result ? job.result.toLowerCase() : null;
+                if (result === null) {
+                    result = job.url ? 'in progress' : 'queued';
+                }
+
+                if (result === 'in progress') {
+                    return this.job_progress_bar(job.elapsed_time,
+                                                        job.remaining_time);
+                }
+                else {
+                    return this.status_label(result);
+                }
+            },
+
+            status_label: function(result) {
+                var $status = $('<span />');
+                $status.addClass('zuul-job-result label');
+
+                switch (result) {
+                    case 'success':
+                        $status.addClass('label-success');
+                        break;
+                    case 'failure':
+                        $status.addClass('label-danger');
+                        break;
+                    case 'unstable':
+                        $status.addClass('label-warning');
+                        break;
+                    case 'in progress':
+                    case 'queued':
+                    case 'lost':
+                        $status.addClass('label-default');
+                        break;
+                }
+                $status.text(result);
+                return $status;
+            },
+
+            job_progress_bar: function(elapsed_time, remaining_time) {
+                var progress_percent = 100 * (elapsed_time / (elapsed_time +
+                                                              remaining_time));
+                var $bar_inner = $('<div />')
+                    .addClass('progress-bar')
+                    .attr('role', 'progressbar')
+                    .attr('aria-valuenow', 'progressbar')
+                    .attr('aria-valuemin', progress_percent)
+                    .attr('aria-valuemin', '0')
+                    .attr('aria-valuemax', '100')
+                    .css('width', progress_percent + '%');
+
+                var $bar_outter = $('<div />')
+                    .addClass('progress zuul-job-result')
+                    .append($bar_inner);
+
+                return $bar_outter;
+            },
+
+            enqueue_time: function(ms) {
+                // Special format case for enqueue time to add style
+                var hours = 60 * 60 * 1000;
+                var now = Date.now();
+                var delta = now - ms;
+                var status = 'text-success';
+                var text = this.time(delta, true);
+                if (delta > (4 * hours)) {
+                    status = 'text-danger';
+                } else if (delta > (2 * hours)) {
+                    status = 'text-warning';
+                }
+                return '<span class="' + status + '">' + text + '</span>';
+            },
+
+            time: function(ms, words) {
+                if (typeof(words) === 'undefined') {
+                    words = false;
+                }
+                var seconds = (+ms)/1000;
+                var minutes = Math.floor(seconds/60);
+                var hours = Math.floor(minutes/60);
+                seconds = Math.floor(seconds % 60);
+                minutes = Math.floor(minutes % 60);
+                var r = '';
+                if (words) {
+                    if (hours) {
+                        r += hours;
+                        r += ' hr ';
+                    }
+                    r += minutes + ' min';
+                } else {
+                    if (hours < 10) {
+                        r += '0';
+                    }
+                    r += hours + ':';
+                    if (minutes < 10) {
+                        r += '0';
+                    }
+                    r += minutes + ':';
+                    if (seconds < 10) {
+                        r += '0';
+                    }
+                    r += seconds;
+                }
+                return r;
+            },
+
+            change_total_progress_bar: function(change) {
+                var job_percent = Math.floor(100 / change.jobs.length);
+                var $bar_outter = $('<div />')
+                    .addClass('progress zuul-change-total-result');
+
+                $.each(change.jobs, function (i, job) {
+                    var result = job.result ? job.result.toLowerCase() : null;
+                    if (result === null) {
+                        result = job.url ? 'in progress' : 'queued';
+                    }
+
+                    if (result !== 'queued') {
+                        var $bar_inner = $('<div />')
+                            .addClass('progress-bar');
+
+                        switch (result) {
+                            case 'success':
+                                $bar_inner.addClass('progress-bar-success');
+                                break;
+                            case 'lost':
+                            case 'failure':
+                                $bar_inner.addClass('progress-bar-danger');
+                                break;
+                            case 'unstable':
+                                $bar_inner.addClass('progress-bar-warning');
+                                break;
+                            case 'in progress':
+                            case 'queued':
+                                break;
+                        }
+                        $bar_inner.attr('title', job.name)
+                            .css('width', job_percent + '%');
+                        $bar_outter.append($bar_inner);
+                    }
+                });
+                return $bar_outter;
+            },
+
+            change_header: function(change) {
+                var change_id = change.id || 'NA';
+                if (change_id.length === 40) {
+                    change_id = change_id.substr(0, 7);
+                }
+
+                var $change_link = $('<small />');
+                if (change.url !== null) {
+                    $change_link.append(
+                        $('<a />').attr('href', change.url).text(change.id)
+                    );
+                }
+                else {
+                    $change_link.text(change_id);
+                }
+
+                var $change_progress_row_left = $('<div />')
+                    .addClass('col-xs-3')
+                    .append($change_link);
+                var $change_progress_row_right = $('<div />')
+                    .addClass('col-xs-9')
+                    .append(this.change_total_progress_bar(change));
+
+                var $change_progress_row = $('<div />')
+                    .addClass('row')
+                    .append($change_progress_row_left)
+                    .append($change_progress_row_right);
+
+                var $project_span = $('<span />')
+                    .addClass('change_project')
+                    .text(change.project);
+
+                var $left = $('<div />')
+                    .addClass('col-xs-8')
+                    .append($project_span, $change_progress_row);
+
+                var remaining_time = this.time(
+                        change.remaining_time, true);
+                var enqueue_time = this.enqueue_time(
+                        change.enqueue_time);
+                var $remaining_time = $('<small />').addClass('time')
+                    .attr('title', 'Remaining Time').html(remaining_time);
+                var $enqueue_time = $('<small />').addClass('time')
+                    .attr('title', 'Elapsed Time').html(enqueue_time);
+
+                var $right = $('<div />')
+                    .addClass('col-xs-4 text-right')
+                    .append($remaining_time, $('<br />'), $enqueue_time);
+
+                var $header = $('<div />')
+                    .addClass('row')
+                    .append($left, $right);
+                return $header;
+            },
+
+            change_list: function(jobs) {
+                var format = this;
+                var $list = $('<ul />')
+                    .addClass('list-group zuul-patchset-body');
+
+                $.each(jobs, function (i, job) {
+                    var $item = $('<li />')
+                        .addClass('list-group-item')
+                        .addClass('zuul-change-job')
+                        .append(format.job(job));
+                    $list.append($item);
+                });
+
+                return $list;
+            },
+
+            change_panel: function (change) {
+                var $header = $('<div />')
+                    .addClass('panel-heading zuul-patchset-header')
+                    .append(this.change_header(change));
+
+                var panel_id = change.id ? change.id.replace(',', '_')
+                                         : change.project.replace('/', '_') +
+                                           '-' + change.enqueue_time;
+                var $panel = $('<div />')
+                    .attr('id', panel_id)
+                    .addClass('panel panel-default zuul-change')
+                    .append($header)
+                    .append(this.change_list(change.jobs));
+
+                $header.click(this.toggle_patchset);
+                return $panel;
+            },
+
+            change_status_icon: function(change) {
+                var icon_name = 'green.png';
+                var icon_title = 'Succeeding';
+
+                if (change.active !== true) {
+                    // Grey icon
+                    icon_name = 'grey.png';
+                    icon_title = 'Waiting until closer to head of queue to' +
+                        ' start jobs';
+                }
+                else if (change.failing_reasons &&
+                         change.failing_reasons.length > 0) {
+                    var reason = change.failing_reasons.join(', ');
+                    icon_title = 'Failing because ' + reason;
+                    if (reason.match(/merge conflict/)) {
+                        // Black icon
+                        icon_name = 'black.png';
+                    }
+                    else {
+                        // Red icon
+                        icon_name = 'red.png';
+                    }
+                }
+
+                var $icon = $('<img />')
+                    .attr('src', 'images/' + icon_name)
+                    .attr('title', icon_title)
+                    .css('margin-top', '-6px');
+
+                return $icon;
+            },
+
+            change_with_status_tree: function(change, change_queue) {
+                var $change_row = $('<tr />');
+
+                for (var i = 0; i < change_queue._tree_columns; i++) {
+                    var $tree_cell  = $('<td />')
+                        .css('height', '100%')
+                        .css('padding', '0 0 10px 0')
+                        .css('margin', '0')
+                        .css('width', '16px')
+                        .css('min-width', '16px')
+                        .css('overflow', 'hidden')
+                        .css('vertical-align', 'top');
+
+                    if (i < change._tree.length && change._tree[i] !== null) {
+                        $tree_cell.css('background-image',
+                                       'url(\'images/line.png\')')
+                            .css('background-repeat', 'repeat-y');
+                    }
+
+                    if (i === change._tree_index) {
+                        $tree_cell.append(
+                            this.change_status_icon(change));
+                    }
+                    if (change._tree_branches.indexOf(i) !== -1) {
+                        var $image = $('<img />')
+                            .css('vertical-align', 'baseline');
+                        if (change._tree_branches.indexOf(i) ===
+                            change._tree_branches.length - 1) {
+                            // Angle line
+                            $image.attr('src', 'images/line-angle.png');
+                        }
+                        else {
+                            // T line
+                            $image.attr('src', 'images/line-t.png');
+                        }
+                        $tree_cell.append($image);
+                    }
+                    $change_row.append($tree_cell);
+                }
+
+                var change_width = 360 - 16*change_queue._tree_columns;
+                var $change_column = $('<td />')
+                    .css('width', change_width + 'px')
+                    .addClass('zuul-change-cell')
+                    .append(this.change_panel(change));
+
+                $change_row.append($change_column);
+
+                var $change_table = $('<table />')
+                    .addClass('zuul-change-box')
+                    .css('-moz-box-sizing', 'content-box')
+                    .css('box-sizing', 'content-box')
+                    .append($change_row);
+
+                return $change_table;
+            },
+
+            pipeline_sparkline: function(pipeline_name) {
+                if (options.graphite_url !== '') {
+                    var $sparkline = $('<img />')
+                        .addClass('pull-right')
+                        .attr('src', get_sparkline_url(pipeline_name));
+                    return $sparkline;
+                }
+                return false;
+            },
+
+            pipeline_header: function(pipeline, count) {
+                // Format the pipeline name, sparkline and description
+                var $header_div = $('<div />')
+                    .addClass('zuul-pipeline-header');
+
+                var $heading = $('<h3 />')
+                    .css('vertical-align', 'middle')
+                    .text(pipeline.name)
+                    .append(
+                        $('<span />')
+                            .addClass('badge pull-right')
+                            .css('vertical-align', 'middle')
+                            .css('margin-top', '0.5em')
+                            .text(count)
+                    )
+                    .append(this.pipeline_sparkline(pipeline.name));
+
+                $header_div.append($heading);
+
+                if (typeof pipeline.description === 'string') {
+                    $header_div.append(
+                        $('<p />').append(
+                            $('<small />').text(pipeline.description)
+                        )
+                    );
+                }
+                return $header_div;
+            },
+
+            pipeline: function (pipeline, count) {
+                var format = this;
+                var $html = $('<div />')
+                    .addClass('zuul-pipeline col-md-4')
+                    .append(this.pipeline_header(pipeline, count));
+
+                $.each(pipeline.change_queues,
+                       function (queue_i, change_queue) {
+                    $.each(change_queue.heads, function (head_i, changes) {
+                        if (pipeline.change_queues.length > 1 &&
+                            head_i === 0) {
+                            var name = change_queue.name;
+                            var short_name = name;
+                            if (short_name.length > 32) {
+                                short_name = short_name.substr(0, 32) + '...';
+                            }
+                            $html.append(
+                                $('<p />')
+                                    .text('Queue: ')
+                                    .append(
+                                        $('<abbr />')
+                                            .attr('title', name)
+                                            .text(short_name)
+                                    )
+                            );
+                        }
+
+                        $.each(changes, function (change_i, change) {
+                            var $change_box =
+                                format.change_with_status_tree(
+                                    change, change_queue);
+                            $html.append($change_box);
+                            format.display_patchset($change_box);
+                        });
+                    });
+                });
+                return $html;
+            },
+
+            toggle_patchset: function(e) {
+                // Toggle showing/hiding the patchset when the header is
+                // clicked.
+
+                // Grab the patchset panel
+                var $panel = $(e.target).parents('.zuul-change');
+                var $body = $panel.children('.zuul-patchset-body');
+                $body.toggle(200);
+                var collapsed_index = collapsed_exceptions.indexOf(
+                    $panel.attr('id'));
+                if (collapsed_index === -1 ) {
+                    // Currently not an exception, add it to list
+                    collapsed_exceptions.push($panel.attr('id'));
+                }
+                else {
+                    // Currently an except, remove from exceptions
+                    collapsed_exceptions.splice(collapsed_index, 1);
+                }
+            },
+
+            display_patchset: function($change_box, animate) {
+                // Determine if to show or hide the patchset and/or the results
+                // when loaded
+
+                // See if we should hide the body/results
+                var $panel = $change_box.find('.zuul-change');
+                var panel_change = $panel.attr('id');
+                var $body = $panel.children('.zuul-patchset-body');
+                var expand_by_default = $('#expand_by_default')
+                    .prop('checked');
+
+                var collapsed_index = collapsed_exceptions
+                    .indexOf(panel_change);
+
+                if (expand_by_default && collapsed_index === -1 ||
+                    !expand_by_default && collapsed_index !== -1) {
+                    // Expand by default, or is an exception
+                    $body.show(animate);
+                }
+                else {
+                    $body.hide(animate);
+                }
+
+                // Check if we should hide the whole panel
+                var panel_project = $panel.find('.change_project').text()
+                    .toLowerCase();
+
+
+                var panel_pipeline = $change_box
+                    .parents('.zuul-pipeline')
+                    .find('.zuul-pipeline-header > h3')
+                    .html()
+                    .toLowerCase();
+
+                if (current_filter !== '') {
+                    var show_panel = false;
+                    var filter = current_filter.trim().split(/[\s,]+/);
+                    $.each(filter, function(index, f_val) {
+                        if (f_val !== '') {
+                            f_val = f_val.toLowerCase();
+                            if (panel_project.indexOf(f_val) !== -1 ||
+                                panel_pipeline.indexOf(f_val) !== -1 ||
+                                panel_change.indexOf(f_val) !== -1) {
+                                show_panel = true;
+                            }
+                        }
+                    });
+                    if (show_panel === true) {
+                        $change_box.show(animate);
+                    }
+                    else {
+                        $change_box.hide(animate);
+                    }
+                }
+                else {
+                    $change_box.show(animate);
+                }
+            },
+        };
+
+        var app = {
+            schedule: function (app) {
+                var app = app || this;
+                if (!options.enabled) {
+                    setTimeout(function() {app.schedule(app);}, 5000);
+                    return;
+                }
+                app.update().complete(function () {
+                    setTimeout(function() {app.schedule(app);}, 5000);
+                });
+
+                /* Only update graphs every minute */
+                if (zuul_graph_update_count > 11) {
+                    zuul_graph_update_count = 0;
+                    zuul.update_sparklines();
+                }
+            },
+
+            /** @return {jQuery.Promise} */
+            update: function () {
+                // Cancel the previous update if it hasn't completed yet.
+                if (xhr) {
+                    xhr.abort();
+                }
+
+                this.emit('update-start');
+                var app = this;
+
+                var $msg = $(options.msg_id)
+                xhr = $.getJSON(options.source)
+                    .done(function (data) {
+                        if ('message' in data) {
+                            $msg.removeClass('alert-danger')
+                                .addClass('alert-info')
+                                .text(data.message)
+                                .show();
+                        } else {
+                            $msg.empty()
+                                .hide();
+                        }
+
+                        if ('zuul_version' in data) {
+                            $('#zuul-version-span').text(data.zuul_version);
+                        }
+                        if ('last_reconfigured' in data) {
+                            var last_reconfigured =
+                                new Date(data.last_reconfigured);
+                            $('#last-reconfigured-span').text(
+                                last_reconfigured.toString());
+                        }
+
+                        var $pipelines = $(options.pipelines_id);
+                        $pipelines.html('');
+                        $.each(data.pipelines, function (i, pipeline) {
+                            var count = app.create_tree(pipeline);
+                            $pipelines.append(
+                                format.pipeline(pipeline, count));
+                        });
+
+                        $(options.queue_events_num).text(
+                            data.trigger_event_queue ?
+                                data.trigger_event_queue.length : '0'
+                        );
+                        $(options.queue_results_num).text(
+                            data.result_event_queue ?
+                                data.result_event_queue.length : '0'
+                        );
+                    })
+                    .fail(function (err, jqXHR, errMsg) {
+                        $msg.text(source + ': ' + errMsg).show();
+                        $msg.removeClass('zuul-msg-wrap-off');
+                    })
+                    .complete(function () {
+                        xhr = undefined;
+                        app.emit('update-end');
+                    });
+
+                return xhr;
+            },
+
+            update_sparklines: function() {
+                $.each(zuul_sparkline_urls, function(name, url) {
+                    var newimg = new Image();
+                    var parts = url.split('#');
+                    newimg.src = parts[0] + '#' + new Date().getTime();
+                    $(newimg).load(function (x) {
+                        zuul_sparkline_urls[name] = newimg.src;
+                    });
+                });
+            },
+
+            emit: function () {
+                $jq.trigger.apply($jq, arguments);
+                return this;
+            },
+            on: function () {
+                $jq.on.apply($jq, arguments);
+                return this;
+            },
+            one: function () {
+                $jq.one.apply($jq, arguments);
+                return this;
+            },
+
+            control_form: function() {
+                // Build the filter form filling anything from cookies
+
+                var $control_form = $('<form />')
+                    .attr('role', 'form')
+                    .addClass('form-inline')
+                    .submit(this.handle_filter_change);
+
+                $control_form
+                    .append(this.filter_form_group())
+                    .append(this.expand_form_group());
+
+                return $control_form;
+            },
+
+            filter_form_group: function() {
+                // Update the filter form with a clear button if required
+
+                var $label = $('<label />')
+                    .addClass('control-label')
+                    .attr('for', 'filter_string')
+                    .text('Filters')
+                    .css('padding-right', '0.5em');
+
+                var $input = $('<input />')
+                    .attr('type', 'text')
+                    .attr('id', 'filter_string')
+                    .addClass('form-control')
+                    .attr('title',
+                          'project(s), pipeline(s) or review(s) comma ' +
+                          'separated')
+                    .attr('value', current_filter);
+
+                $input.change(this.handle_filter_change);
+
+                var $clear_icon = $('<span />')
+                    .addClass('form-control-feedback')
+                    .addClass('glyphicon glyphicon-remove-circle')
+                    .attr('id', 'filter_form_clear_box')
+                    .attr('title', 'clear filter')
+                    .css('cursor', 'pointer');
+
+                $clear_icon.click(function() {
+                    $('#filter_string').val('').change();
+                });
+
+                if (current_filter === '') {
+                    $clear_icon.hide();
+                }
+
+                var $form_group = $('<div />')
+                    .addClass('form-group has-feedback')
+                    .append($label, $input, $clear_icon);
+                return $form_group;
+            },
+
+            expand_form_group: function() {
+                var expand_by_default = (
+                    read_cookie('zuul_expand_by_default', false) === 'true');
+
+                var $checkbox = $('<input />')
+                    .attr('type', 'checkbox')
+                    .attr('id', 'expand_by_default')
+                    .prop('checked', expand_by_default)
+                    .change(this.handle_expand_by_default);
+
+                var $label = $('<label />')
+                    .css('padding-left', '1em')
+                    .html('Expand by default: ')
+                    .append($checkbox);
+
+                var $form_group = $('<div />')
+                    .addClass('checkbox')
+                    .append($label);
+                return $form_group;
+            },
+
+            handle_filter_change: function() {
+                // Update the filter and save it to a cookie
+                current_filter = $('#filter_string').val();
+                set_cookie('zuul_filter_string', current_filter);
+                if (current_filter === '') {
+                    $('#filter_form_clear_box').hide();
+                }
+                else {
+                    $('#filter_form_clear_box').show();
+                }
+
+                $('.zuul-change-box').each(function(index, obj) {
+                    var $change_box = $(obj);
+                    format.display_patchset($change_box, 200);
+                });
+                return false;
+            },
+
+            handle_expand_by_default: function(e) {
+                // Handle toggling expand by default
+                set_cookie('zuul_expand_by_default', e.target.checked);
+                collapsed_exceptions = [];
+                $('.zuul-change-box').each(function(index, obj) {
+                    var $change_box = $(obj);
+                    format.display_patchset($change_box, 200);
+                });
+            },
+
+            create_tree: function(pipeline) {
+                var count = 0;
+                var pipeline_max_tree_columns = 1;
+                $.each(pipeline.change_queues, function(change_queue_i,
+                                                           change_queue) {
+                    var tree = [];
+                    var max_tree_columns = 1;
+                    var changes = [];
+                    var last_tree_length = 0;
+                    $.each(change_queue.heads, function(head_i, head) {
+                        $.each(head, function(change_i, change) {
+                            changes[change.id] = change;
+                            change._tree_position = change_i;
+                        });
+                    });
+                    $.each(change_queue.heads, function(head_i, head) {
+                        $.each(head, function(change_i, change) {
+                            count += 1;
+                            var idx = tree.indexOf(change.id);
+                            if (idx > -1) {
+                                change._tree_index = idx;
+                                // remove...
+                                tree[idx] = null;
+                                while (tree[tree.length - 1] === null) {
+                                    tree.pop();
+                                }
+                            } else {
+                                change._tree_index = 0;
+                            }
+                            change._tree_branches = [];
+                            change._tree = [];
+                            if (typeof(change.items_behind) === 'undefined') {
+                                change.items_behind = [];
+                            }
+                            change.items_behind.sort(function(a, b) {
+                                return (changes[b]._tree_position -
+                                        changes[a]._tree_position);
+                            });
+                            $.each(change.items_behind, function(i, id) {
+                                tree.push(id);
+                                if (tree.length>last_tree_length &&
+                                    last_tree_length > 0) {
+                                    change._tree_branches.push(
+                                        tree.length - 1);
+                                }
+                            });
+                            if (tree.length > max_tree_columns) {
+                                max_tree_columns = tree.length;
+                            }
+                            if (tree.length > pipeline_max_tree_columns) {
+                                pipeline_max_tree_columns = tree.length;
+                            }
+                            change._tree = tree.slice(0);  // make a copy
+                            last_tree_length = tree.length;
+                        });
+                    });
+                    change_queue._tree_columns = max_tree_columns;
+                });
+                pipeline._tree_columns = pipeline_max_tree_columns;
+                return count;
+            },
+        };
+
+        $jq = $(app);
+        return {
+            options: options,
+            format: format,
+            app: app,
+            jq: $jq
+        };
+    }
+}(jQuery));
diff --git a/etc/status/public_html/zuul.app.js b/etc/status/public_html/zuul.app.js
new file mode 100644
index 0000000..6f87a92
--- /dev/null
+++ b/etc/status/public_html/zuul.app.js
@@ -0,0 +1,97 @@
+// Client script for Zuul status page
+//
+// Copyright 2013 OpenStack Foundation
+// Copyright 2013 Timo Tijhof
+// Copyright 2013 Wikimedia Foundation
+// Copyright 2014 Rackspace Australia
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+function zuul_build_dom($, container) {
+    // Build a default-looking DOM
+    default_layout = '<div class="container">'
+        + '<h1>Zuul Status</h1>'
+        + '<p>Real-time status monitor of Zuul, the pipeline manager between Gerrit and Workers.</p>'
+        + '<div class="zuul-container" id="zuul-container">'
+        + '<div style="display: none;" class="alert" id="zuul_msg"></div>'
+        + '<button class="btn pull-right zuul-spinner">updating <span class="glyphicon glyphicon-refresh"></span></button>'
+        + '<p>Queue lengths: <span id="zuul_queue_events_num">0</span> events, <span id="zuul_queue_results_num">0</span> results.</p>'
+        + '<div id="zuul_controls"></div>'
+        + '<div id="zuul_pipelines" class="row"></div>'
+        + '<p>Zuul version: <span id="zuul-version-span"></span></p>'
+        + '<p>Last reconfigured: <span id="last-reconfigured-span"></span></p>'
+        + '</div></div>';
+
+    $(function ($) {
+        // DOM ready
+        $container = $(container);
+        $container.html(default_layout);
+    });
+}
+
+function zuul_start($) {
+    // Start the zuul app (expects default dom)
+
+    var $container, $indicator;
+    var demo = location.search.match(/[?&]demo=([^?&]*)/),
+        source_url = location.search.match(/[?&]source_url=([^?&]*)/),
+        source = demo ? './status-' + (demo[1] || 'basic') + '.json-sample' :
+            'status.json';
+    source = source_url ? source_url[1] : source;
+
+    var zuul = $.zuul({
+        source: source,
+        //graphite_url: 'http://graphite.openstack.org/render/'
+    });
+
+    zuul.jq.on('update-start', function () {
+        $container.addClass('zuul-container-loading');
+        $indicator.addClass('zuul-spinner-on');
+    });
+
+    zuul.jq.on('update-end', function () {
+        $container.removeClass('zuul-container-loading');
+        setTimeout(function () {
+            $indicator.removeClass('zuul-spinner-on');
+        }, 500);
+    });
+
+    zuul.jq.one('update-end', function () {
+        // Do this asynchronous so that if the first update adds a
+        // message, it will not animate while we fade in the content.
+        // Instead it simply appears with the rest of the content.
+        setTimeout(function () {
+            // Fade in the content
+            $container.addClass('zuul-container-ready');
+        });
+    });
+
+    $(function ($) {
+        // DOM ready
+        $container = $('#zuul-container');
+        $indicator = $('#zuul-spinner');
+        $('#zuul_controls').append(zuul.app.control_form());
+
+        zuul.app.schedule();
+
+        $(document).on({
+            'show.visibility': function () {
+                zuul.options.enabled = true;
+                zuul.app.update();
+            },
+            'hide.visibility': function () {
+                zuul.options.enabled = false;
+            }
+        });
+    });
+}
\ No newline at end of file
diff --git a/tests/base.py b/tests/base.py
new file mode 100755
index 0000000..01b42f2
--- /dev/null
+++ b/tests/base.py
@@ -0,0 +1,1204 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import ConfigParser
+import gc
+import hashlib
+import json
+import logging
+import os
+import pprint
+import Queue
+import random
+import re
+import select
+import shutil
+import socket
+import string
+import subprocess
+import swiftclient
+import threading
+import time
+import urllib2
+
+import git
+import gear
+import fixtures
+import six.moves.urllib.parse as urlparse
+import statsd
+import testtools
+
+import zuul.scheduler
+import zuul.webapp
+import zuul.rpclistener
+import zuul.launcher.gearman
+import zuul.lib.swift
+import zuul.merger.server
+import zuul.merger.client
+import zuul.reporter.gerrit
+import zuul.reporter.smtp
+import zuul.trigger.gerrit
+import zuul.trigger.timer
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+                           'fixtures')
+
+logging.basicConfig(level=logging.DEBUG,
+                    format='%(asctime)s %(name)-32s '
+                    '%(levelname)-8s %(message)s')
+
+
+def repack_repo(path):
+    cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
+    output = subprocess.Popen(cmd, close_fds=True,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+    out = output.communicate()
+    if output.returncode:
+        raise Exception("git repack returned %d" % output.returncode)
+    return out
+
+
+def random_sha1():
+    return hashlib.sha1(str(random.random())).hexdigest()
+
+
+class ChangeReference(git.Reference):
+    _common_path_default = "refs/changes"
+    _points_to_commits_only = True
+
+
+class FakeChange(object):
+    categories = {'APRV': ('Approved', -1, 1),
+                  'CRVW': ('Code-Review', -2, 2),
+                  'VRFY': ('Verified', -2, 2)}
+
+    def __init__(self, gerrit, number, project, branch, subject,
+                 status='NEW', upstream_root=None):
+        self.gerrit = gerrit
+        self.reported = 0
+        self.queried = 0
+        self.patchsets = []
+        self.number = number
+        self.project = project
+        self.branch = branch
+        self.subject = subject
+        self.latest_patchset = 0
+        self.depends_on_change = None
+        self.needed_by_changes = []
+        self.fail_merge = False
+        self.messages = []
+        self.data = {
+            'branch': branch,
+            'comments': [],
+            'commitMessage': subject,
+            'createdOn': time.time(),
+            'id': 'I' + random_sha1(),
+            'lastUpdated': time.time(),
+            'number': str(number),
+            'open': status == 'NEW',
+            'owner': {'email': 'user@example.com',
+                      'name': 'User Name',
+                      'username': 'username'},
+            'patchSets': self.patchsets,
+            'project': project,
+            'status': status,
+            'subject': subject,
+            'submitRecords': [],
+            'url': 'https://hostname/%s' % number}
+
+        self.upstream_root = upstream_root
+        self.addPatchset()
+        self.data['submitRecords'] = self.getSubmitRecords()
+        self.open = status == 'NEW'
+
+    def add_fake_change_to_repo(self, msg, fn, large):
+        path = os.path.join(self.upstream_root, self.project)
+        repo = git.Repo(path)
+        ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
+                                                        self.latest_patchset),
+                                     'refs/tags/init')
+        repo.head.reference = ref
+        repo.head.reset(index=True, working_tree=True)
+        repo.git.clean('-x', '-f', '-d')
+
+        path = os.path.join(self.upstream_root, self.project)
+        if not large:
+            fn = os.path.join(path, fn)
+            f = open(fn, 'w')
+            f.write("test %s %s %s\n" %
+                    (self.branch, self.number, self.latest_patchset))
+            f.close()
+            repo.index.add([fn])
+        else:
+            for fni in range(100):
+                fn = os.path.join(path, str(fni))
+                f = open(fn, 'w')
+                for ci in range(4096):
+                    f.write(random.choice(string.printable))
+                f.close()
+                repo.index.add([fn])
+
+        r = repo.index.commit(msg)
+        repo.head.reference = 'master'
+        repo.head.reset(index=True, working_tree=True)
+        repo.git.clean('-x', '-f', '-d')
+        repo.heads['master'].checkout()
+        return r
+
+    def addPatchset(self, files=[], large=False):
+        self.latest_patchset += 1
+        if files:
+            fn = files[0]
+        else:
+            fn = '%s-%s' % (self.branch, self.number)
+        msg = self.subject + '-' + str(self.latest_patchset)
+        c = self.add_fake_change_to_repo(msg, fn, large)
+        ps_files = [{'file': '/COMMIT_MSG',
+                     'type': 'ADDED'},
+                    {'file': 'README',
+                     'type': 'MODIFIED'}]
+        for f in files:
+            ps_files.append({'file': f, 'type': 'ADDED'})
+        d = {'approvals': [],
+             'createdOn': time.time(),
+             'files': ps_files,
+             'number': str(self.latest_patchset),
+             'ref': 'refs/changes/1/%s/%s' % (self.number,
+                                              self.latest_patchset),
+             'revision': c.hexsha,
+             'uploader': {'email': 'user@example.com',
+                          'name': 'User name',
+                          'username': 'user'}}
+        self.data['currentPatchSet'] = d
+        self.patchsets.append(d)
+        self.data['submitRecords'] = self.getSubmitRecords()
+
+    def getPatchsetCreatedEvent(self, patchset):
+        event = {"type": "patchset-created",
+                 "change": {"project": self.project,
+                            "branch": self.branch,
+                            "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
+                            "number": str(self.number),
+                            "subject": self.subject,
+                            "owner": {"name": "User Name"},
+                            "url": "https://hostname/3"},
+                 "patchSet": self.patchsets[patchset - 1],
+                 "uploader": {"name": "User Name"}}
+        return event
+
+    def getChangeRestoredEvent(self):
+        event = {"type": "change-restored",
+                 "change": {"project": self.project,
+                            "branch": self.branch,
+                            "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
+                            "number": str(self.number),
+                            "subject": self.subject,
+                            "owner": {"name": "User Name"},
+                            "url": "https://hostname/3"},
+                 "restorer": {"name": "User Name"},
+                 "reason": ""}
+        return event
+
+    def getChangeCommentEvent(self, patchset):
+        event = {"type": "comment-added",
+                 "change": {"project": self.project,
+                            "branch": self.branch,
+                            "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
+                            "number": str(self.number),
+                            "subject": self.subject,
+                            "owner": {"name": "User Name"},
+                            "url": "https://hostname/3"},
+                 "patchSet": self.patchsets[patchset - 1],
+                 "author": {"name": "User Name"},
+                 "approvals": [{"type": "Code-Review",
+                                "description": "Code-Review",
+                                "value": "0"}],
+                 "comment": "This is a comment"}
+        return event
+
+    def addApproval(self, category, value, username='jenkins',
+                    granted_on=None):
+        if not granted_on:
+            granted_on = time.time()
+        approval = {'description': self.categories[category][0],
+                    'type': category,
+                    'value': str(value),
+                    'by': {
+                        'username': username,
+                        'email': username + '@example.com',
+                    },
+                    'grantedOn': int(granted_on)}
+        for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
+            if x['by']['username'] == username and x['type'] == category:
+                del self.patchsets[-1]['approvals'][i]
+        self.patchsets[-1]['approvals'].append(approval)
+        event = {'approvals': [approval],
+                 'author': {'email': 'user@example.com',
+                            'name': 'User Name',
+                            'username': 'username'},
+                 'change': {'branch': self.branch,
+                            'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
+                            'number': str(self.number),
+                            'owner': {'email': 'user@example.com',
+                                      'name': 'User Name',
+                                      'username': 'username'},
+                            'project': self.project,
+                            'subject': self.subject,
+                            'topic': 'master',
+                            'url': 'https://hostname/459'},
+                 'comment': '',
+                 'patchSet': self.patchsets[-1],
+                 'type': 'comment-added'}
+        self.data['submitRecords'] = self.getSubmitRecords()
+        return json.loads(json.dumps(event))
+
+    def getSubmitRecords(self):
+        status = {}
+        for cat in self.categories.keys():
+            status[cat] = 0
+
+        for a in self.patchsets[-1]['approvals']:
+            cur = status[a['type']]
+            cat_min, cat_max = self.categories[a['type']][1:]
+            new = int(a['value'])
+            if new == cat_min:
+                cur = new
+            elif abs(new) > abs(cur):
+                cur = new
+            status[a['type']] = cur
+
+        labels = []
+        ok = True
+        for typ, cat in self.categories.items():
+            cur = status[typ]
+            cat_min, cat_max = cat[1:]
+            if cur == cat_min:
+                value = 'REJECT'
+                ok = False
+            elif cur == cat_max:
+                value = 'OK'
+            else:
+                value = 'NEED'
+                ok = False
+            labels.append({'label': cat[0], 'status': value})
+        if ok:
+            return [{'status': 'OK'}]
+        return [{'status': 'NOT_READY',
+                 'labels': labels}]
+
+    def setDependsOn(self, other, patchset):
+        self.depends_on_change = other
+        d = {'id': other.data['id'],
+             'number': other.data['number'],
+             'ref': other.patchsets[patchset - 1]['ref']
+             }
+        self.data['dependsOn'] = [d]
+
+        other.needed_by_changes.append(self)
+        needed = other.data.get('neededBy', [])
+        d = {'id': self.data['id'],
+             'number': self.data['number'],
+             'ref': self.patchsets[patchset - 1]['ref'],
+             'revision': self.patchsets[patchset - 1]['revision']
+             }
+        needed.append(d)
+        other.data['neededBy'] = needed
+
+    def query(self):
+        self.queried += 1
+        d = self.data.get('dependsOn')
+        if d:
+            d = d[0]
+            if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
+                d['isCurrentPatchSet'] = True
+            else:
+                d['isCurrentPatchSet'] = False
+        return json.loads(json.dumps(self.data))
+
+    def setMerged(self):
+        if (self.depends_on_change and
+            self.depends_on_change.data['status'] != 'MERGED'):
+            return
+        if self.fail_merge:
+            return
+        self.data['status'] = 'MERGED'
+        self.open = False
+
+        path = os.path.join(self.upstream_root, self.project)
+        repo = git.Repo(path)
+        repo.heads[self.branch].commit = \
+            repo.commit(self.patchsets[-1]['revision'])
+
+    def setReported(self):
+        self.reported += 1
+
+
+class FakeGerrit(object):
+    def __init__(self, *args, **kw):
+        self.event_queue = Queue.Queue()
+        self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
+        self.change_number = 0
+        self.changes = {}
+
+    def addFakeChange(self, project, branch, subject, status='NEW'):
+        self.change_number += 1
+        c = FakeChange(self, self.change_number, project, branch, subject,
+                       upstream_root=self.upstream_root,
+                       status=status)
+        self.changes[self.change_number] = c
+        return c
+
+    def addEvent(self, data):
+        return self.event_queue.put(data)
+
+    def getEvent(self):
+        return self.event_queue.get()
+
+    def eventDone(self):
+        self.event_queue.task_done()
+
+    def review(self, project, changeid, message, action):
+        number, ps = changeid.split(',')
+        change = self.changes[int(number)]
+        change.messages.append(message)
+        if 'submit' in action:
+            change.setMerged()
+        if message:
+            change.setReported()
+
+    def query(self, number):
+        change = self.changes.get(int(number))
+        if change:
+            return change.query()
+        return {}
+
+    def startWatching(self, *args, **kw):
+        pass
+
+
+class BuildHistory(object):
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
+    def __repr__(self):
+        return ("<Completed build, result: %s name: %s #%s changes: %s>" %
+                (self.result, self.name, self.number, self.changes))
+
+
+class FakeURLOpener(object):
+    def __init__(self, upstream_root, fake_gerrit, url):
+        self.upstream_root = upstream_root
+        self.fake_gerrit = fake_gerrit
+        self.url = url
+
+    def read(self):
+        res = urlparse.urlparse(self.url)
+        path = res.path
+        project = '/'.join(path.split('/')[2:-2])
+        ret = '001e# service=git-upload-pack\n'
+        ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
+                'multi_ack thin-pack side-band side-band-64k ofs-delta '
+                'shallow no-progress include-tag multi_ack_detailed no-done\n')
+        path = os.path.join(self.upstream_root, project)
+        repo = git.Repo(path)
+        for ref in repo.refs:
+            r = ref.object.hexsha + ' ' + ref.path + '\n'
+            ret += '%04x%s' % (len(r) + 4, r)
+        ret += '0000'
+        return ret
+
+
+class FakeGerritTrigger(zuul.trigger.gerrit.Gerrit):
+    name = 'gerrit'
+
+    def __init__(self, upstream_root, *args):
+        super(FakeGerritTrigger, self).__init__(*args)
+        self.upstream_root = upstream_root
+
+    def getGitUrl(self, project):
+        return os.path.join(self.upstream_root, project.name)
+
+
+class FakeStatsd(threading.Thread):
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.sock.bind(('', 0))
+        self.port = self.sock.getsockname()[1]
+        self.wake_read, self.wake_write = os.pipe()
+        self.stats = []
+
+    def run(self):
+        while True:
+            poll = select.poll()
+            poll.register(self.sock, select.POLLIN)
+            poll.register(self.wake_read, select.POLLIN)
+            ret = poll.poll()
+            for (fd, event) in ret:
+                if fd == self.sock.fileno():
+                    data = self.sock.recvfrom(1024)
+                    if not data:
+                        return
+                    self.stats.append(data[0])
+                if fd == self.wake_read:
+                    return
+
+    def stop(self):
+        os.write(self.wake_write, '1\n')
+
+
+class FakeBuild(threading.Thread):
+    log = logging.getLogger("zuul.test")
+
+    def __init__(self, worker, job, number, node):
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.worker = worker
+        self.job = job
+        self.name = job.name.split(':')[1]
+        self.number = number
+        self.node = node
+        self.parameters = json.loads(job.arguments)
+        self.unique = self.parameters['ZUUL_UUID']
+        self.wait_condition = threading.Condition()
+        self.waiting = False
+        self.aborted = False
+        self.created = time.time()
+        self.description = ''
+        self.run_error = False
+
+    def release(self):
+        self.wait_condition.acquire()
+        self.wait_condition.notify()
+        self.waiting = False
+        self.log.debug("Build %s released" % self.unique)
+        self.wait_condition.release()
+
+    def isWaiting(self):
+        self.wait_condition.acquire()
+        if self.waiting:
+            ret = True
+        else:
+            ret = False
+        self.wait_condition.release()
+        return ret
+
+    def _wait(self):
+        self.wait_condition.acquire()
+        self.waiting = True
+        self.log.debug("Build %s waiting" % self.unique)
+        self.wait_condition.wait()
+        self.wait_condition.release()
+
+    def run(self):
+        data = {
+            'url': 'https://server/job/%s/%s/' % (self.name, self.number),
+            'name': self.name,
+            'number': self.number,
+            'manager': self.worker.worker_id,
+            'worker_name': 'My Worker',
+            'worker_hostname': 'localhost',
+            'worker_ips': ['127.0.0.1', '192.168.1.1'],
+            'worker_fqdn': 'zuul.example.org',
+            'worker_program': 'FakeBuilder',
+            'worker_version': 'v1.1',
+            'worker_extra': {'something': 'else'}
+        }
+
+        self.log.debug('Running build %s' % self.unique)
+
+        self.job.sendWorkData(json.dumps(data))
+        self.log.debug('Sent WorkData packet with %s' % json.dumps(data))
+        self.job.sendWorkStatus(0, 100)
+
+        if self.worker.hold_jobs_in_build:
+            self.log.debug('Holding build %s' % self.unique)
+            self._wait()
+        self.log.debug("Build %s continuing" % self.unique)
+
+        self.worker.lock.acquire()
+
+        result = 'SUCCESS'
+        if (('ZUUL_REF' in self.parameters) and
+            self.worker.shouldFailTest(self.name,
+                                       self.parameters['ZUUL_REF'])):
+            result = 'FAILURE'
+        if self.aborted:
+            result = 'ABORTED'
+
+        if self.run_error:
+            work_fail = True
+            result = 'RUN_ERROR'
+        else:
+            data['result'] = result
+            work_fail = False
+
+        changes = None
+        if 'ZUUL_CHANGE_IDS' in self.parameters:
+            changes = self.parameters['ZUUL_CHANGE_IDS']
+
+        self.worker.build_history.append(
+            BuildHistory(name=self.name, number=self.number,
+                         result=result, changes=changes, node=self.node,
+                         uuid=self.unique, description=self.description,
+                         pipeline=self.parameters['ZUUL_PIPELINE'])
+        )
+
+        self.job.sendWorkData(json.dumps(data))
+        if work_fail:
+            self.job.sendWorkFail()
+        else:
+            self.job.sendWorkComplete(json.dumps(data))
+        del self.worker.gearman_jobs[self.job.unique]
+        self.worker.running_builds.remove(self)
+        self.worker.lock.release()
+
+
+class FakeWorker(gear.Worker):
+    def __init__(self, worker_id, test):
+        super(FakeWorker, self).__init__(worker_id)
+        self.gearman_jobs = {}
+        self.build_history = []
+        self.running_builds = []
+        self.build_counter = 0
+        self.fail_tests = {}
+        self.test = test
+
+        self.hold_jobs_in_build = False
+        self.lock = threading.Lock()
+        self.__work_thread = threading.Thread(target=self.work)
+        self.__work_thread.daemon = True
+        self.__work_thread.start()
+
+    def handleJob(self, job):
+        parts = job.name.split(":")
+        cmd = parts[0]
+        name = parts[1]
+        if len(parts) > 2:
+            node = parts[2]
+        else:
+            node = None
+        if cmd == 'build':
+            self.handleBuild(job, name, node)
+        elif cmd == 'stop':
+            self.handleStop(job, name)
+        elif cmd == 'set_description':
+            self.handleSetDescription(job, name)
+
+    def handleBuild(self, job, name, node):
+        build = FakeBuild(self, job, self.build_counter, node)
+        job.build = build
+        self.gearman_jobs[job.unique] = job
+        self.build_counter += 1
+
+        self.running_builds.append(build)
+        build.start()
+
+    def handleStop(self, job, name):
+        self.log.debug("handle stop")
+        parameters = json.loads(job.arguments)
+        name = parameters['name']
+        number = parameters['number']
+        for build in self.running_builds:
+            if build.name == name and build.number == number:
+                build.aborted = True
+                build.release()
+                job.sendWorkComplete()
+                return
+        job.sendWorkFail()
+
+    def handleSetDescription(self, job, name):
+        self.log.debug("handle set description")
+        parameters = json.loads(job.arguments)
+        name = parameters['name']
+        number = parameters['number']
+        descr = parameters['html_description']
+        for build in self.running_builds:
+            if build.name == name and build.number == number:
+                build.description = descr
+                job.sendWorkComplete()
+                return
+        for build in self.build_history:
+            if build.name == name and build.number == number:
+                build.description = descr
+                job.sendWorkComplete()
+                return
+        job.sendWorkFail()
+
+    def work(self):
+        while self.running:
+            try:
+                job = self.getJob()
+            except gear.InterruptedError:
+                continue
+            try:
+                self.handleJob(job)
+            except:
+                self.log.exception("Worker exception:")
+
+    def addFailTest(self, name, change):
+        l = self.fail_tests.get(name, [])
+        l.append(change)
+        self.fail_tests[name] = l
+
+    def shouldFailTest(self, name, ref):
+        l = self.fail_tests.get(name, [])
+        for change in l:
+            if self.test.ref_has_change(ref, change):
+                return True
+        return False
+
+    def release(self, regex=None):
+        builds = self.running_builds[:]
+        self.log.debug("releasing build %s (%s)" % (regex,
+                                                    len(self.running_builds)))
+        for build in builds:
+            if not regex or re.match(regex, build.name):
+                self.log.debug("releasing build %s" %
+                               (build.parameters['ZUUL_UUID']))
+                build.release()
+            else:
+                self.log.debug("not releasing build %s" %
+                               (build.parameters['ZUUL_UUID']))
+        self.log.debug("done releasing builds %s (%s)" %
+                       (regex, len(self.running_builds)))
+
+
+class FakeGearmanServer(gear.Server):
+    def __init__(self):
+        self.hold_jobs_in_queue = False
+        super(FakeGearmanServer, self).__init__(0)
+
+    def getJobForConnection(self, connection, peek=False):
+        for queue in [self.high_queue, self.normal_queue, self.low_queue]:
+            for job in queue:
+                if not hasattr(job, 'waiting'):
+                    if job.name.startswith('build:'):
+                        job.waiting = self.hold_jobs_in_queue
+                    else:
+                        job.waiting = False
+                if job.waiting:
+                    continue
+                if job.name in connection.functions:
+                    if not peek:
+                        queue.remove(job)
+                        connection.related_jobs[job.handle] = job
+                        job.worker_connection = connection
+                    job.running = True
+                    return job
+        return None
+
+    def release(self, regex=None):
+        released = False
+        qlen = (len(self.high_queue) + len(self.normal_queue) +
+                len(self.low_queue))
+        self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
+        for job in self.getQueue():
+            cmd, name = job.name.split(':')
+            if cmd != 'build':
+                continue
+            if not regex or re.match(regex, name):
+                self.log.debug("releasing queued job %s" %
+                               job.unique)
+                job.waiting = False
+                released = True
+            else:
+                self.log.debug("not releasing queued job %s" %
+                               job.unique)
+        if released:
+            self.wakeConnections()
+        qlen = (len(self.high_queue) + len(self.normal_queue) +
+                len(self.low_queue))
+        self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
+
+
+class FakeSMTP(object):
+    log = logging.getLogger('zuul.FakeSMTP')
+
+    def __init__(self, messages, server, port):
+        self.server = server
+        self.port = port
+        self.messages = messages
+
+    def sendmail(self, from_email, to_email, msg):
+        self.log.info("Sending email from %s, to %s, with msg %s" % (
+                      from_email, to_email, msg))
+
+        headers = msg.split('\n\n', 1)[0]
+        body = msg.split('\n\n', 1)[1]
+
+        self.messages.append(dict(
+            from_email=from_email,
+            to_email=to_email,
+            msg=msg,
+            headers=headers,
+            body=body,
+        ))
+
+        return True
+
+    def quit(self):
+        return True
+
+
+class FakeSwiftClientConnection(swiftclient.client.Connection):
+    def post_account(self, headers):
+        # Do nothing
+        pass
+
+    def get_auth(self):
+        # Returns endpoint and (unused) auth token
+        endpoint = os.path.join('https://storage.example.org', 'V1',
+                                'AUTH_account')
+        return endpoint, ''
+
+
+class ZuulTestCase(testtools.TestCase):
+    log = logging.getLogger("zuul.test")
+
+    def setUp(self):
+        super(ZuulTestCase, self).setUp()
+        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
+        try:
+            test_timeout = int(test_timeout)
+        except ValueError:
+            # If timeout value is invalid do not set a timeout.
+            test_timeout = 0
+        if test_timeout > 0:
+            self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
+
+        if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
+            os.environ.get('OS_STDOUT_CAPTURE') == '1'):
+            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
+        if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
+            os.environ.get('OS_STDERR_CAPTURE') == '1'):
+            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+        if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
+            os.environ.get('OS_LOG_CAPTURE') == '1'):
+            self.useFixture(fixtures.FakeLogger(
+                level=logging.DEBUG,
+                format='%(asctime)s %(name)-32s '
+                '%(levelname)-8s %(message)s'))
+        tmp_root = self.useFixture(fixtures.TempDir(
+            rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
+        self.test_root = os.path.join(tmp_root, "zuul-test")
+        self.upstream_root = os.path.join(self.test_root, "upstream")
+        self.git_root = os.path.join(self.test_root, "git")
+
+        if os.path.exists(self.test_root):
+            shutil.rmtree(self.test_root)
+        os.makedirs(self.test_root)
+        os.makedirs(self.upstream_root)
+        os.makedirs(self.git_root)
+
+        # Make per test copy of Configuration.
+        self.setup_config()
+        self.config.set('zuul', 'layout_config',
+                        os.path.join(FIXTURE_DIR, "layout.yaml"))
+        self.config.set('merger', 'git_dir', self.git_root)
+
+        # For each project in config:
+        self.init_repo("org/project")
+        self.init_repo("org/project1")
+        self.init_repo("org/project2")
+        self.init_repo("org/project3")
+        self.init_repo("org/one-job-project")
+        self.init_repo("org/nonvoting-project")
+        self.init_repo("org/templated-project")
+        self.init_repo("org/layered-project")
+        self.init_repo("org/node-project")
+        self.init_repo("org/conflict-project")
+        self.init_repo("org/noop-project")
+        self.init_repo("org/experimental-project")
+
+        self.statsd = FakeStatsd()
+        os.environ['STATSD_HOST'] = 'localhost'
+        os.environ['STATSD_PORT'] = str(self.statsd.port)
+        self.statsd.start()
+        # the statsd client object is configured in the statsd module import
+        reload(statsd)
+        reload(zuul.scheduler)
+
+        self.gearman_server = FakeGearmanServer()
+
+        self.config.set('gearman', 'port', str(self.gearman_server.port))
+
+        self.worker = FakeWorker('fake_worker', self)
+        self.worker.addServer('127.0.0.1', self.gearman_server.port)
+        self.gearman_server.worker = self.worker
+
+        self.merge_server = zuul.merger.server.MergeServer(self.config)
+        self.merge_server.start()
+
+        self.sched = zuul.scheduler.Scheduler()
+
+        self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
+                                             FakeSwiftClientConnection))
+        self.swift = zuul.lib.swift.Swift(self.config)
+
+        def URLOpenerFactory(*args, **kw):
+            if isinstance(args[0], urllib2.Request):
+                return old_urlopen(*args, **kw)
+            args = [self.fake_gerrit] + list(args)
+            return FakeURLOpener(self.upstream_root, *args, **kw)
+
+        old_urlopen = urllib2.urlopen
+        urllib2.urlopen = URLOpenerFactory
+
+        self.launcher = zuul.launcher.gearman.Gearman(self.config, self.sched,
+                                                      self.swift)
+        self.merge_client = zuul.merger.client.MergeClient(
+            self.config, self.sched)
+
+        self.smtp_messages = []
+
+        def FakeSMTPFactory(*args, **kw):
+            args = [self.smtp_messages] + list(args)
+            return FakeSMTP(*args, **kw)
+
+        zuul.lib.gerrit.Gerrit = FakeGerrit
+        self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
+
+        self.gerrit = FakeGerritTrigger(
+            self.upstream_root, self.config, self.sched)
+        self.gerrit.replication_timeout = 1.5
+        self.gerrit.replication_retry_interval = 0.5
+        self.fake_gerrit = self.gerrit.gerrit
+        self.fake_gerrit.upstream_root = self.upstream_root
+
+        self.webapp = zuul.webapp.WebApp(self.sched, port=0)
+        self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
+
+        self.sched.setLauncher(self.launcher)
+        self.sched.setMerger(self.merge_client)
+        self.sched.registerTrigger(self.gerrit)
+        self.timer = zuul.trigger.timer.Timer(self.config, self.sched)
+        self.sched.registerTrigger(self.timer)
+
+        self.sched.registerReporter(
+            zuul.reporter.gerrit.Reporter(self.gerrit))
+        self.smtp_reporter = zuul.reporter.smtp.Reporter(
+            self.config.get('smtp', 'default_from'),
+            self.config.get('smtp', 'default_to'),
+            self.config.get('smtp', 'server'))
+        self.sched.registerReporter(self.smtp_reporter)
+
+        self.sched.start()
+        self.sched.reconfigure(self.config)
+        self.sched.resume()
+        self.webapp.start()
+        self.rpc.start()
+        self.launcher.gearman.waitForServer()
+        self.registerJobs()
+        self.builds = self.worker.running_builds
+        self.history = self.worker.build_history
+
+        self.addCleanup(self.assertFinalState)
+        self.addCleanup(self.shutdown)
+
+    def setup_config(self):
+        """Per test config object. Override to set different config."""
+        self.config = ConfigParser.ConfigParser()
+        self.config.read(os.path.join(FIXTURE_DIR, "zuul.conf"))
+
+    def assertFinalState(self):
+        # Make sure that the change cache is cleared
+        self.assertEqual(len(self.gerrit._change_cache.keys()), 0)
+        # Make sure that git.Repo objects have been garbage collected.
+        repos = []
+        gc.collect()
+        for obj in gc.get_objects():
+            if isinstance(obj, git.Repo):
+                repos.append(obj)
+        self.assertEqual(len(repos), 0)
+        self.assertEmptyQueues()
+
+    def shutdown(self):
+        self.log.debug("Shutting down after tests")
+        self.launcher.stop()
+        self.merge_server.stop()
+        self.merge_server.join()
+        self.merge_client.stop()
+        self.worker.shutdown()
+        self.gerrit.stop()
+        self.timer.stop()
+        self.sched.stop()
+        self.sched.join()
+        self.statsd.stop()
+        self.statsd.join()
+        self.webapp.stop()
+        self.webapp.join()
+        self.rpc.stop()
+        self.rpc.join()
+        self.gearman_server.shutdown()
+        threads = threading.enumerate()
+        if len(threads) > 1:
+            self.log.error("More than one thread is running: %s" % threads)
+        super(ZuulTestCase, self).tearDown()
+
+    def init_repo(self, project):
+        parts = project.split('/')
+        path = os.path.join(self.upstream_root, *parts[:-1])
+        if not os.path.exists(path):
+            os.makedirs(path)
+        path = os.path.join(self.upstream_root, project)
+        repo = git.Repo.init(path)
+
+        repo.config_writer().set_value('user', 'email', 'user@example.com')
+        repo.config_writer().set_value('user', 'name', 'User Name')
+        repo.config_writer().write()
+
+        fn = os.path.join(path, 'README')
+        f = open(fn, 'w')
+        f.write("test\n")
+        f.close()
+        repo.index.add([fn])
+        repo.index.commit('initial commit')
+        master = repo.create_head('master')
+        repo.create_tag('init')
+
+        mp = repo.create_head('mp')
+        repo.head.reference = mp
+        f = open(fn, 'a')
+        f.write("test mp\n")
+        f.close()
+        repo.index.add([fn])
+        repo.index.commit('mp commit')
+
+        repo.head.reference = master
+        repo.head.reset(index=True, working_tree=True)
+        repo.git.clean('-x', '-f', '-d')
+
+    def ref_has_change(self, ref, change):
+        path = os.path.join(self.git_root, change.project)
+        repo = git.Repo(path)
+        for commit in repo.iter_commits(ref):
+            if commit.message.strip() == ('%s-1' % change.subject):
+                return True
+        return False
+
+    def job_has_changes(self, *args):
+        job = args[0]
+        commits = args[1:]
+        if isinstance(job, FakeBuild):
+            parameters = job.parameters
+        else:
+            parameters = json.loads(job.arguments)
+        project = parameters['ZUUL_PROJECT']
+        path = os.path.join(self.git_root, project)
+        repo = git.Repo(path)
+        ref = parameters['ZUUL_REF']
+        sha = parameters['ZUUL_COMMIT']
+        repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
+        repo_shas = [c.hexsha for c in repo.iter_commits(ref)]
+        commit_messages = ['%s-1' % commit.subject for commit in commits]
+        self.log.debug("Checking if job %s has changes; commit_messages %s;"
+                       " repo_messages %s; sha %s" % (job, commit_messages,
+                                                      repo_messages, sha))
+        for msg in commit_messages:
+            if msg not in repo_messages:
+                self.log.debug("  messages do not match")
+                return False
+        if repo_shas[0] != sha:
+            self.log.debug("  sha does not match")
+            return False
+        self.log.debug("  OK")
+        return True
+
+    def registerJobs(self):
+        count = 0
+        for job in self.sched.layout.jobs.keys():
+            self.worker.registerFunction('build:' + job)
+            count += 1
+        self.worker.registerFunction('stop:' + self.worker.worker_id)
+        count += 1
+
+        while len(self.gearman_server.functions) < count:
+            time.sleep(0)
+
+    def release(self, job):
+        if isinstance(job, FakeBuild):
+            job.release()
+        else:
+            job.waiting = False
+            self.log.debug("Queued job %s released" % job.unique)
+            self.gearman_server.wakeConnections()
+
+    def getParameter(self, job, name):
+        if isinstance(job, FakeBuild):
+            return job.parameters[name]
+        else:
+            parameters = json.loads(job.arguments)
+            return parameters[name]
+
+    def resetGearmanServer(self):
+        self.worker.setFunctions([])
+        while True:
+            done = True
+            for connection in self.gearman_server.active_connections:
+                if (connection.functions and
+                    connection.client_id not in ['Zuul RPC Listener',
+                                                 'Zuul Merger']):
+                    done = False
+            if done:
+                break
+            time.sleep(0)
+        self.gearman_server.functions = set()
+        self.rpc.register()
+        self.merge_server.register()
+
+    def haveAllBuildsReported(self):
+        # See if Zuul is waiting on a meta job to complete
+        if self.launcher.meta_jobs:
+            return False
+        # Find out if every build that the worker has completed has been
+        # reported back to Zuul.  If it hasn't then that means a Gearman
+        # event is still in transit and the system is not stable.
+        for build in self.worker.build_history:
+            zbuild = self.launcher.builds.get(build.uuid)
+            if not zbuild:
+                # It has already been reported
+                continue
+            # It hasn't been reported yet.
+            return False
+        # Make sure that none of the worker connections are in GRAB_WAIT
+        for connection in self.worker.active_connections:
+            if connection.state == 'GRAB_WAIT':
+                return False
+        return True
+
+    def areAllBuildsWaiting(self):
+        ret = True
+
+        builds = self.launcher.builds.values()
+        for build in builds:
+            client_job = None
+            for conn in self.launcher.gearman.active_connections:
+                for j in conn.related_jobs.values():
+                    if j.unique == build.uuid:
+                        client_job = j
+                        break
+            if not client_job:
+                self.log.debug("%s is not known to the gearman client" %
+                               build)
+                ret = False
+                continue
+            if not client_job.handle:
+                self.log.debug("%s has no handle" % client_job)
+                ret = False
+                continue
+            server_job = self.gearman_server.jobs.get(client_job.handle)
+            if not server_job:
+                self.log.debug("%s is not known to the gearman server" %
+                               client_job)
+                ret = False
+                continue
+            if not hasattr(server_job, 'waiting'):
+                self.log.debug("%s is being enqueued" % server_job)
+                ret = False
+                continue
+            if server_job.waiting:
+                continue
+            worker_job = self.worker.gearman_jobs.get(server_job.unique)
+            if worker_job:
+                if worker_job.build.isWaiting():
+                    continue
+                else:
+                    self.log.debug("%s is running" % worker_job)
+                    ret = False
+            else:
+                self.log.debug("%s is unassigned" % server_job)
+                ret = False
+        return ret
+
+    def waitUntilSettled(self):
+        self.log.debug("Waiting until settled...")
+        start = time.time()
+        while True:
+            if time.time() - start > 10:
+                print 'queue status:',
+                print self.sched.trigger_event_queue.empty(),
+                print self.sched.result_event_queue.empty(),
+                print self.fake_gerrit.event_queue.empty(),
+                print self.areAllBuildsWaiting()
+                raise Exception("Timeout waiting for Zuul to settle")
+            # Make sure no new events show up while we're checking
+            self.worker.lock.acquire()
+            # have all build states propogated to zuul?
+            if self.haveAllBuildsReported():
+                # Join ensures that the queue is empty _and_ events have been
+                # processed
+                self.fake_gerrit.event_queue.join()
+                self.sched.trigger_event_queue.join()
+                self.sched.result_event_queue.join()
+                self.sched.run_handler_lock.acquire()
+                if (self.sched.trigger_event_queue.empty() and
+                    self.sched.result_event_queue.empty() and
+                    self.fake_gerrit.event_queue.empty() and
+                    not self.merge_client.build_sets and
+                    self.haveAllBuildsReported() and
+                    self.areAllBuildsWaiting()):
+                    self.sched.run_handler_lock.release()
+                    self.worker.lock.release()
+                    self.log.debug("...settled.")
+                    return
+                self.sched.run_handler_lock.release()
+            self.worker.lock.release()
+            self.sched.wake_event.wait(0.1)
+
+    def countJobResults(self, jobs, result):
+        jobs = filter(lambda x: x.result == result, jobs)
+        return len(jobs)
+
+    def getJobFromHistory(self, name):
+        history = self.worker.build_history
+        for job in history:
+            if job.name == name:
+                return job
+        raise Exception("Unable to find job %s in history" % name)
+
+    def assertEmptyQueues(self):
+        # Make sure there are no orphaned jobs
+        for pipeline in self.sched.layout.pipelines.values():
+            for queue in pipeline.queues:
+                if len(queue.queue) != 0:
+                    print 'pipeline %s queue %s contents %s' % (
+                        pipeline.name, queue.name, queue.queue)
+                self.assertEqual(len(queue.queue), 0)
+
+    def assertReportedStat(self, key, value=None, kind=None):
+        start = time.time()
+        while time.time() < (start + 5):
+            for stat in self.statsd.stats:
+                pprint.pprint(self.statsd.stats)
+                k, v = stat.split(':')
+                if key == k:
+                    if value is None and kind is None:
+                        return
+                    elif value:
+                        if value == v:
+                            return
+                    elif kind:
+                        if v.endswith('|' + kind):
+                            return
+            time.sleep(0.1)
+
+        pprint.pprint(self.statsd.stats)
+        raise Exception("Key %s not found in reported stats" % key)
diff --git a/tests/fixtures/layout-idle.yaml b/tests/fixtures/layout-idle.yaml
index e4574fa..0870788 100644
--- a/tests/fixtures/layout-idle.yaml
+++ b/tests/fixtures/layout-idle.yaml
@@ -8,5 +8,5 @@
 projects:
   - name: org/project
     periodic:
-      - project-test1
-      - project-test2
+      - project-bitrot-stable-old
+      - project-bitrot-stable-older
diff --git a/tests/fixtures/layout-no-timer.yaml b/tests/fixtures/layout-no-timer.yaml
new file mode 100644
index 0000000..9436821
--- /dev/null
+++ b/tests/fixtures/layout-no-timer.yaml
@@ -0,0 +1,14 @@
+pipelines:
+  - name: periodic
+    manager: IndependentPipelineManager
+    # Trigger is required, set it to one that is a noop
+    # during tests that check the timer trigger.
+    trigger:
+      gerrit:
+        - event: patchset-created
+
+projects:
+  - name: org/project
+    periodic:
+      - project-bitrot-stable-old
+      - project-bitrot-stable-older
diff --git a/tests/fixtures/layout-timer-smtp.yaml b/tests/fixtures/layout-timer-smtp.yaml
index ac59df4..b5a6ce0 100644
--- a/tests/fixtures/layout-timer-smtp.yaml
+++ b/tests/fixtures/layout-timer-smtp.yaml
@@ -3,7 +3,7 @@
     manager: IndependentPipelineManager
     trigger:
       timer:
-        - time: '* * * * * */10'
+        - time: '* * * * * */1'
     success:
       smtp:
         to: alternative_me@example.com
diff --git a/tests/fixtures/layout-timer.yaml b/tests/fixtures/layout-timer.yaml
index 9e0f66b..4904f87 100644
--- a/tests/fixtures/layout-timer.yaml
+++ b/tests/fixtures/layout-timer.yaml
@@ -15,7 +15,7 @@
     manager: IndependentPipelineManager
     trigger:
       timer:
-        - time: '* * * * * */10'
+        - time: '* * * * * */1'
 
 projects:
   - name: org/project
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index d489ff1..f698d59 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -14,1199 +14,33 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import ConfigParser
 from cStringIO import StringIO
-import gc
 import gzip
-import hashlib
 import json
 import logging
 import os
-import pprint
-import Queue
-import random
 import re
-import select
 import shutil
-import socket
-import string
-import subprocess
-import swiftclient
-import threading
 import time
 import urllib
 import urllib2
 
 import git
-import gear
-import fixtures
-import six.moves.urllib.parse as urlparse
-import statsd
 import testtools
 
 import zuul.scheduler
-import zuul.webapp
-import zuul.rpclistener
 import zuul.rpcclient
-import zuul.launcher.gearman
-import zuul.lib.swift
-import zuul.merger.server
-import zuul.merger.client
 import zuul.reporter.gerrit
 import zuul.reporter.smtp
-import zuul.trigger.gerrit
-import zuul.trigger.timer
 
-FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
-                           'fixtures')
-CONFIG = ConfigParser.ConfigParser()
-CONFIG.read(os.path.join(FIXTURE_DIR, "zuul.conf"))
-
-CONFIG.set('zuul', 'layout_config',
-           os.path.join(FIXTURE_DIR, "layout.yaml"))
+from tests.base import ZuulTestCase, repack_repo
 
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s %(name)-32s '
                     '%(levelname)-8s %(message)s')
 
 
-def repack_repo(path):
-    cmd = ['git', '--git-dir=%s/.git' % path, 'repack', '-afd']
-    output = subprocess.Popen(cmd, close_fds=True,
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE)
-    out = output.communicate()
-    if output.returncode:
-        raise Exception("git repack returned %d" % output.returncode)
-    return out
-
-
-def random_sha1():
-    return hashlib.sha1(str(random.random())).hexdigest()
-
-
-class ChangeReference(git.Reference):
-    _common_path_default = "refs/changes"
-    _points_to_commits_only = True
-
-
-class FakeChange(object):
-    categories = {'APRV': ('Approved', -1, 1),
-                  'CRVW': ('Code-Review', -2, 2),
-                  'VRFY': ('Verified', -2, 2)}
-
-    def __init__(self, gerrit, number, project, branch, subject,
-                 status='NEW', upstream_root=None):
-        self.gerrit = gerrit
-        self.reported = 0
-        self.queried = 0
-        self.patchsets = []
-        self.number = number
-        self.project = project
-        self.branch = branch
-        self.subject = subject
-        self.latest_patchset = 0
-        self.depends_on_change = None
-        self.needed_by_changes = []
-        self.fail_merge = False
-        self.messages = []
-        self.data = {
-            'branch': branch,
-            'comments': [],
-            'commitMessage': subject,
-            'createdOn': time.time(),
-            'id': 'I' + random_sha1(),
-            'lastUpdated': time.time(),
-            'number': str(number),
-            'open': status == 'NEW',
-            'owner': {'email': 'user@example.com',
-                      'name': 'User Name',
-                      'username': 'username'},
-            'patchSets': self.patchsets,
-            'project': project,
-            'status': status,
-            'subject': subject,
-            'submitRecords': [],
-            'url': 'https://hostname/%s' % number}
-
-        self.upstream_root = upstream_root
-        self.addPatchset()
-        self.data['submitRecords'] = self.getSubmitRecords()
-        self.open = True
-
-    def add_fake_change_to_repo(self, msg, fn, large):
-        path = os.path.join(self.upstream_root, self.project)
-        repo = git.Repo(path)
-        ref = ChangeReference.create(repo, '1/%s/%s' % (self.number,
-                                                        self.latest_patchset),
-                                     'refs/tags/init')
-        repo.head.reference = ref
-        repo.head.reset(index=True, working_tree=True)
-        repo.git.clean('-x', '-f', '-d')
-
-        path = os.path.join(self.upstream_root, self.project)
-        if not large:
-            fn = os.path.join(path, fn)
-            f = open(fn, 'w')
-            f.write("test %s %s %s\n" %
-                    (self.branch, self.number, self.latest_patchset))
-            f.close()
-            repo.index.add([fn])
-        else:
-            for fni in range(100):
-                fn = os.path.join(path, str(fni))
-                f = open(fn, 'w')
-                for ci in range(4096):
-                    f.write(random.choice(string.printable))
-                f.close()
-                repo.index.add([fn])
-
-        r = repo.index.commit(msg)
-        repo.head.reference = 'master'
-        repo.head.reset(index=True, working_tree=True)
-        repo.git.clean('-x', '-f', '-d')
-        repo.heads['master'].checkout()
-        return r
-
-    def addPatchset(self, files=[], large=False):
-        self.latest_patchset += 1
-        if files:
-            fn = files[0]
-        else:
-            fn = '%s-%s' % (self.branch, self.number)
-        msg = self.subject + '-' + str(self.latest_patchset)
-        c = self.add_fake_change_to_repo(msg, fn, large)
-        ps_files = [{'file': '/COMMIT_MSG',
-                     'type': 'ADDED'},
-                    {'file': 'README',
-                     'type': 'MODIFIED'}]
-        for f in files:
-            ps_files.append({'file': f, 'type': 'ADDED'})
-        d = {'approvals': [],
-             'createdOn': time.time(),
-             'files': ps_files,
-             'number': str(self.latest_patchset),
-             'ref': 'refs/changes/1/%s/%s' % (self.number,
-                                              self.latest_patchset),
-             'revision': c.hexsha,
-             'uploader': {'email': 'user@example.com',
-                          'name': 'User name',
-                          'username': 'user'}}
-        self.data['currentPatchSet'] = d
-        self.patchsets.append(d)
-        self.data['submitRecords'] = self.getSubmitRecords()
-
-    def getPatchsetCreatedEvent(self, patchset):
-        event = {"type": "patchset-created",
-                 "change": {"project": self.project,
-                            "branch": self.branch,
-                            "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
-                            "number": str(self.number),
-                            "subject": self.subject,
-                            "owner": {"name": "User Name"},
-                            "url": "https://hostname/3"},
-                 "patchSet": self.patchsets[patchset - 1],
-                 "uploader": {"name": "User Name"}}
-        return event
-
-    def getChangeRestoredEvent(self):
-        event = {"type": "change-restored",
-                 "change": {"project": self.project,
-                            "branch": self.branch,
-                            "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
-                            "number": str(self.number),
-                            "subject": self.subject,
-                            "owner": {"name": "User Name"},
-                            "url": "https://hostname/3"},
-                 "restorer": {"name": "User Name"},
-                 "reason": ""}
-        return event
-
-    def getChangeCommentEvent(self, patchset):
-        event = {"type": "comment-added",
-                 "change": {"project": self.project,
-                            "branch": self.branch,
-                            "id": "I5459869c07352a31bfb1e7a8cac379cabfcb25af",
-                            "number": str(self.number),
-                            "subject": self.subject,
-                            "owner": {"name": "User Name"},
-                            "url": "https://hostname/3"},
-                 "patchSet": self.patchsets[patchset - 1],
-                 "author": {"name": "User Name"},
-                 "approvals": [{"type": "Code-Review",
-                                "description": "Code-Review",
-                                "value": "0"}],
-                 "comment": "This is a comment"}
-        return event
-
-    def addApproval(self, category, value, username='jenkins',
-                    granted_on=None):
-        if not granted_on:
-            granted_on = time.time()
-        approval = {'description': self.categories[category][0],
-                    'type': category,
-                    'value': str(value),
-                    'by': {
-                        'username': username,
-                        'email': username + '@example.com',
-                    },
-                    'grantedOn': int(granted_on)}
-        for i, x in enumerate(self.patchsets[-1]['approvals'][:]):
-            if x['by']['username'] == username and x['type'] == category:
-                del self.patchsets[-1]['approvals'][i]
-        self.patchsets[-1]['approvals'].append(approval)
-        event = {'approvals': [approval],
-                 'author': {'email': 'user@example.com',
-                            'name': 'User Name',
-                            'username': 'username'},
-                 'change': {'branch': self.branch,
-                            'id': 'Iaa69c46accf97d0598111724a38250ae76a22c87',
-                            'number': str(self.number),
-                            'owner': {'email': 'user@example.com',
-                                      'name': 'User Name',
-                                      'username': 'username'},
-                            'project': self.project,
-                            'subject': self.subject,
-                            'topic': 'master',
-                            'url': 'https://hostname/459'},
-                 'comment': '',
-                 'patchSet': self.patchsets[-1],
-                 'type': 'comment-added'}
-        self.data['submitRecords'] = self.getSubmitRecords()
-        return json.loads(json.dumps(event))
-
-    def getSubmitRecords(self):
-        status = {}
-        for cat in self.categories.keys():
-            status[cat] = 0
-
-        for a in self.patchsets[-1]['approvals']:
-            cur = status[a['type']]
-            cat_min, cat_max = self.categories[a['type']][1:]
-            new = int(a['value'])
-            if new == cat_min:
-                cur = new
-            elif abs(new) > abs(cur):
-                cur = new
-            status[a['type']] = cur
-
-        labels = []
-        ok = True
-        for typ, cat in self.categories.items():
-            cur = status[typ]
-            cat_min, cat_max = cat[1:]
-            if cur == cat_min:
-                value = 'REJECT'
-                ok = False
-            elif cur == cat_max:
-                value = 'OK'
-            else:
-                value = 'NEED'
-                ok = False
-            labels.append({'label': cat[0], 'status': value})
-        if ok:
-            return [{'status': 'OK'}]
-        return [{'status': 'NOT_READY',
-                 'labels': labels}]
-
-    def setDependsOn(self, other, patchset):
-        self.depends_on_change = other
-        d = {'id': other.data['id'],
-             'number': other.data['number'],
-             'ref': other.patchsets[patchset - 1]['ref']
-             }
-        self.data['dependsOn'] = [d]
-
-        other.needed_by_changes.append(self)
-        needed = other.data.get('neededBy', [])
-        d = {'id': self.data['id'],
-             'number': self.data['number'],
-             'ref': self.patchsets[patchset - 1]['ref'],
-             'revision': self.patchsets[patchset - 1]['revision']
-             }
-        needed.append(d)
-        other.data['neededBy'] = needed
-
-    def query(self):
-        self.queried += 1
-        d = self.data.get('dependsOn')
-        if d:
-            d = d[0]
-            if (self.depends_on_change.patchsets[-1]['ref'] == d['ref']):
-                d['isCurrentPatchSet'] = True
-            else:
-                d['isCurrentPatchSet'] = False
-        return json.loads(json.dumps(self.data))
-
-    def setMerged(self):
-        if (self.depends_on_change and
-            self.depends_on_change.data['status'] != 'MERGED'):
-            return
-        if self.fail_merge:
-            return
-        self.data['status'] = 'MERGED'
-        self.open = False
-
-        path = os.path.join(self.upstream_root, self.project)
-        repo = git.Repo(path)
-        repo.heads[self.branch].commit = \
-            repo.commit(self.patchsets[-1]['revision'])
-
-    def setReported(self):
-        self.reported += 1
-
-
-class FakeGerrit(object):
-    def __init__(self, *args, **kw):
-        self.event_queue = Queue.Queue()
-        self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
-        self.change_number = 0
-        self.changes = {}
-
-    def addFakeChange(self, project, branch, subject, status='NEW'):
-        self.change_number += 1
-        c = FakeChange(self, self.change_number, project, branch, subject,
-                       upstream_root=self.upstream_root,
-                       status=status)
-        self.changes[self.change_number] = c
-        return c
-
-    def addEvent(self, data):
-        return self.event_queue.put(data)
-
-    def getEvent(self):
-        return self.event_queue.get()
-
-    def eventDone(self):
-        self.event_queue.task_done()
-
-    def review(self, project, changeid, message, action):
-        number, ps = changeid.split(',')
-        change = self.changes[int(number)]
-        change.messages.append(message)
-        if 'submit' in action:
-            change.setMerged()
-        if message:
-            change.setReported()
-
-    def query(self, number):
-        change = self.changes.get(int(number))
-        if change:
-            return change.query()
-        return {}
-
-    def startWatching(self, *args, **kw):
-        pass
-
-
-class BuildHistory(object):
-    def __init__(self, **kw):
-        self.__dict__.update(kw)
-
-    def __repr__(self):
-        return ("<Completed build, result: %s name: %s #%s changes: %s>" %
-                (self.result, self.name, self.number, self.changes))
-
-
-class FakeURLOpener(object):
-    def __init__(self, upstream_root, fake_gerrit, url):
-        self.upstream_root = upstream_root
-        self.fake_gerrit = fake_gerrit
-        self.url = url
-
-    def read(self):
-        res = urlparse.urlparse(self.url)
-        path = res.path
-        project = '/'.join(path.split('/')[2:-2])
-        ret = '001e# service=git-upload-pack\n'
-        ret += ('000000a31270149696713ba7e06f1beb760f20d359c4abed HEAD\x00'
-                'multi_ack thin-pack side-band side-band-64k ofs-delta '
-                'shallow no-progress include-tag multi_ack_detailed no-done\n')
-        path = os.path.join(self.upstream_root, project)
-        repo = git.Repo(path)
-        for ref in repo.refs:
-            r = ref.object.hexsha + ' ' + ref.path + '\n'
-            ret += '%04x%s' % (len(r) + 4, r)
-        ret += '0000'
-        return ret
-
-
-class FakeGerritTrigger(zuul.trigger.gerrit.Gerrit):
-    name = 'gerrit'
-
-    def __init__(self, upstream_root, *args):
-        super(FakeGerritTrigger, self).__init__(*args)
-        self.upstream_root = upstream_root
-
-    def getGitUrl(self, project):
-        return os.path.join(self.upstream_root, project.name)
-
-
-class FakeStatsd(threading.Thread):
-    def __init__(self):
-        threading.Thread.__init__(self)
-        self.daemon = True
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        self.sock.bind(('', 0))
-        self.port = self.sock.getsockname()[1]
-        self.wake_read, self.wake_write = os.pipe()
-        self.stats = []
-
-    def run(self):
-        while True:
-            poll = select.poll()
-            poll.register(self.sock, select.POLLIN)
-            poll.register(self.wake_read, select.POLLIN)
-            ret = poll.poll()
-            for (fd, event) in ret:
-                if fd == self.sock.fileno():
-                    data = self.sock.recvfrom(1024)
-                    if not data:
-                        return
-                    self.stats.append(data[0])
-                if fd == self.wake_read:
-                    return
-
-    def stop(self):
-        os.write(self.wake_write, '1\n')
-
-
-class FakeBuild(threading.Thread):
-    log = logging.getLogger("zuul.test")
-
-    def __init__(self, worker, job, number, node):
-        threading.Thread.__init__(self)
-        self.daemon = True
-        self.worker = worker
-        self.job = job
-        self.name = job.name.split(':')[1]
-        self.number = number
-        self.node = node
-        self.parameters = json.loads(job.arguments)
-        self.unique = self.parameters['ZUUL_UUID']
-        self.wait_condition = threading.Condition()
-        self.waiting = False
-        self.aborted = False
-        self.created = time.time()
-        self.description = ''
-        self.run_error = False
-
-    def release(self):
-        self.wait_condition.acquire()
-        self.wait_condition.notify()
-        self.waiting = False
-        self.log.debug("Build %s released" % self.unique)
-        self.wait_condition.release()
-
-    def isWaiting(self):
-        self.wait_condition.acquire()
-        if self.waiting:
-            ret = True
-        else:
-            ret = False
-        self.wait_condition.release()
-        return ret
-
-    def _wait(self):
-        self.wait_condition.acquire()
-        self.waiting = True
-        self.log.debug("Build %s waiting" % self.unique)
-        self.wait_condition.wait()
-        self.wait_condition.release()
-
-    def run(self):
-        data = {
-            'url': 'https://server/job/%s/%s/' % (self.name, self.number),
-            'name': self.name,
-            'number': self.number,
-            'manager': self.worker.worker_id,
-            'worker_name': 'My Worker',
-            'worker_hostname': 'localhost',
-            'worker_ips': ['127.0.0.1', '192.168.1.1'],
-            'worker_fqdn': 'zuul.example.org',
-            'worker_program': 'FakeBuilder',
-            'worker_version': 'v1.1',
-            'worker_extra': {'something': 'else'}
-        }
-
-        self.log.debug('Running build %s' % self.unique)
-
-        self.job.sendWorkData(json.dumps(data))
-        self.log.debug('Sent WorkData packet with %s' % json.dumps(data))
-        self.job.sendWorkStatus(0, 100)
-
-        if self.worker.hold_jobs_in_build:
-            self.log.debug('Holding build %s' % self.unique)
-            self._wait()
-        self.log.debug("Build %s continuing" % self.unique)
-
-        self.worker.lock.acquire()
-
-        result = 'SUCCESS'
-        if (('ZUUL_REF' in self.parameters) and
-            self.worker.shouldFailTest(self.name,
-                                       self.parameters['ZUUL_REF'])):
-            result = 'FAILURE'
-        if self.aborted:
-            result = 'ABORTED'
-
-        if self.run_error:
-            work_fail = True
-            result = 'RUN_ERROR'
-        else:
-            data['result'] = result
-            work_fail = False
-
-        changes = None
-        if 'ZUUL_CHANGE_IDS' in self.parameters:
-            changes = self.parameters['ZUUL_CHANGE_IDS']
-
-        self.worker.build_history.append(
-            BuildHistory(name=self.name, number=self.number,
-                         result=result, changes=changes, node=self.node,
-                         uuid=self.unique, description=self.description,
-                         pipeline=self.parameters['ZUUL_PIPELINE'])
-        )
-
-        self.job.sendWorkData(json.dumps(data))
-        if work_fail:
-            self.job.sendWorkFail()
-        else:
-            self.job.sendWorkComplete(json.dumps(data))
-        del self.worker.gearman_jobs[self.job.unique]
-        self.worker.running_builds.remove(self)
-        self.worker.lock.release()
-
-
-class FakeWorker(gear.Worker):
-    def __init__(self, worker_id, test):
-        super(FakeWorker, self).__init__(worker_id)
-        self.gearman_jobs = {}
-        self.build_history = []
-        self.running_builds = []
-        self.build_counter = 0
-        self.fail_tests = {}
-        self.test = test
-
-        self.hold_jobs_in_build = False
-        self.lock = threading.Lock()
-        self.__work_thread = threading.Thread(target=self.work)
-        self.__work_thread.daemon = True
-        self.__work_thread.start()
-
-    def handleJob(self, job):
-        parts = job.name.split(":")
-        cmd = parts[0]
-        name = parts[1]
-        if len(parts) > 2:
-            node = parts[2]
-        else:
-            node = None
-        if cmd == 'build':
-            self.handleBuild(job, name, node)
-        elif cmd == 'stop':
-            self.handleStop(job, name)
-        elif cmd == 'set_description':
-            self.handleSetDescription(job, name)
-
-    def handleBuild(self, job, name, node):
-        build = FakeBuild(self, job, self.build_counter, node)
-        job.build = build
-        self.gearman_jobs[job.unique] = job
-        self.build_counter += 1
-
-        self.running_builds.append(build)
-        build.start()
-
-    def handleStop(self, job, name):
-        self.log.debug("handle stop")
-        parameters = json.loads(job.arguments)
-        name = parameters['name']
-        number = parameters['number']
-        for build in self.running_builds:
-            if build.name == name and build.number == number:
-                build.aborted = True
-                build.release()
-                job.sendWorkComplete()
-                return
-        job.sendWorkFail()
-
-    def handleSetDescription(self, job, name):
-        self.log.debug("handle set description")
-        parameters = json.loads(job.arguments)
-        name = parameters['name']
-        number = parameters['number']
-        descr = parameters['html_description']
-        for build in self.running_builds:
-            if build.name == name and build.number == number:
-                build.description = descr
-                job.sendWorkComplete()
-                return
-        for build in self.build_history:
-            if build.name == name and build.number == number:
-                build.description = descr
-                job.sendWorkComplete()
-                return
-        job.sendWorkFail()
-
-    def work(self):
-        while self.running:
-            try:
-                job = self.getJob()
-            except gear.InterruptedError:
-                continue
-            try:
-                self.handleJob(job)
-            except:
-                self.log.exception("Worker exception:")
-
-    def addFailTest(self, name, change):
-        l = self.fail_tests.get(name, [])
-        l.append(change)
-        self.fail_tests[name] = l
-
-    def shouldFailTest(self, name, ref):
-        l = self.fail_tests.get(name, [])
-        for change in l:
-            if self.test.ref_has_change(ref, change):
-                return True
-        return False
-
-    def release(self, regex=None):
-        builds = self.running_builds[:]
-        self.log.debug("releasing build %s (%s)" % (regex,
-                                                    len(self.running_builds)))
-        for build in builds:
-            if not regex or re.match(regex, build.name):
-                self.log.debug("releasing build %s" %
-                               (build.parameters['ZUUL_UUID']))
-                build.release()
-            else:
-                self.log.debug("not releasing build %s" %
-                               (build.parameters['ZUUL_UUID']))
-        self.log.debug("done releasing builds %s (%s)" %
-                       (regex, len(self.running_builds)))
-
-
-class FakeGearmanServer(gear.Server):
-    def __init__(self):
-        self.hold_jobs_in_queue = False
-        super(FakeGearmanServer, self).__init__(0)
-
-    def getJobForConnection(self, connection, peek=False):
-        for queue in [self.high_queue, self.normal_queue, self.low_queue]:
-            for job in queue:
-                if not hasattr(job, 'waiting'):
-                    if job.name.startswith('build:'):
-                        job.waiting = self.hold_jobs_in_queue
-                    else:
-                        job.waiting = False
-                if job.waiting:
-                    continue
-                if job.name in connection.functions:
-                    if not peek:
-                        queue.remove(job)
-                        connection.related_jobs[job.handle] = job
-                        job.worker_connection = connection
-                    job.running = True
-                    return job
-        return None
-
-    def release(self, regex=None):
-        released = False
-        qlen = (len(self.high_queue) + len(self.normal_queue) +
-                len(self.low_queue))
-        self.log.debug("releasing queued job %s (%s)" % (regex, qlen))
-        for job in self.getQueue():
-            cmd, name = job.name.split(':')
-            if cmd != 'build':
-                continue
-            if not regex or re.match(regex, name):
-                self.log.debug("releasing queued job %s" %
-                               job.unique)
-                job.waiting = False
-                released = True
-            else:
-                self.log.debug("not releasing queued job %s" %
-                               job.unique)
-        if released:
-            self.wakeConnections()
-        qlen = (len(self.high_queue) + len(self.normal_queue) +
-                len(self.low_queue))
-        self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen))
-
-
-class FakeSMTP(object):
-    log = logging.getLogger('zuul.FakeSMTP')
-
-    def __init__(self, messages, server, port):
-        self.server = server
-        self.port = port
-        self.messages = messages
-
-    def sendmail(self, from_email, to_email, msg):
-        self.log.info("Sending email from %s, to %s, with msg %s" % (
-                      from_email, to_email, msg))
-
-        headers = msg.split('\n\n', 1)[0]
-        body = msg.split('\n\n', 1)[1]
-
-        self.messages.append(dict(
-            from_email=from_email,
-            to_email=to_email,
-            msg=msg,
-            headers=headers,
-            body=body,
-        ))
-
-        return True
-
-    def quit(self):
-        return True
-
-
-class FakeSwiftClientConnection(swiftclient.client.Connection):
-    def post_account(self, headers):
-        # Do nothing
-        pass
-
-    def get_auth(self):
-        # Returns endpoint and (unused) auth token
-        endpoint = os.path.join('https://storage.example.org', 'V1',
-                                'AUTH_account')
-        return endpoint, ''
-
-
-class TestScheduler(testtools.TestCase):
-    log = logging.getLogger("zuul.test")
-
-    def setUp(self):
-        super(TestScheduler, self).setUp()
-        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
-        try:
-            test_timeout = int(test_timeout)
-        except ValueError:
-            # If timeout value is invalid do not set a timeout.
-            test_timeout = 0
-        if test_timeout > 0:
-            self.useFixture(fixtures.Timeout(test_timeout, gentle=False))
-
-        if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
-            os.environ.get('OS_STDOUT_CAPTURE') == '1'):
-            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
-            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
-        if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
-            os.environ.get('OS_STDERR_CAPTURE') == '1'):
-            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
-            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
-        if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
-            os.environ.get('OS_LOG_CAPTURE') == '1'):
-            self.useFixture(fixtures.FakeLogger(
-                level=logging.DEBUG,
-                format='%(asctime)s %(name)-32s '
-                '%(levelname)-8s %(message)s'))
-        tmp_root = self.useFixture(fixtures.TempDir(
-            rootdir=os.environ.get("ZUUL_TEST_ROOT"))).path
-        self.test_root = os.path.join(tmp_root, "zuul-test")
-        self.upstream_root = os.path.join(self.test_root, "upstream")
-        self.git_root = os.path.join(self.test_root, "git")
-
-        CONFIG.set('merger', 'git_dir', self.git_root)
-        if os.path.exists(self.test_root):
-            shutil.rmtree(self.test_root)
-        os.makedirs(self.test_root)
-        os.makedirs(self.upstream_root)
-        os.makedirs(self.git_root)
-
-        # For each project in config:
-        self.init_repo("org/project")
-        self.init_repo("org/project1")
-        self.init_repo("org/project2")
-        self.init_repo("org/project3")
-        self.init_repo("org/one-job-project")
-        self.init_repo("org/nonvoting-project")
-        self.init_repo("org/templated-project")
-        self.init_repo("org/layered-project")
-        self.init_repo("org/node-project")
-        self.init_repo("org/conflict-project")
-        self.init_repo("org/noop-project")
-        self.init_repo("org/experimental-project")
-
-        self.statsd = FakeStatsd()
-        os.environ['STATSD_HOST'] = 'localhost'
-        os.environ['STATSD_PORT'] = str(self.statsd.port)
-        self.statsd.start()
-        # the statsd client object is configured in the statsd module import
-        reload(statsd)
-        reload(zuul.scheduler)
-
-        self.gearman_server = FakeGearmanServer()
-
-        self.config = ConfigParser.ConfigParser()
-        cfg = StringIO()
-        CONFIG.write(cfg)
-        cfg.seek(0)
-        self.config.readfp(cfg)
-        self.config.set('gearman', 'port', str(self.gearman_server.port))
-
-        self.worker = FakeWorker('fake_worker', self)
-        self.worker.addServer('127.0.0.1', self.gearman_server.port)
-        self.gearman_server.worker = self.worker
-
-        self.merge_server = zuul.merger.server.MergeServer(self.config)
-        self.merge_server.start()
-
-        self.sched = zuul.scheduler.Scheduler()
-
-        self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
-                                             FakeSwiftClientConnection))
-        self.swift = zuul.lib.swift.Swift(self.config)
-
-        def URLOpenerFactory(*args, **kw):
-            if isinstance(args[0], urllib2.Request):
-                return old_urlopen(*args, **kw)
-            args = [self.fake_gerrit] + list(args)
-            return FakeURLOpener(self.upstream_root, *args, **kw)
-
-        old_urlopen = urllib2.urlopen
-        urllib2.urlopen = URLOpenerFactory
-
-        self.launcher = zuul.launcher.gearman.Gearman(self.config, self.sched,
-                                                      self.swift)
-        self.merge_client = zuul.merger.client.MergeClient(
-            self.config, self.sched)
-
-        self.smtp_messages = []
-
-        def FakeSMTPFactory(*args, **kw):
-            args = [self.smtp_messages] + list(args)
-            return FakeSMTP(*args, **kw)
-
-        zuul.lib.gerrit.Gerrit = FakeGerrit
-        self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
-
-        self.gerrit = FakeGerritTrigger(
-            self.upstream_root, self.config, self.sched)
-        self.gerrit.replication_timeout = 1.5
-        self.gerrit.replication_retry_interval = 0.5
-        self.fake_gerrit = self.gerrit.gerrit
-        self.fake_gerrit.upstream_root = self.upstream_root
-
-        self.webapp = zuul.webapp.WebApp(self.sched, port=0)
-        self.rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
-
-        self.sched.setLauncher(self.launcher)
-        self.sched.setMerger(self.merge_client)
-        self.sched.registerTrigger(self.gerrit)
-        self.timer = zuul.trigger.timer.Timer(self.config, self.sched)
-        self.sched.registerTrigger(self.timer)
-
-        self.sched.registerReporter(
-            zuul.reporter.gerrit.Reporter(self.gerrit))
-        self.smtp_reporter = zuul.reporter.smtp.Reporter(
-            self.config.get('smtp', 'default_from'),
-            self.config.get('smtp', 'default_to'),
-            self.config.get('smtp', 'server'))
-        self.sched.registerReporter(self.smtp_reporter)
-
-        self.sched.start()
-        self.sched.reconfigure(self.config)
-        self.sched.resume()
-        self.webapp.start()
-        self.rpc.start()
-        self.launcher.gearman.waitForServer()
-        self.registerJobs()
-        self.builds = self.worker.running_builds
-        self.history = self.worker.build_history
-
-        self.addCleanup(self.assertFinalState)
-        self.addCleanup(self.shutdown)
-
-    def assertFinalState(self):
-        # Make sure that the change cache is cleared
-        self.assertEqual(len(self.gerrit._change_cache.keys()), 0)
-        # Make sure that git.Repo objects have been garbage collected.
-        repos = []
-        gc.collect()
-        for obj in gc.get_objects():
-            if isinstance(obj, git.Repo):
-                repos.append(obj)
-        self.assertEqual(len(repos), 0)
-        self.assertEmptyQueues()
-
-    def shutdown(self):
-        self.log.debug("Shutting down after tests")
-        self.launcher.stop()
-        self.merge_server.stop()
-        self.merge_server.join()
-        self.merge_client.stop()
-        self.worker.shutdown()
-        self.gerrit.stop()
-        self.timer.stop()
-        self.sched.stop()
-        self.sched.join()
-        self.statsd.stop()
-        self.statsd.join()
-        self.webapp.stop()
-        self.webapp.join()
-        self.rpc.stop()
-        self.rpc.join()
-        self.gearman_server.shutdown()
-        threads = threading.enumerate()
-        if len(threads) > 1:
-            self.log.error("More than one thread is running: %s" % threads)
-        super(TestScheduler, self).tearDown()
-
-    def init_repo(self, project):
-        parts = project.split('/')
-        path = os.path.join(self.upstream_root, *parts[:-1])
-        if not os.path.exists(path):
-            os.makedirs(path)
-        path = os.path.join(self.upstream_root, project)
-        repo = git.Repo.init(path)
-
-        repo.config_writer().set_value('user', 'email', 'user@example.com')
-        repo.config_writer().set_value('user', 'name', 'User Name')
-        repo.config_writer().write()
-
-        fn = os.path.join(path, 'README')
-        f = open(fn, 'w')
-        f.write("test\n")
-        f.close()
-        repo.index.add([fn])
-        repo.index.commit('initial commit')
-        master = repo.create_head('master')
-        repo.create_tag('init')
-
-        mp = repo.create_head('mp')
-        repo.head.reference = mp
-        f = open(fn, 'a')
-        f.write("test mp\n")
-        f.close()
-        repo.index.add([fn])
-        repo.index.commit('mp commit')
-
-        repo.head.reference = master
-        repo.head.reset(index=True, working_tree=True)
-        repo.git.clean('-x', '-f', '-d')
-
-    def ref_has_change(self, ref, change):
-        path = os.path.join(self.git_root, change.project)
-        repo = git.Repo(path)
-        for commit in repo.iter_commits(ref):
-            if commit.message.strip() == ('%s-1' % change.subject):
-                return True
-        return False
-
-    def job_has_changes(self, *args):
-        job = args[0]
-        commits = args[1:]
-        if isinstance(job, FakeBuild):
-            parameters = job.parameters
-        else:
-            parameters = json.loads(job.arguments)
-        project = parameters['ZUUL_PROJECT']
-        path = os.path.join(self.git_root, project)
-        repo = git.Repo(path)
-        ref = parameters['ZUUL_REF']
-        sha = parameters['ZUUL_COMMIT']
-        repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
-        repo_shas = [c.hexsha for c in repo.iter_commits(ref)]
-        commit_messages = ['%s-1' % commit.subject for commit in commits]
-        self.log.debug("Checking if job %s has changes; commit_messages %s;"
-                       " repo_messages %s; sha %s" % (job, commit_messages,
-                                                      repo_messages, sha))
-        for msg in commit_messages:
-            if msg not in repo_messages:
-                self.log.debug("  messages do not match")
-                return False
-        if repo_shas[0] != sha:
-            self.log.debug("  sha does not match")
-            return False
-        self.log.debug("  OK")
-        return True
-
-    def registerJobs(self):
-        count = 0
-        for job in self.sched.layout.jobs.keys():
-            self.worker.registerFunction('build:' + job)
-            count += 1
-        self.worker.registerFunction('stop:' + self.worker.worker_id)
-        count += 1
-
-        while len(self.gearman_server.functions) < count:
-            time.sleep(0)
-
-    def release(self, job):
-        if isinstance(job, FakeBuild):
-            job.release()
-        else:
-            job.waiting = False
-            self.log.debug("Queued job %s released" % job.unique)
-            self.gearman_server.wakeConnections()
-
-    def getParameter(self, job, name):
-        if isinstance(job, FakeBuild):
-            return job.parameters[name]
-        else:
-            parameters = json.loads(job.arguments)
-            return parameters[name]
-
-    def resetGearmanServer(self):
-        self.worker.setFunctions([])
-        while True:
-            done = True
-            for connection in self.gearman_server.active_connections:
-                if (connection.functions and
-                    connection.client_id not in ['Zuul RPC Listener',
-                                                 'Zuul Merger']):
-                    done = False
-            if done:
-                break
-            time.sleep(0)
-        self.gearman_server.functions = set()
-        self.rpc.register()
-        self.merge_server.register()
-
-    def haveAllBuildsReported(self):
-        # See if Zuul is waiting on a meta job to complete
-        if self.launcher.meta_jobs:
-            return False
-        # Find out if every build that the worker has completed has been
-        # reported back to Zuul.  If it hasn't then that means a Gearman
-        # event is still in transit and the system is not stable.
-        for build in self.worker.build_history:
-            zbuild = self.launcher.builds.get(build.uuid)
-            if not zbuild:
-                # It has already been reported
-                continue
-            # It hasn't been reported yet.
-            return False
-        # Make sure that none of the worker connections are in GRAB_WAIT
-        for connection in self.worker.active_connections:
-            if connection.state == 'GRAB_WAIT':
-                return False
-        return True
-
-    def areAllBuildsWaiting(self):
-        ret = True
-
-        builds = self.launcher.builds.values()
-        for build in builds:
-            client_job = None
-            for conn in self.launcher.gearman.active_connections:
-                for j in conn.related_jobs.values():
-                    if j.unique == build.uuid:
-                        client_job = j
-                        break
-            if not client_job:
-                self.log.debug("%s is not known to the gearman client" %
-                               build)
-                ret = False
-                continue
-            if not client_job.handle:
-                self.log.debug("%s has no handle" % client_job)
-                ret = False
-                continue
-            server_job = self.gearman_server.jobs.get(client_job.handle)
-            if not server_job:
-                self.log.debug("%s is not known to the gearman server" %
-                               client_job)
-                ret = False
-                continue
-            if not hasattr(server_job, 'waiting'):
-                self.log.debug("%s is being enqueued" % server_job)
-                ret = False
-                continue
-            if server_job.waiting:
-                continue
-            worker_job = self.worker.gearman_jobs.get(server_job.unique)
-            if worker_job:
-                if worker_job.build.isWaiting():
-                    continue
-                else:
-                    self.log.debug("%s is running" % worker_job)
-                    ret = False
-            else:
-                self.log.debug("%s is unassigned" % server_job)
-                ret = False
-        return ret
-
-    def waitUntilSettled(self):
-        self.log.debug("Waiting until settled...")
-        start = time.time()
-        while True:
-            if time.time() - start > 10:
-                print 'queue status:',
-                print self.sched.trigger_event_queue.empty(),
-                print self.sched.result_event_queue.empty(),
-                print self.fake_gerrit.event_queue.empty(),
-                print self.areAllBuildsWaiting()
-                raise Exception("Timeout waiting for Zuul to settle")
-            # Make sure no new events show up while we're checking
-            self.worker.lock.acquire()
-            # have all build states propagated to zuul?
-            if self.haveAllBuildsReported():
-                # Join ensures that the queue is empty _and_ events have been
-                # processed
-                self.fake_gerrit.event_queue.join()
-                self.sched.trigger_event_queue.join()
-                self.sched.result_event_queue.join()
-                self.sched.run_handler_lock.acquire()
-                if (self.sched.trigger_event_queue.empty() and
-                    self.sched.result_event_queue.empty() and
-                    self.fake_gerrit.event_queue.empty() and
-                    not self.merge_client.build_sets and
-                    self.haveAllBuildsReported() and
-                    self.areAllBuildsWaiting()):
-                    self.sched.run_handler_lock.release()
-                    self.worker.lock.release()
-                    self.log.debug("...settled.")
-                    return
-                self.sched.run_handler_lock.release()
-            self.worker.lock.release()
-            self.sched.wake_event.wait(0.1)
-
-    def countJobResults(self, jobs, result):
-        jobs = filter(lambda x: x.result == result, jobs)
-        return len(jobs)
-
-    def getJobFromHistory(self, name):
-        history = self.worker.build_history
-        for job in history:
-            if job.name == name:
-                return job
-        raise Exception("Unable to find job %s in history" % name)
-
-    def assertEmptyQueues(self):
-        # Make sure there are no orphaned jobs
-        for pipeline in self.sched.layout.pipelines.values():
-            for queue in pipeline.queues:
-                if len(queue.queue) != 0:
-                    print 'pipeline %s queue %s contents %s' % (
-                        pipeline.name, queue.name, queue.queue)
-                self.assertEqual(len(queue.queue), 0)
-
-    def assertReportedStat(self, key, value=None, kind=None):
-        start = time.time()
-        while time.time() < (start + 5):
-            for stat in self.statsd.stats:
-                pprint.pprint(self.statsd.stats)
-                k, v = stat.split(':')
-                if key == k:
-                    if value is None and kind is None:
-                        return
-                    elif value:
-                        if value == v:
-                            return
-                    elif kind:
-                        if v.endswith('|' + kind):
-                            return
-            time.sleep(0.1)
-
-        pprint.pprint(self.statsd.stats)
-        raise Exception("Key %s not found in reported stats" % key)
-
+class TestScheduler(ZuulTestCase):
     def test_jobs_launched(self):
         "Test that jobs are launched and a change is merged"
 
@@ -3028,7 +1862,7 @@
         sched = zuul.scheduler.Scheduler()
         sched.registerTrigger(None, 'gerrit')
         sched.registerTrigger(None, 'timer')
-        sched.testConfig(CONFIG.get('zuul', 'layout_config'))
+        sched.testConfig(self.config.get('zuul', 'layout_config'))
 
     def test_build_description(self):
         "Test that build descriptions update"
@@ -3275,25 +2109,25 @@
         self.sched.reconfigure(self.config)
         self.registerJobs()
 
-        start = time.time()
-        failed = True
-        while ((time.time() - start) < 30):
-            if len(self.builds) == 2:
-                failed = False
-                break
-            else:
-                time.sleep(1)
-
-        if failed:
-            raise Exception("Expected jobs never ran")
-
+        # The pipeline triggers every second, so we should have seen
+        # several by now.
+        time.sleep(5)
         self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 2)
+
         port = self.webapp.server.socket.getsockname()[1]
 
         f = urllib.urlopen("http://localhost:%s/status.json" % port)
         data = f.read()
 
         self.worker.hold_jobs_in_build = False
+        # Stop queuing timer triggered jobs so that the assertions
+        # below don't race against more jobs being queued.
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-no-timer.yaml')
+        self.sched.reconfigure(self.config)
+        self.registerJobs()
         self.worker.release()
         self.waitUntilSettled()
 
@@ -3317,29 +2151,33 @@
     def test_idle(self):
         "Test that frequent periodic jobs work"
         self.worker.hold_jobs_in_build = True
-        self.config.set('zuul', 'layout_config',
-                        'tests/fixtures/layout-idle.yaml')
-        self.sched.reconfigure(self.config)
-        self.registerJobs()
 
-        # The pipeline triggers every second, so we should have seen
-        # several by now.
-        time.sleep(5)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 2)
-        self.worker.release('.*')
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-        self.assertEqual(len(self.history), 2)
+        for x in range(1, 3):
+            # Test that timer triggers periodic jobs even across
+            # layout config reloads.
+            # Start timer trigger
+            self.config.set('zuul', 'layout_config',
+                            'tests/fixtures/layout-idle.yaml')
+            self.sched.reconfigure(self.config)
+            self.registerJobs()
 
-        time.sleep(5)
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 2)
-        self.assertEqual(len(self.history), 2)
-        self.worker.release('.*')
-        self.waitUntilSettled()
-        self.assertEqual(len(self.builds), 0)
-        self.assertEqual(len(self.history), 4)
+            # The pipeline triggers every second, so we should have seen
+            # several by now.
+            time.sleep(5)
+            self.waitUntilSettled()
+
+            # Stop queuing timer triggered jobs so that the assertions
+            # below don't race against more jobs being queued.
+            self.config.set('zuul', 'layout_config',
+                            'tests/fixtures/layout-no-timer.yaml')
+            self.sched.reconfigure(self.config)
+            self.registerJobs()
+
+            self.assertEqual(len(self.builds), 2)
+            self.worker.release('.*')
+            self.waitUntilSettled()
+            self.assertEqual(len(self.builds), 0)
+            self.assertEqual(len(self.history), x * 2)
 
     def test_check_smtp_pool(self):
         self.config.set('zuul', 'layout_config',
@@ -3374,25 +2212,22 @@
 
     def test_timer_smtp(self):
         "Test that a periodic job is triggered"
+        self.worker.hold_jobs_in_build = True
         self.config.set('zuul', 'layout_config',
                         'tests/fixtures/layout-timer-smtp.yaml')
         self.sched.reconfigure(self.config)
         self.registerJobs()
 
-        start = time.time()
-        failed = True
-        while ((time.time() - start) < 30):
-            if len(self.history) == 2:
-                failed = False
-                break
-            else:
-                time.sleep(1)
-
-        if failed:
-            raise Exception("Expected jobs never ran")
-
+        # The pipeline triggers every second, so we should have seen
+        # several by now.
+        time.sleep(5)
         self.waitUntilSettled()
 
+        self.assertEqual(len(self.builds), 2)
+        self.worker.release('.*')
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 2)
+
         self.assertEqual(self.getJobFromHistory(
             'project-bitrot-stable-old').result, 'SUCCESS')
         self.assertEqual(self.getJobFromHistory(
@@ -3411,6 +2246,15 @@
         self.assertIn('Subject: Periodic check for org/project succeeded',
                       self.smtp_messages[0]['headers'])
 
+        # Stop queuing timer triggered jobs and let any that may have
+        # queued through so that end of test assertions pass.
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-no-timer.yaml')
+        self.sched.reconfigure(self.config)
+        self.registerJobs()
+        self.worker.release('.*')
+        self.waitUntilSettled()
+
     def test_client_enqueue(self):
         "Test that the RPC client can enqueue a change"
         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')