Merge "Add support for sqlalchemy reporter"
diff --git a/doc/source/connections.rst b/doc/source/connections.rst
index e0d88d0..298100a 100644
--- a/doc/source/connections.rst
+++ b/doc/source/connections.rst
@@ -38,6 +38,9 @@
Path to SSH key to use when logging into above server.
``sshkey=/home/zuul/.ssh/id_rsa``
+**keepalive**
+ Optional: Keepalive timeout, 0 means no keepalive.
+ ``keepalive=60``
Gerrit Configuration
~~~~~~~~~~~~~~~~~~~~
diff --git a/etc/status/public_html/jquery.zuul.js b/etc/status/public_html/jquery.zuul.js
index 9df44ce..d973948 100644
--- a/etc/status/public_html/jquery.zuul.js
+++ b/etc/status/public_html/jquery.zuul.js
@@ -148,11 +148,9 @@
case 'skipped':
$status.addClass('label-info');
break;
- case 'in progress':
- case 'queued':
- case 'lost':
+ // 'in progress' 'queued' 'lost' 'aborted' ...
+ default:
$status.addClass('label-default');
- break;
}
$status.text(result);
return $status;
diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample
index df4c008..9998a70 100644
--- a/etc/zuul.conf-sample
+++ b/etc/zuul.conf-sample
@@ -36,6 +36,7 @@
;baseurl=https://review.example.com/r
user=jenkins
sshkey=/home/jenkins/.ssh/id_rsa
+;keepalive=60
[connection smtp]
driver=smtp
diff --git a/tests/fixtures/layout-mutex-reconfiguration.yaml b/tests/fixtures/layout-mutex-reconfiguration.yaml
new file mode 100644
index 0000000..76cf1e9
--- /dev/null
+++ b/tests/fixtures/layout-mutex-reconfiguration.yaml
@@ -0,0 +1,23 @@
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+jobs:
+ - name: mutex-one
+ mutex: test-mutex
+ - name: mutex-two
+ mutex: test-mutex
+
+projects:
+ - name: org/project
+ check:
+ - project-test1
diff --git a/tests/fixtures/zuul-connections-same-gerrit.conf b/tests/fixtures/zuul-connections-same-gerrit.conf
index 8c76c6c..2609d30 100644
--- a/tests/fixtures/zuul-connections-same-gerrit.conf
+++ b/tests/fixtures/zuul-connections-same-gerrit.conf
@@ -26,13 +26,13 @@
driver=gerrit
server=review.example.com
user=jenkins
-sshkey=none
+sshkey=fake_id_rsa1
[connection alt_voting_gerrit]
driver=gerrit
server=review.example.com
user=civoter
-sshkey=none
+sshkey=fake_id_rsa2
[connection outgoing_smtp]
driver=smtp
diff --git a/tests/fixtures/zuul.conf b/tests/fixtures/zuul.conf
index b250c6d..0956cc4 100644
--- a/tests/fixtures/zuul.conf
+++ b/tests/fixtures/zuul.conf
@@ -26,7 +26,7 @@
driver=gerrit
server=review.example.com
user=jenkins
-sshkey=none
+sshkey=fake_id_rsa_path
[connection smtp]
driver=smtp
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index b6fa4a3..d205395 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2394,6 +2394,63 @@
self.assertEqual(B.reported, 1)
self.assertFalse('test-mutex' in self.sched.mutex.mutexes)
+ def test_mutex_abandon(self):
+ "Test abandon with job mutexes"
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-mutex.yaml')
+ self.sched.reconfigure(self.config)
+
+ self.worker.hold_jobs_in_build = True
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.assertFalse('test-mutex' in self.sched.mutex.mutexes)
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertTrue('test-mutex' in self.sched.mutex.mutexes)
+
+ self.fake_gerrit.addEvent(A.getChangeAbandonedEvent())
+ self.waitUntilSettled()
+
+ # The check pipeline should be empty
+ items = self.sched.layout.pipelines['check'].getAllItems()
+ self.assertEqual(len(items), 0)
+
+ # The mutex should be released
+ self.assertFalse('test-mutex' in self.sched.mutex.mutexes)
+
+ def test_mutex_reconfigure(self):
+ "Test reconfigure with job mutexes"
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-mutex.yaml')
+ self.sched.reconfigure(self.config)
+
+ self.worker.hold_jobs_in_build = True
+
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ self.assertFalse('test-mutex' in self.sched.mutex.mutexes)
+
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+
+ self.assertTrue('test-mutex' in self.sched.mutex.mutexes)
+
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-mutex-reconfiguration.yaml')
+ self.sched.reconfigure(self.config)
+ self.waitUntilSettled()
+
+ self.worker.release('project-test1')
+ self.waitUntilSettled()
+
+ # The check pipeline should be empty
+ items = self.sched.layout.pipelines['check'].getAllItems()
+ self.assertEqual(len(items), 0)
+
+ # The mutex should be released
+ self.assertFalse('test-mutex' in self.sched.mutex.mutexes)
+
def test_node_label(self):
"Test that a job runs on a specific node label"
self.worker.registerFunction('build:node-project-test1:debian')
@@ -3020,6 +3077,49 @@
self.worker.release('.*')
self.waitUntilSettled()
+ def test_timer_sshkey(self):
+ "Test that a periodic job can setup SSH key authentication"
+ self.worker.hold_jobs_in_build = True
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-timer.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
+
+ # The pipeline triggers every second, so we should have seen
+ # several by now.
+ time.sleep(5)
+ self.waitUntilSettled()
+
+ self.assertEqual(len(self.builds), 2)
+
+ ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit")
+ self.assertTrue(os.path.isfile(ssh_wrapper))
+ with open(ssh_wrapper) as f:
+ ssh_wrapper_content = f.read()
+ self.assertIn("fake_id_rsa", ssh_wrapper_content)
+ # In the unit tests Merger runs in the same process,
+ # so we see its' environment variables
+ self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper)
+
+ self.worker.release('.*')
+ self.waitUntilSettled()
+ self.assertEqual(len(self.history), 2)
+
+ self.assertEqual(self.getJobFromHistory(
+ 'project-bitrot-stable-old').result, 'SUCCESS')
+ self.assertEqual(self.getJobFromHistory(
+ 'project-bitrot-stable-older').result, 'SUCCESS')
+
+ # Stop queuing timer triggered jobs and let any that may have
+ # queued through so that end of test assertions pass.
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-no-timer.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
+ self.waitUntilSettled()
+ self.worker.release('.*')
+ self.waitUntilSettled()
+
def test_client_enqueue_change(self):
"Test that the RPC client can enqueue a change"
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
diff --git a/zuul/connection/gerrit.py b/zuul/connection/gerrit.py
index 62891cd..6e8d085 100644
--- a/zuul/connection/gerrit.py
+++ b/zuul/connection/gerrit.py
@@ -63,7 +63,7 @@
if change:
event.project_name = change.get('project')
event.branch = change.get('branch')
- event.change_number = change.get('number')
+ event.change_number = str(change.get('number'))
event.change_url = change.get('url')
patchset = data.get('patchSet')
if patchset:
@@ -135,13 +135,14 @@
poll_timeout = 500
def __init__(self, gerrit_connection, username, hostname, port=29418,
- keyfile=None):
+ keyfile=None, keepalive=60):
threading.Thread.__init__(self)
self.username = username
self.keyfile = keyfile
self.hostname = hostname
self.port = port
self.gerrit_connection = gerrit_connection
+ self.keepalive = keepalive
self._stopped = False
def _read(self, fd):
@@ -172,6 +173,8 @@
username=self.username,
port=self.port,
key_filename=self.keyfile)
+ transport = client.get_transport()
+ transport.set_keepalive(self.keepalive)
stdin, stdout, stderr = client.exec_command("gerrit stream-events")
@@ -208,7 +211,7 @@
class GerritConnection(BaseConnection):
driver_name = 'gerrit'
- log = logging.getLogger("connection.gerrit")
+ log = logging.getLogger("zuul.GerritConnection")
def __init__(self, connection_name, connection_config):
super(GerritConnection, self).__init__(connection_name,
@@ -224,6 +227,7 @@
self.server = self.connection_config.get('server')
self.port = int(self.connection_config.get('port', 29418))
self.keyfile = self.connection_config.get('sshkey', None)
+ self.keepalive = int(self.connection_config.get('keepalive', 60))
self.watcher_thread = None
self.event_queue = None
self.client = None
@@ -356,6 +360,8 @@
username=self.user,
port=self.port,
key_filename=self.keyfile)
+ transport = client.get_transport()
+ transport.set_keepalive(self.keepalive)
self.client = client
def _ssh(self, command, stdin_data=None):
@@ -461,7 +467,8 @@
self.user,
self.server,
self.port,
- keyfile=self.keyfile)
+ keyfile=self.keyfile,
+ keepalive=self.keepalive)
self.watcher_thread.start()
def _stop_event_connector(self):
diff --git a/zuul/connection/smtp.py b/zuul/connection/smtp.py
index d3eccff..125cb15 100644
--- a/zuul/connection/smtp.py
+++ b/zuul/connection/smtp.py
@@ -23,7 +23,7 @@
class SMTPConnection(BaseConnection):
driver_name = 'smtp'
- log = logging.getLogger("connection.smtp")
+ log = logging.getLogger("zuul.SMTPConnection")
def __init__(self, connection_name, connection_config):
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 952bbbd..7d47775 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+import logging
import re
import zuul.connection.gerrit
@@ -20,6 +21,7 @@
def configure_connections(config):
+ log = logging.getLogger("configure_connections")
# Register connections from the config
# TODO(jhesketh): import connection modules dynamically
@@ -58,13 +60,21 @@
# connection named 'gerrit' or 'smtp' respectfully
if 'gerrit' in config.sections():
- connections['gerrit'] = \
- zuul.connection.gerrit.GerritConnection(
- 'gerrit', dict(config.items('gerrit')))
+ if 'gerrit' in connections:
+ log.warning("The legacy [gerrit] section will be ignored in favour"
+ " of the [connection gerrit].")
+ else:
+ connections['gerrit'] = \
+ zuul.connection.gerrit.GerritConnection(
+ 'gerrit', dict(config.items('gerrit')))
if 'smtp' in config.sections():
- connections['smtp'] = \
- zuul.connection.smtp.SMTPConnection(
- 'smtp', dict(config.items('smtp')))
+ if 'smtp' in connections:
+ log.warning("The legacy [smtp] section will be ignored in favour"
+ " of the [connection smtp].")
+ else:
+ connections['smtp'] = \
+ zuul.connection.smtp.SMTPConnection(
+ 'smtp', dict(config.items('smtp')))
return connections
diff --git a/zuul/lib/swift.py b/zuul/lib/swift.py
index b5d3bc7..5660819 100644
--- a/zuul/lib/swift.py
+++ b/zuul/lib/swift.py
@@ -24,7 +24,7 @@
class Swift(object):
- log = logging.getLogger("zuul.lib.swift")
+ log = logging.getLogger("zuul.Swift")
def __init__(self, config):
self.config = config
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index 950c385..9e8c243 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -97,9 +97,10 @@
data = dict(items=items)
self.submitJob('merger:merge', data, build_set, precedence)
- def updateRepo(self, project, url, build_set,
+ def updateRepo(self, project, connection_name, url, build_set,
precedence=zuul.model.PRECEDENCE_NORMAL):
data = dict(project=project,
+ connection_name=connection_name,
url=url)
self.submitJob('merger:update', data, build_set, precedence)
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index b82a7de..a974e9c 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -223,6 +223,14 @@
fd.close()
os.chmod(name, 0o755)
+ def _setGitSsh(self, connection_name):
+ wrapper_name = '.ssh_wrapper_%s' % connection_name
+ name = os.path.join(self.working_root, wrapper_name)
+ if os.path.isfile(name):
+ os.environ['GIT_SSH'] = name
+ elif 'GIT_SSH' in os.environ:
+ del os.environ['GIT_SSH']
+
def addProject(self, project, url):
repo = None
try:
@@ -242,7 +250,8 @@
" without a url" % (project,))
return self.addProject(project, url)
- def updateRepo(self, project, url):
+ def updateRepo(self, project, connection_name, url):
+ self._setGitSsh(connection_name)
repo = self.getRepo(project, url)
try:
self.log.info("Updating local repository %s", project)
@@ -279,14 +288,6 @@
return commit
- def _setGitSsh(self, connection_name):
- wrapper_name = '.ssh_wrapper_%s' % connection_name
- name = os.path.join(self.working_root, wrapper_name)
- if os.path.isfile(name):
- os.environ['GIT_SSH'] = name
- elif 'GIT_SSH' in os.environ:
- del os.environ['GIT_SSH']
-
def _mergeItem(self, item, recent):
self.log.debug("Processing refspec %s for project %s / %s ref %s" %
(item['refspec'], item['project'], item['branch'],
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index d56993c..b1921d9 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -109,7 +109,9 @@
def update(self, job):
args = json.loads(job.arguments)
- self.merger.updateRepo(args['project'], args['url'])
+ self.merger.updateRepo(args['project'],
+ args['connection_name'],
+ args['url'])
result = dict(updated=True,
zuul_url=self.zuul_url)
job.sendWorkComplete(json.dumps(result))
diff --git a/zuul/reporter/gerrit.py b/zuul/reporter/gerrit.py
index 1427449..d9c671d 100644
--- a/zuul/reporter/gerrit.py
+++ b/zuul/reporter/gerrit.py
@@ -23,7 +23,7 @@
"""Sends off reports to Gerrit."""
name = 'gerrit'
- log = logging.getLogger("zuul.reporter.gerrit.Reporter")
+ log = logging.getLogger("zuul.GerritReporter")
def report(self, source, pipeline, item):
"""Send a message to gerrit."""
diff --git a/zuul/reporter/smtp.py b/zuul/reporter/smtp.py
index 586b941..3935098 100644
--- a/zuul/reporter/smtp.py
+++ b/zuul/reporter/smtp.py
@@ -22,7 +22,7 @@
"""Sends off reports to emails via SMTP."""
name = 'smtp'
- log = logging.getLogger("zuul.reporter.smtp.Reporter")
+ log = logging.getLogger("zuul.SMTPReporter")
def report(self, source, pipeline, item):
"""Send the compiled report message via smtp."""
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index 58fe23b..931571f 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -868,6 +868,8 @@
self.log.exception(
"Exception while canceling build %s "
"for change %s" % (build, item.change))
+ finally:
+ self.mutex.release(build.build_set.item, build.job)
self.layout = layout
self.maintainConnectionCache()
for trigger in self.triggers.values():
@@ -1502,8 +1504,9 @@
else:
self.log.debug("Preparing update repo for: %s" % item.change)
url = self.pipeline.source.getGitUrl(item.change.project)
+ connection_name = self.pipeline.source.connection.connection_name
self.sched.merger.updateRepo(item.change.project.name,
- url, build_set,
+ connection_name, url, build_set,
self.pipeline.precedence)
# merge:merge has been emitted properly:
build_set.merge_state = build_set.PENDING
@@ -1542,6 +1545,8 @@
except:
self.log.exception("Exception while canceling build %s "
"for change %s" % (build, item.change))
+ finally:
+ self.sched.mutex.release(build.build_set.item, build.job)
build.result = 'CANCELED'
canceled = True
self.updateBuildDescriptions(old_build_set)
diff --git a/zuul/source/gerrit.py b/zuul/source/gerrit.py
index 463f315..828e201 100644
--- a/zuul/source/gerrit.py
+++ b/zuul/source/gerrit.py
@@ -36,7 +36,7 @@
class GerritSource(BaseSource):
name = 'gerrit'
- log = logging.getLogger("zuul.source.Gerrit")
+ log = logging.getLogger("zuul.GerritSource")
replication_timeout = 300
replication_retry_interval = 5
diff --git a/zuul/trigger/gerrit.py b/zuul/trigger/gerrit.py
index 8a3fe42..c678bce 100644
--- a/zuul/trigger/gerrit.py
+++ b/zuul/trigger/gerrit.py
@@ -20,7 +20,7 @@
class GerritTrigger(BaseTrigger):
name = 'gerrit'
- log = logging.getLogger("zuul.trigger.Gerrit")
+ log = logging.getLogger("zuul.GerritTrigger")
def getEventFilters(self, trigger_conf):
def toList(item):
diff --git a/zuul/trigger/timer.py b/zuul/trigger/timer.py
index f81312e..f982914 100644
--- a/zuul/trigger/timer.py
+++ b/zuul/trigger/timer.py
@@ -23,7 +23,7 @@
class TimerTrigger(BaseTrigger):
name = 'timer'
- log = logging.getLogger("zuul.Timer")
+ log = logging.getLogger("zuul.TimerTrigger")
def __init__(self, trigger_config={}, sched=None, connection=None):
super(TimerTrigger, self).__init__(trigger_config, sched, connection)