Merge "Switch statsd config to zuul.conf" into feature/zuulv3
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 464cb60..99817f7 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -101,6 +101,27 @@
An openssl file containing the client private key in PEM format.
+.. attr:: statsd
+
+ Information about the optional statsd server. If the ``statsd``
+ python module is installed and this section is configured,
+ statistics will be reported to statsd. See :ref:`statsd` for more
+ information.
+
+ .. attr:: server
+
+ Hostname or IP address of the statsd server.
+
+ .. attr:: port
+ :default: 8125
+
+ The UDP port on which the statsd server is listening.
+
+ .. attr:: prefix
+
+ If present, this will be prefixed to all of the keys before
+ transmitting to the statsd server.
+
.. NOTE: this is a white lie at this point, since only the scheduler
uses this, however, we expect other components to use it later, so
it's reasonable for admins to plan for this now.
diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst
index a8b2324..7c2ac80 100644
--- a/doc/source/admin/monitoring.rst
+++ b/doc/source/admin/monitoring.rst
@@ -3,6 +3,8 @@
Monitoring
==========
+.. _statsd:
+
Statsd reporting
----------------
@@ -13,20 +15,11 @@
Configuration
~~~~~~~~~~~~~
-Statsd support uses the statsd python module. Note that Zuul will start without
-the statsd python module, so an existing Zuul installation may be missing it.
+Statsd support uses the ``statsd`` python module. Note that support
+is optional and Zuul will start without the statsd python module
+present.
-The configuration is done via environment variables STATSD_HOST and
-STATSD_PORT. They are interpreted by the statsd module directly and there is no
-such parameter in zuul.conf yet. Your init script will have to initialize both
-of them before executing Zuul.
-
-Your init script most probably loads a configuration file named
-``/etc/default/zuul`` which would contain the environment variables::
-
- $ cat /etc/default/zuul
- STATSD_HOST=10.0.0.1
- STATSD_PORT=8125
+Configuration is in the :attr:`statsd` section of ``zuul.conf``.
Metrics
~~~~~~~
diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample
index ba7aace..76494ad 100644
--- a/etc/zuul.conf-sample
+++ b/etc/zuul.conf-sample
@@ -5,6 +5,9 @@
;ssl_cert=/path/to/client.pem
;ssl_key=/path/to/client.key
+[statsd]
+server=127.0.0.1
+
[zookeeper]
hosts=127.0.0.1:2181
diff --git a/tests/base.py b/tests/base.py
index dacb1ef..b029f0b 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -20,7 +20,6 @@
import datetime
import gc
import hashlib
-import importlib
from io import StringIO
import json
import logging
@@ -48,7 +47,6 @@
import kazoo.client
import kazoo.exceptions
import pymysql
-import statsd
import testtools
import testtools.content
import testtools.content_type
@@ -2047,14 +2045,9 @@
self.config.set('executor', 'state_dir', self.executor_state_root)
self.statsd = FakeStatsd()
- # note, use 127.0.0.1 rather than localhost to avoid getting ipv6
- # see: https://github.com/jsocol/pystatsd/issues/61
- os.environ['STATSD_HOST'] = '127.0.0.1'
- os.environ['STATSD_PORT'] = str(self.statsd.port)
+ if self.config.has_section('statsd'):
+ self.config.set('statsd', 'port', str(self.statsd.port))
self.statsd.start()
- # the statsd client object is configured in the statsd module import
- importlib.reload(statsd)
- importlib.reload(zuul.scheduler)
self.gearman_server = FakeGearmanServer(self.use_ssl)
diff --git a/tests/fixtures/zuul.conf b/tests/fixtures/zuul.conf
index 7bc8c59..e6f997c 100644
--- a/tests/fixtures/zuul.conf
+++ b/tests/fixtures/zuul.conf
@@ -1,6 +1,11 @@
[gearman]
server=127.0.0.1
+[statsd]
+# note, use 127.0.0.1 rather than localhost to avoid getting ipv6
+# see: https://github.com/jsocol/pystatsd/issues/61
+server=127.0.0.1
+
[scheduler]
tenant_config=main.yaml
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 3203960..c1f0a0e 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -2152,8 +2152,7 @@
def test_statsd(self):
"Test each of the statsd methods used in the scheduler"
- import extras
- statsd = extras.try_import('statsd.statsd')
+ statsd = self.sched.statsd
statsd.incr('test-incr')
statsd.timing('test-timing', 3)
statsd.gauge('test-gauge', 12)
diff --git a/tox.ini b/tox.ini
index cc5ea58..7e84677 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,10 +5,7 @@
[testenv]
basepython = python3
-# Set STATSD env variables so that statsd code paths are tested.
-setenv = STATSD_HOST=127.0.0.1
- STATSD_PORT=8125
- VIRTUAL_ENV={envdir}
+setenv = VIRTUAL_ENV={envdir}
OS_TEST_TIMEOUT=120
passenv = ZUUL_TEST_ROOT OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_LOG_CAPTURE OS_LOG_DEFAULTS
usedevelop = True
diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py
index bba1922..d920d8e 100755
--- a/zuul/cmd/scheduler.py
+++ b/zuul/cmd/scheduler.py
@@ -29,6 +29,7 @@
import zuul.cmd
from zuul.lib.config import get_default
+from zuul.lib.statsd import get_statsd_config
# No zuul imports here because they pull in paramiko which must not be
# imported until after the daemonization.
@@ -97,8 +98,14 @@
os.close(pipe_write)
self.setup_logging('gearman_server', 'log_config')
import zuul.lib.gearserver
- statsd_host = os.environ.get('STATSD_HOST')
- statsd_port = int(os.environ.get('STATSD_PORT', 8125))
+
+ (statsd_host, statsd_port, statsd_prefix) = get_statsd_config(
+ self.config)
+ if statsd_prefix:
+ statsd_prefix += '.zuul.geard'
+ else:
+ statsd_prefix = 'zuul.geard'
+
host = get_default(self.config, 'gearman_server', 'listen_address')
port = int(get_default(self.config, 'gearman_server', 'port',
4730))
@@ -112,7 +119,7 @@
host=host,
statsd_host=statsd_host,
statsd_port=statsd_port,
- statsd_prefix='zuul.geard')
+ statsd_prefix=statsd_prefix)
# Keep running until the parent dies:
pipe_read = os.fdopen(pipe_read)
diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py
index 4fb49e3..b44fa46 100644
--- a/zuul/connection/__init__.py
+++ b/zuul/connection/__init__.py
@@ -14,8 +14,6 @@
import abc
-import extras
-
class BaseConnection(object, metaclass=abc.ABCMeta):
"""Base class for connections.
@@ -42,7 +40,6 @@
self.driver = driver
self.connection_name = connection_name
self.connection_config = connection_config
- self.statsd = extras.try_import('statsd.statsd')
def logEvent(self, event):
self.log.debug(
@@ -50,11 +47,11 @@
connection=self.connection_name,
event=event))
try:
- if self.statsd:
- self.statsd.incr(
+ if self.sched.statsd:
+ self.sched.statsd.incr(
'zuul.event.{driver}.{event}'.format(
driver=self.driver.name, event=event.type))
- self.statsd.incr(
+ self.sched.statsd.incr(
'zuul.event.{driver}.{connection}.{event}'.format(
driver=self.driver.name,
connection=self.connection_name,
diff --git a/zuul/lib/statsd.py b/zuul/lib/statsd.py
new file mode 100644
index 0000000..6c74f32
--- /dev/null
+++ b/zuul/lib/statsd.py
@@ -0,0 +1,33 @@
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import extras
+from zuul.lib.config import get_default
+
+
+def get_statsd_config(config):
+ statsd_host = get_default(config, 'statsd', 'server')
+ statsd_port = int(get_default(config, 'statsd', 'port', 8125))
+ statsd_prefix = get_default(config, 'statsd', 'prefix')
+ return (statsd_host, statsd_port, statsd_prefix)
+
+
+def get_statsd(config):
+ statsd = extras.try_import('statsd')
+ if statsd is None:
+ return None
+ (statsd_host, statsd_port, statsd_prefix) = get_statsd_config(config)
+ if statsd_host is None:
+ return None
+ return statsd.StatsClient(statsd_host, statsd_port, statsd_prefix)
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index bca62dc..e5924f8 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -15,7 +15,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-import extras
import json
import logging
import os
@@ -31,6 +30,7 @@
from zuul import exceptions
from zuul import version as zuul_version
from zuul.lib.config import get_default
+from zuul.lib.statsd import get_statsd
class ManagementEvent(object):
@@ -211,7 +211,7 @@
self.executor = None
self.merger = None
self.connections = None
- self.statsd = extras.try_import('statsd.statsd')
+ self.statsd = get_statsd(config)
# TODO(jeblair): fix this
# Despite triggers being part of the pipeline, there is one trigger set
# per scheduler. The pipeline handles the trigger filters but since