Reorganize connections into drivers
This change, while substantial, is mostly organizational.
Currently, connections, sources, triggers, and reporters are
discrete concepts, and yet are related by virtue of the fact that
the ConnectionRegistry is used to instantiate each of them. The
method used to instantiate them is called "_getDriver", in
recognition that behind each "trigger", etc., which appears in
the config file, there is a class in the zuul.trigger hierarchy
implementing the driver for that trigger. Connections also
specify a "driver" in the config file.
In this change, we redefine a "driver" as a single class that
organizes related connections, sources, triggers and reporters.
The connection, source, trigger, and reporter interfaces still
exist. A driver class is responsible for indicating which of
those interfaces it supports and instantiating them when asked to
do so.
Zuul instantiates a single instance of each driver class it knows
about (currently hardcoded, but in the future, we will be able to
easily ask entrypoints for these). That instance will be
retained for the life of the Zuul server process.
When Zuul is (re-)configured, it asks the driver instances to
create new connection, source, trigger, reporter instances as
necessary. For instance, a user may specify a connection that
uses the "gerrit" driver, and the ConnectionRegistry would call
getConnection() on the Gerrit driver instance.
This is done for two reasons: first, it allows us to organize all
of the code related to interfacing with an external system
together. All of the existing connection, source, trigger, and
reporter classes are moved as follows:
zuul.connection.FOO -> zuul.driver.FOO.FOOconnection
zuul.source.FOO -> zuul.driver.FOO.FOOsource
zuul.trigger.FOO -> zuul.driver.FOO.FOOtrigger
zuul.reporter.FOO -> zuul.driver.FOO.FOOreporter
For instance, all of the code related to interfacing with Gerrit
is now is zuul.driver.gerrit.
Second, the addition of a single, long-lived object associated
with each of these systems allows us to better support some types
of interfaces. For instance, the Zuul trigger maintains a list
of events it is required to emit -- this list relates to a tenant
as a whole rather than individual pipelines or triggers. The
timer trigger maintains a single scheduler instance for all
tenants, but must be able to add or remove cron jobs based on an
individual tenant being reconfigured. The global driver instance
for each of these can be used to accomplish this.
As a result of using the driver interface to create new
connection, source, trigger and reporter instances, the
connection setup in ConnectionRegistry is much simpler, and can
easily be extended with entrypoints in the future.
The existing tests of connections, sources, triggers, and
reporters which only tested that they could be instantiated and
have names have been removed, as there are functional tests which
cover them.
Change-Id: Ib2f7297d81f7a003de48f799dc1b09e82d4894bc
diff --git a/tests/base.py b/tests/base.py
index 9e3c07b..1b65416 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -45,8 +45,8 @@
import testtools
from git.exc import NoSuchPathError
-import zuul.connection.gerrit
-import zuul.connection.smtp
+import zuul.driver.gerrit.gerritsource as gerritsource
+import zuul.driver.gerrit.gerritconnection as gerritconnection
import zuul.scheduler
import zuul.webapp
import zuul.rpclistener
@@ -58,12 +58,6 @@
import zuul.merger.merger
import zuul.merger.server
import zuul.nodepool
-import zuul.reporter.gerrit
-import zuul.reporter.smtp
-import zuul.source.gerrit
-import zuul.trigger.gerrit
-import zuul.trigger.timer
-import zuul.trigger.zuultrigger
import zuul.zk
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
@@ -388,7 +382,7 @@
self.reported += 1
-class FakeGerritConnection(zuul.connection.gerrit.GerritConnection):
+class FakeGerritConnection(gerritconnection.GerritConnection):
"""A Fake Gerrit connection for use in tests.
This subclasses
@@ -398,9 +392,9 @@
log = logging.getLogger("zuul.test.FakeGerritConnection")
- def __init__(self, connection_name, connection_config,
+ def __init__(self, driver, connection_name, connection_config,
changes_db=None, upstream_root=None):
- super(FakeGerritConnection, self).__init__(connection_name,
+ super(FakeGerritConnection, self).__init__(driver, connection_name,
connection_config)
self.event_queue = Queue.Queue()
@@ -1225,14 +1219,15 @@
self.config.set('gearman', 'port', str(self.gearman_server.port))
- zuul.source.gerrit.GerritSource.replication_timeout = 1.5
- zuul.source.gerrit.GerritSource.replication_retry_interval = 0.5
- zuul.connection.gerrit.GerritEventConnector.delay = 0.0
+ gerritsource.GerritSource.replication_timeout = 1.5
+ gerritsource.GerritSource.replication_retry_interval = 0.5
+ gerritconnection.GerritEventConnector.delay = 0.0
self.sched = zuul.scheduler.Scheduler(self.config)
self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
FakeSwiftClientConnection))
+
self.swift = zuul.lib.swift.Swift(self.config)
self.event_queues = [
@@ -1294,7 +1289,25 @@
self.assertFinalState()
def configure_connections(self):
- # Register connections from the config
+ # Set up gerrit related fakes
+ # Set a changes database so multiple FakeGerrit's can report back to
+ # a virtual canonical database given by the configured hostname
+ self.gerrit_changes_dbs = {}
+
+ def getGerritConnection(driver, name, config):
+ db = self.gerrit_changes_dbs.setdefault(config['server'], {})
+ con = FakeGerritConnection(driver, name, config,
+ changes_db=db,
+ upstream_root=self.upstream_root)
+ self.event_queues.append(con.event_queue)
+ setattr(self, 'fake_' + name, con)
+ return con
+
+ self.useFixture(fixtures.MonkeyPatch(
+ 'zuul.driver.gerrit.GerritDriver.getConnection',
+ getGerritConnection))
+
+ # Set up smtp related fakes
self.smtp_messages = []
def FakeSMTPFactory(*args, **kw):
@@ -1303,60 +1316,9 @@
self.useFixture(fixtures.MonkeyPatch('smtplib.SMTP', FakeSMTPFactory))
- # Set a changes database so multiple FakeGerrit's can report back to
- # a virtual canonical database given by the configured hostname
- self.gerrit_changes_dbs = {}
+ # Register connections from the config using fakes
self.connections = zuul.lib.connections.ConnectionRegistry()
-
- for section_name in self.config.sections():
- con_match = re.match(r'^connection ([\'\"]?)(.*)(\1)$',
- section_name, re.I)
- if not con_match:
- continue
- con_name = con_match.group(2)
- con_config = dict(self.config.items(section_name))
-
- if 'driver' not in con_config:
- raise Exception("No driver specified for connection %s."
- % con_name)
-
- con_driver = con_config['driver']
-
- # TODO(jhesketh): load the required class automatically
- if con_driver == 'gerrit':
- if con_config['server'] not in self.gerrit_changes_dbs.keys():
- self.gerrit_changes_dbs[con_config['server']] = {}
- self.connections.connections[con_name] = FakeGerritConnection(
- con_name, con_config,
- changes_db=self.gerrit_changes_dbs[con_config['server']],
- upstream_root=self.upstream_root
- )
- self.event_queues.append(
- self.connections.connections[con_name].event_queue)
- setattr(self, 'fake_' + con_name,
- self.connections.connections[con_name])
- elif con_driver == 'smtp':
- self.connections.connections[con_name] = \
- zuul.connection.smtp.SMTPConnection(con_name, con_config)
- else:
- raise Exception("Unknown driver, %s, for connection %s"
- % (con_config['driver'], con_name))
-
- # If the [gerrit] or [smtp] sections still exist, load them in as a
- # connection named 'gerrit' or 'smtp' respectfully
-
- if 'gerrit' in self.config.sections():
- self.gerrit_changes_dbs['gerrit'] = {}
- self.event_queues.append(
- self.connections.connections[con_name].event_queue)
- self.connections.connections['gerrit'] = FakeGerritConnection(
- '_legacy_gerrit', dict(self.config.items('gerrit')),
- changes_db=self.gerrit_changes_dbs['gerrit'])
-
- if 'smtp' in self.config.sections():
- self.connections.connections['smtp'] = \
- zuul.connection.smtp.SMTPConnection(
- '_legacy_smtp', dict(self.config.items('smtp')))
+ self.connections.configure(self.config)
def setup_config(self):
# This creates the per-test configuration object. It can be