Merge "Add an internals doc to the documentation" into feature/zuulv3
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 9e0d2c7..f8ae368 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -25,7 +25,11 @@
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinxcontrib.blockdiag', 'sphinxcontrib.programoutput' ]
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinxcontrib.blockdiag',
+ 'sphinxcontrib.programoutput'
+]
#extensions = ['sphinx.ext.intersphinx']
#intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 3c793da..8c12138 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -24,6 +24,7 @@
launchers
statsd
client
+ internals
Indices and tables
==================
diff --git a/doc/source/internals.rst b/doc/source/internals.rst
new file mode 100644
index 0000000..762eb6b
--- /dev/null
+++ b/doc/source/internals.rst
@@ -0,0 +1,87 @@
+Zuul Internals
+==============
+
+While most people should not need to understand the details of Zuul's internal
+data model, understanding the data model is essential for people writing
+code for Zuul, and might be interesting to advanced users. The model is
+defined in `zuul/model.py`_.
+
+.. _zuul/model.py: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/model.py
+
+Data Model
+----------
+
+It all starts with the :py:class:`~zuul.model.Pipeline`. A Pipeline is the
+basic organizational structure that everything else hangs off.
+
+.. autoclass:: zuul.model.Pipeline
+
+Pipelines have a configured
+:py:class:`Manager <zuul.manager.BasePipelineManager>` which controlls how
+the :py:class:`Change <zuul.model.Changeish>` objects are enqueued and
+processed.
+
+There are currently two,
+:py:class:`~zuul.manager.dependent.DependentPipelineManager` and
+:py:class:`~zuul.manager.independent.IndependentPipelineManager`
+
+.. autoclass:: zuul.manager.BasePipelineManager
+.. autoclass:: zuul.manager.dependent.DependentPipelineManager
+.. autoclass:: zuul.manager.independent.IndependentPipelineManager
+
+A :py:class:`~zuul.model.Pipeline` has one or more
+:py:class:`~zuul.model.ChangeQueue` objects.
+
+.. autoclass:: zuul.model.ChangeQueue
+
+A :py:class:`~zuul.model.Job` represents the definition of what to do. A
+:py:class:`~zuul.model.Build` represents a single run of a
+:py:class:`~zuul.model.Job`. A :py:class:`~zuul.model.JobTree` is used to
+encapsulate the dependencies between one or more :py:class:`~zuul.model.Job`
+objects.
+
+.. autoclass:: zuul.model.Job
+.. autoclass:: zuul.model.JobTree
+.. autoclass:: zuul.model.Build
+
+The :py:class:`~zuul.manager.base.PipelineManager` enqueues each
+:py:class:`Change <zuul.model.Changeish>` into the
+:py:class:`~zuul.model.ChangeQueue` in a :py:class:`~zuul.model.QueueItem`.
+
+.. autoclass:: zuul.model.QueueItem
+
+As the Changes are processed, each :py:class:`~zuul.model.Build` is put into
+a :py:class:`~zuul.model.BuildSet`
+
+.. autoclass:: zuul.model.BuildSet
+
+Changes
+~~~~~~~
+
+.. autoclass:: zuul.model.Changeish
+.. autoclass:: zuul.model.Change
+.. autoclass:: zuul.model.Ref
+
+Filters
+~~~~~~~
+
+.. autoclass:: zuul.model.ChangeishFilter
+.. autoclass:: zuul.model.EventFilter
+
+
+Tenants
+~~~~~~~
+
+An abide is a collection of tenants.
+
+.. autoclass:: zuul.model.UnparsedAbideConfig
+.. autoclass:: zuul.model.UnparsedTenantConfig
+
+Other Global Objects
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: zuul.model.Project
+.. autoclass:: zuul.model.Layout
+.. autoclass:: zuul.model.RepoFiles
+.. autoclass:: zuul.model.Worker
+.. autoclass:: zuul.model.TriggerEvent
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 8fdf14c..c136343 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -13,7 +13,6 @@
import logging
from zuul import exceptions
-import zuul.configloader
from zuul.model import NullChange
@@ -41,6 +40,8 @@
class BasePipelineManager(object):
+ """Abstract Base Class for enqueing and processing Changes in a Pipeline"""
+
log = logging.getLogger("zuul.BasePipelineManager")
def __init__(self, sched, pipeline):
@@ -449,6 +450,8 @@
if build_set.unable_to_merge:
return None
# Load layout
+ # Late import to break an import loop
+ import zuul.configloader
loader = zuul.configloader.ConfigLoader()
self.log.debug("Load dynamic layout with %s" % build_set.files)
layout = loader.createDynamicLayout(item.pipeline.layout.tenant,
diff --git a/zuul/manager/dependent.py b/zuul/manager/dependent.py
index 02dc9f6..d175c67 100644
--- a/zuul/manager/dependent.py
+++ b/zuul/manager/dependent.py
@@ -17,6 +17,13 @@
class DependentPipelineManager(BasePipelineManager):
+ """PipelineManager for handling interrelated Changes.
+
+ The DependentPipelineManager puts Changes that share a Pipeline
+ into a shared :py:class:`~zuul.model.ChangeQueue`. It them processes them
+ using the Optmistic Branch Prediction logic with Nearest Non-Failing Item
+ reparenting algorithm for handling errors.
+ """
log = logging.getLogger("zuul.DependentPipelineManager")
changes_merge = True
diff --git a/zuul/manager/independent.py b/zuul/manager/independent.py
index 5690189..ca65018 100644
--- a/zuul/manager/independent.py
+++ b/zuul/manager/independent.py
@@ -17,6 +17,8 @@
class IndependentPipelineManager(BasePipelineManager):
+ """PipelineManager that puts every Change into its own ChangeQueue."""
+
log = logging.getLogger("zuul.IndependentPipelineManager")
changes_merge = False
diff --git a/zuul/model.py b/zuul/model.py
index 8220f85..2b957a5 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -70,11 +70,18 @@
class Pipeline(object):
"""A configuration that ties triggers, reporters, managers and sources.
- Source is where changes should come from. It is a named connection to
+ Source
+ Where changes should come from. It is a named connection to
an external service defined in zuul.conf
- Trigger is a description of which events should be processed
- Manager is responsible for enqueing and dequeing Changes
- Reporters communicate success and failure results somewhere
+
+ Trigger
+ A description of which events should be processed
+
+ Manager
+ Responsible for enqueing and dequeing Changes
+
+ Reporter
+ Communicates success and failure results somewhere
"""
def __init__(self, name, layout):
self.name = name
@@ -196,12 +203,13 @@
class ChangeQueue(object):
"""A ChangeQueue contains Changes to be processed related projects.
- DependentPipelines have multiple parallel ChangeQueues shared by
- different projects. For instance, there may a ChangeQueue shared by
- interrelated projects foo and bar, and a second queue for independent
- project baz.
+ A Pipeline with a DependentPipelineManager has multiple parallel
+ ChangeQueues shared by different projects. For instance, there may a
+ ChangeQueue shared by interrelated projects foo and bar, and a second queue
+ for independent project baz.
- IndependentPipelinesManager puts every Change into its own ChangeQueue
+ A Pipeline with an IndependentPipelineManager puts every Change into its
+ own ChangeQueue
The ChangeQueue Window is inspired by TCP windows and controlls how many
Changes in a given ChangeQueue will be considered active and ready to