Merge "Support the fragment form of Gerrit URLs"
diff --git a/.zuul.yaml b/.zuul.yaml
index c820c8e..d73be8f 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -29,10 +29,9 @@
- playbooks/zuul-stream/.*
- project:
- name: openstack-infra/zuul
check:
jobs:
- - build-openstack-sphinx-docs:
+ - build-sphinx-docs:
irrelevant-files:
- zuul/cmd/migrate.py
- playbooks/zuul-migrate/.*
@@ -46,7 +45,7 @@
- zuul-stream-functional
gate:
jobs:
- - build-openstack-sphinx-docs:
+ - build-sphinx-docs:
irrelevant-files:
- zuul/cmd/migrate.py
- playbooks/zuul-migrate/.*
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index d6b0984..2e18b51 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -575,6 +575,16 @@
The executor will observe system load and determine whether
to accept more jobs every 30 seconds.
+ .. attr:: min_avail_mem
+ :default: 5.0
+
+ This is the minimum percentage of system RAM available. The
+ executor will stop accepting more than 1 job at a time until
+ more memory is available. The available memory percentage is
+ calculated from the total available memory divided by the
+ total real memory multiplied by 100. Buffers and cache are
+ considered available in the calculation.
+
.. attr:: hostname
:default: hostname of the server
diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst
index e6e6139..0fdb3b2 100644
--- a/doc/source/admin/monitoring.rst
+++ b/doc/source/admin/monitoring.rst
@@ -26,7 +26,7 @@
These metrics are emitted by the Zuul :ref:`scheduler`:
-.. stat:: zuul.event.<driver>.event.<type>
+.. stat:: zuul.event.<driver>.<type>
:type: counter
Zuul will report counters for each type of event it receives from
@@ -146,6 +146,12 @@
The one-minute load average of this executor, multiplied by 100.
+ .. stat:: pct_available_ram
+ :type: gauge
+
+ The available RAM (including buffers and cache) on this
+ executor, as a percentage multiplied by 100.
+
.. stat:: zuul.nodepool
Holds metrics related to Zuul requests from Nodepool.
diff --git a/requirements.txt b/requirements.txt
index 39a2b02..f24f195 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@
PyYAML>=3.1.0
Paste
WebOb>=1.2.3
-paramiko>=1.8.0,<2.0.0
+paramiko>=2.0.1
GitPython>=2.1.8
python-daemon>=2.0.4,<2.1.0
extras
@@ -27,3 +27,4 @@
iso8601
aiohttp
uvloop;python_version>='3.5'
+psutil
diff --git a/test-requirements.txt b/test-requirements.txt
index b444297..70f8e78 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,12 +1,9 @@
-pep8
-pyflakes
flake8
coverage>=3.6
sphinx>=1.5.1,<1.6
sphinxcontrib-blockdiag>=1.1.0
fixtures>=0.3.14
-python-keystoneclient>=0.4.2
python-subunit
testrepository>=0.0.17
testtools>=0.9.32
diff --git a/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml b/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml b/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml
new file mode 100644
index 0000000..3000df5
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml
@@ -0,0 +1,27 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- job:
+ name: base
+ run: playbooks/base.yaml
+ parent: null
+
+- job:
+ name: restricted-job
+ allowed-projects:
+ - org/project1
+
+- project:
+ name: common-config
+ check:
+ jobs: []
diff --git a/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..d3c98f3
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml
@@ -0,0 +1,10 @@
+- job:
+ name: test-project1
+ parent: restricted-job
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - test-project1
+ - restricted-job
diff --git a/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..bf0f07a
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+ name: test-project2
+ parent: restricted-job
+ allowed-projects:
+ - org/project2
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - test-project2
diff --git a/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml
new file mode 100644
index 0000000..43b59a6
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml
@@ -0,0 +1,5 @@
+- project:
+ name: org/project3
+ check:
+ jobs:
+ - restricted-job
diff --git a/tests/fixtures/config/allowed-projects/main.yaml b/tests/fixtures/config/allowed-projects/main.yaml
new file mode 100644
index 0000000..49ed838
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/main.yaml
@@ -0,0 +1,10 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
+ - org/project3
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 784fcb3..5c586ca 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -320,50 +320,6 @@
"to shadow job base in base_project"):
layout.addJob(base2)
- def test_job_allowed_projects(self):
- job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
- '_source_context': self.context,
- '_start_mark': self.start_mark,
- 'name': 'job',
- 'parent': None,
- 'allowed-projects': ['project'],
- })
- self.layout.addJob(job)
-
- project2 = model.Project('project2', self.source)
- tpc2 = model.TenantProjectConfig(project2)
- self.tenant.addUntrustedProject(tpc2)
- context2 = model.SourceContext(project2, 'master',
- 'test', True)
-
- project_template_parser = configloader.ProjectTemplateParser(
- self.tenant, self.layout)
- project_parser = configloader.ProjectParser(
- self.tenant, self.layout, project_template_parser)
- project2_config = project_parser.fromYaml(
- [{
- '_source_context': context2,
- '_start_mark': self.start_mark,
- 'name': 'project2',
- 'gate': {
- 'jobs': [
- 'job'
- ]
- }
- }]
- )
- self.layout.addProjectConfig(project2_config)
-
- change = model.Change(project2)
- # Test master
- change.branch = 'master'
- item = self.queue.enqueueChange(change)
- item.layout = self.layout
- with testtools.ExpectedException(
- Exception,
- "Project project2 is not allowed to run job job"):
- item.freezeJobGraph()
-
def test_job_pipeline_allow_untrusted_secrets(self):
self.pipeline.post_review = False
job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 4cb4a41..44eda82 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -533,6 +533,36 @@
], ordered=False)
+class TestAllowedProjects(ZuulTestCase):
+ tenant_config_file = 'config/allowed-projects/main.yaml'
+
+ def test_allowed_projects(self):
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1)
+ self.assertIn('Build succeeded', A.messages[0])
+
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 1)
+ self.assertIn('Project org/project2 is not allowed '
+ 'to run job test-project2', B.messages[0])
+
+ C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(C.reported, 1)
+ self.assertIn('Project org/project3 is not allowed '
+ 'to run job restricted-job', C.messages[0])
+
+ self.assertHistory([
+ dict(name='test-project1', result='SUCCESS', changes='1,1'),
+ dict(name='restricted-job', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+
class TestCentralJobs(ZuulTestCase):
tenant_config_file = 'config/central-jobs/main.yaml'
diff --git a/tox.ini b/tox.ini
index 5efc4c0..73915ad 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,7 +36,8 @@
python setup.py test --coverage
[testenv:docs]
-commands = python setup.py build_sphinx
+commands =
+ sphinx-build -W -d doc/build/doctrees -b html doc/source/ doc/build/html
[testenv:venv]
commands = {posargs}
@@ -49,6 +50,6 @@
[flake8]
# These are ignored intentionally in openstack-infra projects;
# please don't submit patches that solely correct them or enable them.
-ignore = E125,E129,E402,E741,H,W503
+ignore = E124,E125,E129,E402,E741,H,W503
show-source = True
exclude = .venv,.tox,dist,doc,build,*.egg
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index b766c6f..02cbfdb 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -343,7 +343,7 @@
if login:
# TODO(tobiash): it might be better to plumb in the installation id
project = body.get('repository', {}).get('full_name')
- return self.connection.getUser(login, project=project)
+ return self.connection.getUser(login, project)
def run(self):
while True:
@@ -360,10 +360,11 @@
class GithubUser(collections.Mapping):
log = logging.getLogger('zuul.GithubUser')
- def __init__(self, github, username):
- self._github = github
+ def __init__(self, username, connection, project):
+ self._connection = connection
self._username = username
self._data = None
+ self._project = project
def __getitem__(self, key):
self._init_data()
@@ -379,9 +380,10 @@
def _init_data(self):
if self._data is None:
- user = self._github.user(self._username)
+ github = self._connection.getGithubClient(self._project)
+ user = github.user(self._username)
self.log.debug("Initialized data for user %s", self._username)
- log_rate_limit(self.log, self._github)
+ log_rate_limit(self.log, github)
self._data = {
'username': user.login,
'name': user.name,
@@ -722,10 +724,10 @@
# installation -- change queues aren't likely to span more
# than one installation.
for project in projects:
- installation_id = self.installation_map.get(project)
+ installation_id = self.installation_map.get(project.name)
if installation_id not in installation_ids:
installation_ids.add(installation_id)
- installation_projects.add(project)
+ installation_projects.add(project.name)
else:
# We aren't in the context of a change queue and we just
# need to query all installations. This currently only
@@ -972,8 +974,8 @@
log_rate_limit(self.log, github)
return reviews
- def getUser(self, login, project=None):
- return GithubUser(self.getGithubClient(project), login)
+ def getUser(self, login, project):
+ return GithubUser(login, self, project)
def getUserUri(self, login):
return 'https://%s/%s' % (self.server, login)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 03a3a12..52e54bb 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -18,6 +18,7 @@
import logging
import multiprocessing
import os
+import psutil
import shutil
import signal
import shlex
@@ -1261,6 +1262,9 @@
config.write('internal_poll_interval = 0.01\n')
config.write('[ssh_connection]\n')
+ # NOTE(pabelanger): Try up to 3 times to run a task on a host, this
+ # helps to mitigate UNREACHABLE host errors with SSH.
+ config.write('retries = 3\n')
# NB: when setting pipelining = True, keep_remote_files
# must be False (the default). Otherwise it apparently
# will override the pipelining option and effectively
@@ -1628,6 +1632,8 @@
load_multiplier = float(get_default(self.config, 'executor',
'load_multiplier', '2.5'))
self.max_load_avg = multiprocessing.cpu_count() * load_multiplier
+ self.min_avail_mem = float(get_default(self.config, 'executor',
+ 'min_avail_mem', '5.0'))
self.accepting_work = False
self.execution_wrapper = connections.drivers[execution_wrapper_name]
@@ -1794,6 +1800,7 @@
if self.statsd:
base_key = 'zuul.executor.%s' % self.hostname
self.statsd.gauge(base_key + '.load_average', 0)
+ self.statsd.gauge(base_key + '.pct_available_ram', 0)
self.statsd.gauge(base_key + '.running_builds', 0)
self.log.debug("Stopped")
@@ -1949,6 +1956,7 @@
''' Apply some heuristics to decide whether or not we should
be askign for more jobs '''
load_avg = os.getloadavg()[0]
+ avail_mem_pct = 100.0 - psutil.virtual_memory().percent
if self.accepting_work:
# Don't unregister if we don't have any active jobs.
if load_avg > self.max_load_avg and self.job_workers:
@@ -1956,15 +1964,26 @@
"Unregistering due to high system load {} > {}".format(
load_avg, self.max_load_avg))
self.unregister_work()
- elif load_avg <= self.max_load_avg:
+ elif avail_mem_pct < self.min_avail_mem:
+ self.log.info(
+ "Unregistering due to low memory {:3.1f}% < {}".format(
+ avail_mem_pct, self.min_avail_mem))
+ self.unregister_work()
+ elif (load_avg <= self.max_load_avg and
+ avail_mem_pct >= self.min_avail_mem):
self.log.info(
- "Re-registering as load is within limits {} <= {}".format(
- load_avg, self.max_load_avg))
+ "Re-registering as job is within limits "
+ "{} <= {} {:3.1f}% <= {}".format(load_avg,
+ self.max_load_avg,
+ avail_mem_pct,
+ self.min_avail_mem))
self.register_work()
if self.statsd:
base_key = 'zuul.executor.%s' % self.hostname
self.statsd.gauge(base_key + '.load_average',
int(load_avg * 100))
+ self.statsd.gauge(base_key + '.pct_available_ram',
+ int(avail_mem_pct * 100))
self.statsd.gauge(base_key + '.running_builds',
len(self.job_workers))
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index c221478..07f3e69 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -53,11 +53,6 @@
raise
-class ZuulReference(git.Reference):
- _common_path_default = "refs/zuul"
- _points_to_commits_only = True
-
-
class Repo(object):
def __init__(self, remote, local, email, username, speed_limit, speed_time,
sshkey=None, cache_path=None, logger=None, git_timeout=300,
@@ -307,12 +302,6 @@
repo = self.createRepoObject()
self._git_fetch(repo, repository, ref)
- def createZuulRef(self, ref, commit='HEAD'):
- repo = self.createRepoObject()
- self.log.debug("CreateZuulRef %s at %s on %s" % (ref, commit, repo))
- ref = ZuulReference.create(repo, ref, commit)
- return ref.commit
-
def push(self, local, remote):
repo = self.createRepoObject()
self.log.debug("Pushing %s:%s to %s" % (local, remote,
@@ -543,20 +532,6 @@
return None
# Store this commit as the most recent for this project-branch
recent[key] = commit
- # Set the Zuul ref for this item to point to the most recent
- # commits of each project-branch
- for key, mrc in recent.items():
- connection, project, branch = key
- zuul_ref = None
- try:
- repo = self.getRepo(connection, project)
- zuul_ref = branch + '/' + item['buildset_uuid']
- if not repo.getCommitFromRef(zuul_ref):
- repo.createZuulRef(zuul_ref, mrc)
- except Exception:
- self.log.exception("Unable to set zuul ref %s for "
- "item %s" % (zuul_ref, item))
- return None
return commit
def mergeChanges(self, items, files=None, dirs=None, repo_state=None):
diff --git a/zuul/model.py b/zuul/model.py
index 0685f82..38f2d6b 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1060,7 +1060,8 @@
"from other projects."
% (repr(self), this_origin))
if k not in set(['pre_run', 'run', 'post_run', 'roles',
- 'variables', 'required_projects']):
+ 'variables', 'required_projects',
+ 'allowed_projects']):
# TODO(jeblair): determine if deepcopy is required
setattr(self, k, copy.deepcopy(other._get(k)))
@@ -1097,6 +1098,12 @@
self.updateVariables(other.variables)
if other._get('required_projects') is not None:
self.updateProjects(other.required_projects)
+ if (other._get('allowed_projects') is not None and
+ self._get('allowed_projects') is not None):
+ self.allowed_projects = self.allowed_projects.intersection(
+ other.allowed_projects)
+ elif other._get('allowed_projects') is not None:
+ self.allowed_projects = copy.deepcopy(other.allowed_projects)
for k in self.context_attributes:
if (other._get(k) is not None and
@@ -2542,6 +2549,7 @@
# that override some attribute of the job. These aspects all
# inherit from the reference definition.
noop = Job('noop')
+ noop.description = 'A job that will always succeed, no operation.'
noop.parent = noop.BASE_JOB_MARKER
noop.run = 'noop.yaml'
self.jobs = {'noop': [noop]}
@@ -2828,7 +2836,7 @@
item.debug("No matching pipeline variants for {jobname}".
format(jobname=jobname), indent=2)
continue
- if (frozen_job.allowed_projects and
+ if (frozen_job.allowed_projects is not None and
change.project.name not in frozen_job.allowed_projects):
raise Exception("Project %s is not allowed to run job %s" %
(change.project.name, frozen_job.name))
diff --git a/zuul/scheduler.py b/zuul/scheduler.py
index a2e3b6e..8b7a3f1 100644
--- a/zuul/scheduler.py
+++ b/zuul/scheduler.py
@@ -1085,6 +1085,10 @@
pipelines = []
data['pipelines'] = pipelines
tenant = self.abide.tenants.get(tenant_name)
+ if not tenant:
+ self.log.warning("Tenant %s isn't loaded" % tenant_name)
+ return json.dumps(
+ {"message": "Tenant %s isn't ready" % tenant_name})
for pipeline in tenant.layout.pipelines.values():
pipelines.append(pipeline.formatStatusJSON(websocket_url))
return json.dumps(data)