Merge "Add memory awareness to system load governor"
diff --git a/doc/source/admin/monitoring.rst b/doc/source/admin/monitoring.rst
index e6e6139..6dbdb31 100644
--- a/doc/source/admin/monitoring.rst
+++ b/doc/source/admin/monitoring.rst
@@ -26,7 +26,7 @@
These metrics are emitted by the Zuul :ref:`scheduler`:
-.. stat:: zuul.event.<driver>.event.<type>
+.. stat:: zuul.event.<driver>.<type>
:type: counter
Zuul will report counters for each type of event it receives from
diff --git a/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml b/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml
new file mode 100644
index 0000000..f679dce
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/common-config/playbooks/base.yaml
@@ -0,0 +1,2 @@
+- hosts: all
+ tasks: []
diff --git a/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml b/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml
new file mode 100644
index 0000000..3000df5
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/common-config/zuul.yaml
@@ -0,0 +1,27 @@
+- pipeline:
+ name: check
+ manager: independent
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ Verified: 1
+ failure:
+ gerrit:
+ Verified: -1
+
+- job:
+ name: base
+ run: playbooks/base.yaml
+ parent: null
+
+- job:
+ name: restricted-job
+ allowed-projects:
+ - org/project1
+
+- project:
+ name: common-config
+ check:
+ jobs: []
diff --git a/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..d3c98f3
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project1/zuul.yaml
@@ -0,0 +1,10 @@
+- job:
+ name: test-project1
+ parent: restricted-job
+
+- project:
+ name: org/project1
+ check:
+ jobs:
+ - test-project1
+ - restricted-job
diff --git a/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..bf0f07a
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project2/zuul.yaml
@@ -0,0 +1,11 @@
+- job:
+ name: test-project2
+ parent: restricted-job
+ allowed-projects:
+ - org/project2
+
+- project:
+ name: org/project2
+ check:
+ jobs:
+ - test-project2
diff --git a/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml b/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml
new file mode 100644
index 0000000..43b59a6
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/git/org_project3/zuul.yaml
@@ -0,0 +1,5 @@
+- project:
+ name: org/project3
+ check:
+ jobs:
+ - restricted-job
diff --git a/tests/fixtures/config/allowed-projects/main.yaml b/tests/fixtures/config/allowed-projects/main.yaml
new file mode 100644
index 0000000..49ed838
--- /dev/null
+++ b/tests/fixtures/config/allowed-projects/main.yaml
@@ -0,0 +1,10 @@
+- tenant:
+ name: tenant-one
+ source:
+ gerrit:
+ config-projects:
+ - common-config
+ untrusted-projects:
+ - org/project1
+ - org/project2
+ - org/project3
diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py
index 784fcb3..5c586ca 100644
--- a/tests/unit/test_model.py
+++ b/tests/unit/test_model.py
@@ -320,50 +320,6 @@
"to shadow job base in base_project"):
layout.addJob(base2)
- def test_job_allowed_projects(self):
- job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
- '_source_context': self.context,
- '_start_mark': self.start_mark,
- 'name': 'job',
- 'parent': None,
- 'allowed-projects': ['project'],
- })
- self.layout.addJob(job)
-
- project2 = model.Project('project2', self.source)
- tpc2 = model.TenantProjectConfig(project2)
- self.tenant.addUntrustedProject(tpc2)
- context2 = model.SourceContext(project2, 'master',
- 'test', True)
-
- project_template_parser = configloader.ProjectTemplateParser(
- self.tenant, self.layout)
- project_parser = configloader.ProjectParser(
- self.tenant, self.layout, project_template_parser)
- project2_config = project_parser.fromYaml(
- [{
- '_source_context': context2,
- '_start_mark': self.start_mark,
- 'name': 'project2',
- 'gate': {
- 'jobs': [
- 'job'
- ]
- }
- }]
- )
- self.layout.addProjectConfig(project2_config)
-
- change = model.Change(project2)
- # Test master
- change.branch = 'master'
- item = self.queue.enqueueChange(change)
- item.layout = self.layout
- with testtools.ExpectedException(
- Exception,
- "Project project2 is not allowed to run job job"):
- item.freezeJobGraph()
-
def test_job_pipeline_allow_untrusted_secrets(self):
self.pipeline.post_review = False
job = configloader.JobParser.fromYaml(self.tenant, self.layout, {
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index 4cb4a41..44eda82 100755
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -533,6 +533,36 @@
], ordered=False)
+class TestAllowedProjects(ZuulTestCase):
+ tenant_config_file = 'config/allowed-projects/main.yaml'
+
+ def test_allowed_projects(self):
+ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
+ self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(A.reported, 1)
+ self.assertIn('Build succeeded', A.messages[0])
+
+ B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
+ self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(B.reported, 1)
+ self.assertIn('Project org/project2 is not allowed '
+ 'to run job test-project2', B.messages[0])
+
+ C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
+ self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
+ self.assertEqual(C.reported, 1)
+ self.assertIn('Project org/project3 is not allowed '
+ 'to run job restricted-job', C.messages[0])
+
+ self.assertHistory([
+ dict(name='test-project1', result='SUCCESS', changes='1,1'),
+ dict(name='restricted-job', result='SUCCESS', changes='1,1'),
+ ], ordered=False)
+
+
class TestCentralJobs(ZuulTestCase):
tenant_config_file = 'config/central-jobs/main.yaml'
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index b766c6f..02cbfdb 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -343,7 +343,7 @@
if login:
# TODO(tobiash): it might be better to plumb in the installation id
project = body.get('repository', {}).get('full_name')
- return self.connection.getUser(login, project=project)
+ return self.connection.getUser(login, project)
def run(self):
while True:
@@ -360,10 +360,11 @@
class GithubUser(collections.Mapping):
log = logging.getLogger('zuul.GithubUser')
- def __init__(self, github, username):
- self._github = github
+ def __init__(self, username, connection, project):
+ self._connection = connection
self._username = username
self._data = None
+ self._project = project
def __getitem__(self, key):
self._init_data()
@@ -379,9 +380,10 @@
def _init_data(self):
if self._data is None:
- user = self._github.user(self._username)
+ github = self._connection.getGithubClient(self._project)
+ user = github.user(self._username)
self.log.debug("Initialized data for user %s", self._username)
- log_rate_limit(self.log, self._github)
+ log_rate_limit(self.log, github)
self._data = {
'username': user.login,
'name': user.name,
@@ -722,10 +724,10 @@
# installation -- change queues aren't likely to span more
# than one installation.
for project in projects:
- installation_id = self.installation_map.get(project)
+ installation_id = self.installation_map.get(project.name)
if installation_id not in installation_ids:
installation_ids.add(installation_id)
- installation_projects.add(project)
+ installation_projects.add(project.name)
else:
# We aren't in the context of a change queue and we just
# need to query all installations. This currently only
@@ -972,8 +974,8 @@
log_rate_limit(self.log, github)
return reviews
- def getUser(self, login, project=None):
- return GithubUser(self.getGithubClient(project), login)
+ def getUser(self, login, project):
+ return GithubUser(login, self, project)
def getUserUri(self, login):
return 'https://%s/%s' % (self.server, login)
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 1c72275..d2c04ac 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -1262,6 +1262,9 @@
config.write('internal_poll_interval = 0.01\n')
config.write('[ssh_connection]\n')
+ # NOTE(pabelanger): Try up to 3 times to run a task on a host, this
+ # helps to mitigate UNREACHABLE host errors with SSH.
+ config.write('retries = 3\n')
# NB: when setting pipelining = True, keep_remote_files
# must be False (the default). Otherwise it apparently
# will override the pipelining option and effectively
diff --git a/zuul/model.py b/zuul/model.py
index 0685f82..9cfbd0a 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1060,7 +1060,8 @@
"from other projects."
% (repr(self), this_origin))
if k not in set(['pre_run', 'run', 'post_run', 'roles',
- 'variables', 'required_projects']):
+ 'variables', 'required_projects',
+ 'allowed_projects']):
# TODO(jeblair): determine if deepcopy is required
setattr(self, k, copy.deepcopy(other._get(k)))
@@ -1097,6 +1098,12 @@
self.updateVariables(other.variables)
if other._get('required_projects') is not None:
self.updateProjects(other.required_projects)
+ if (other._get('allowed_projects') is not None and
+ self._get('allowed_projects') is not None):
+ self.allowed_projects = self.allowed_projects.intersection(
+ other.allowed_projects)
+ elif other._get('allowed_projects') is not None:
+ self.allowed_projects = copy.deepcopy(other.allowed_projects)
for k in self.context_attributes:
if (other._get(k) is not None and
@@ -2828,7 +2835,7 @@
item.debug("No matching pipeline variants for {jobname}".
format(jobname=jobname), indent=2)
continue
- if (frozen_job.allowed_projects and
+ if (frozen_job.allowed_projects is not None and
change.project.name not in frozen_job.allowed_projects):
raise Exception("Project %s is not allowed to run job %s" %
(change.project.name, frozen_job.name))