Merge "Decode JSON body once for requests" into feature/zuulv3
diff --git a/requirements.txt b/requirements.txt
index 48ac38e..746bbcb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@
# pull from master until https://github.com/sigmavirus24/github3.py/pull/671
# is in a release
--e git://github.com/sigmavirus24/github3.py.git@develop#egg=Github3.py
+-e git+https://github.com/sigmavirus24/github3.py.git@develop#egg=Github3.py
PyYAML>=3.1.0
Paste
WebOb>=1.2.3
diff --git a/tests/base.py b/tests/base.py
index 71c48aa..2bcd1ca 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1698,6 +1698,18 @@
else:
self._log_stream = sys.stdout
+ # NOTE(jeblair): this is temporary extra debugging to try to
+ # track down a possible leak.
+ orig_git_repo_init = git.Repo.__init__
+
+ def git_repo_init(myself, *args, **kw):
+ orig_git_repo_init(myself, *args, **kw)
+ self.log.debug("Created git repo 0x%x %s" %
+ (id(myself), repr(myself)))
+
+ self.useFixture(fixtures.MonkeyPatch('git.Repo.__init__',
+ git_repo_init))
+
handler = logging.StreamHandler(self._log_stream)
formatter = logging.Formatter('%(asctime)s %(name)-32s '
'%(levelname)-8s %(message)s')
@@ -1874,8 +1886,6 @@
old_urlopen = urllib.request.urlopen
urllib.request.urlopen = URLOpenerFactory
- self._startMerger()
-
self.executor_server = RecordingExecutorServer(
self.config, self.connections,
jobdir_root=self.test_root,
@@ -1918,7 +1928,7 @@
self.sched.reconfigure(self.config)
self.sched.resume()
- def configure_connections(self):
+ def configure_connections(self, source_only=False):
# Set up gerrit related fakes
# Set a changes database so multiple FakeGerrit's can report back to
# a virtual canonical database given by the configured hostname
@@ -1961,7 +1971,7 @@
# Register connections from the config using fakes
self.connections = zuul.lib.connections.ConnectionRegistry()
- self.connections.configure(self.config)
+ self.connections.configure(self.config, source_only=source_only)
def setup_config(self):
# This creates the per-test configuration object. It can be
@@ -2126,12 +2136,19 @@
self.assertEqual({}, self.executor_server.job_workers)
# Make sure that git.Repo objects have been garbage collected.
repos = []
+ gc.disable()
gc.collect()
for obj in gc.get_objects():
if isinstance(obj, git.Repo):
- self.log.debug("Leaked git repo object: %s" % repr(obj))
+ self.log.debug("Leaked git repo object: 0x%x %s" %
+ (id(obj), repr(obj)))
+ for ref in gc.get_referrers(obj):
+ self.log.debug(" Referrer %s" % (repr(ref)))
repos.append(obj)
- self.assertEqual(len(repos), 0)
+ if repos:
+ for obj in gc.garbage:
+ self.log.debug(" Garbage %s" % (repr(obj)))
+ gc.enable()
self.assertEmptyQueues()
self.assertNodepoolState()
self.assertNoGeneratedKeys()
@@ -2144,8 +2161,6 @@
def shutdown(self):
self.log.debug("Shutting down after tests")
self.executor_client.stop()
- self.merge_server.stop()
- self.merge_server.join()
self.merge_client.stop()
self.executor_server.stop()
self.sched.stop()
diff --git a/tests/encrypt_secret.py b/tests/encrypt_secret.py
index b8524a0..0b0cf19 100644
--- a/tests/encrypt_secret.py
+++ b/tests/encrypt_secret.py
@@ -30,5 +30,6 @@
ciphertext = encryption.encrypt_pkcs1_oaep(sys.argv[1], public_key)
print(ciphertext.encode('base64'))
+
if __name__ == '__main__':
main()
diff --git a/tests/fixtures/zuul-connections-merger.conf b/tests/fixtures/zuul-connections-merger.conf
new file mode 100644
index 0000000..7a1bc42
--- /dev/null
+++ b/tests/fixtures/zuul-connections-merger.conf
@@ -0,0 +1,35 @@
+[gearman]
+server=127.0.0.1
+
+[zuul]
+job_name_in_report=true
+status_url=http://zuul.example.com/status
+
+[merger]
+git_dir=/tmp/zuul-test/git
+git_user_email=zuul@example.com
+git_user_name=zuul
+zuul_url=http://zuul.example.com/p
+
+[executor]
+git_dir=/tmp/zuul-test/executor-git
+
+[connection github]
+driver=github
+
+[connection gerrit]
+driver=gerrit
+server=review.example.com
+user=jenkins
+sshkey=fake_id_rsa1
+
+[connection resultsdb]
+driver=sql
+dburi=$MYSQL_FIXTURE_DBURI$
+
+[connection smtp]
+driver=smtp
+server=localhost
+port=25
+default_from=zuul@example.com
+default_to=you@example.com
diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py
index db32938..92270b7 100644
--- a/tests/unit/test_connection.py
+++ b/tests/unit/test_connection.py
@@ -265,3 +265,21 @@
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
+
+
+class TestConnectionsMerger(ZuulTestCase):
+ config_file = 'zuul-connections-merger.conf'
+ tenant_config_file = 'config/single-tenant/main.yaml'
+
+ def configure_connections(self):
+ super(TestConnectionsMerger, self).configure_connections(True)
+
+ def test_connections_merger(self):
+ "Test merger only configures source connections"
+
+ self.assertIn("gerrit", self.connections.connections)
+ self.assertIn("github", self.connections.connections)
+ self.assertNotIn("smtp", self.connections.connections)
+ self.assertNotIn("sql", self.connections.connections)
+ self.assertNotIn("timer", self.connections.connections)
+ self.assertNotIn("zuul", self.connections.connections)
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index f394c0c..d416369 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -1012,6 +1012,7 @@
self.fake_gerrit.addEvent(A.addApproval('approved', 1))
self.waitUntilSettled()
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
+ self.waitUntilSettled()
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
@@ -1509,8 +1510,8 @@
tenant = self.sched.abide.tenants.get('tenant-one')
trusted, project = tenant.getProject('org/project')
url = self.fake_gerrit.getGitUrl(project)
- self.merge_server.merger._addProject('review.example.com',
- 'org/project', url)
+ self.executor_server.merger._addProject('review.example.com',
+ 'org/project', url)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addPatchset(large=True)
# TODOv3(jeblair): add hostname to upstream root
diff --git a/tools/trigger-job.py b/tools/trigger-job.py
index 7123afc..dd69f1b 100755
--- a/tools/trigger-job.py
+++ b/tools/trigger-job.py
@@ -73,5 +73,6 @@
while not job.complete:
time.sleep(1)
+
if __name__ == '__main__':
main()
diff --git a/tools/update-storyboard.py b/tools/update-storyboard.py
index 12e6916..51434c9 100644
--- a/tools/update-storyboard.py
+++ b/tools/update-storyboard.py
@@ -96,5 +96,6 @@
if ok_lanes and not task_found:
add_task(sync, task, lanes[ok_lanes[0]])
+
if __name__ == '__main__':
main()
diff --git a/tox.ini b/tox.ini
index 6a50c6d..9b97eca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,6 +51,6 @@
[flake8]
# These are ignored intentionally in openstack-infra projects;
# please don't submit patches that solely correct them or enable them.
-ignore = E305,E125,E129,E402,H,W503
+ignore = E125,E129,E402,H,W503
show-source = True
exclude = .venv,.tox,dist,doc,build,*.egg
diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py
index e6b3461..fd95e92 100644
--- a/zuul/ansible/callback/zuul_stream.py
+++ b/zuul/ansible/callback/zuul_stream.py
@@ -40,6 +40,32 @@
yield buff
+def zuul_filter_result(result):
+ """Remove keys from shell/command output.
+
+ Zuul streams stdout into the log above, so including stdout and stderr
+ in the result dict that ansible displays in the logs is duplicate
+ noise. We keep stdout in the result dict so that other callback plugins
+ like ARA could also have access to it. But drop them here.
+
+ Remove changed so that we don't show a bunch of "changed" titles
+ on successful shell tasks, since that doesn't make sense from a Zuul
+ POV. The super class treats missing "changed" key as False.
+
+ Remove cmd because most of the script content where people want to
+ see the script run is run with -x. It's possible we may want to revist
+ this to be smarter about when we remove it - like, only remove it
+ if it has an embedded newline - so that for normal 'simple' uses
+ of cmd it'll echo what the command was for folks.
+ """
+
+ for key in ('changed', 'cmd',
+ 'stderr', 'stderr_lines',
+ 'stdout', 'stdout_lines'):
+ result.pop(key, None)
+ return result
+
+
class CallbackModule(default.CallbackModule):
'''
@@ -103,3 +129,37 @@
target=self._read_log, args=(host, ip))
p.daemon = True
p.start()
+
+ def v2_runner_on_failed(self, result, ignore_errors=False):
+ if result._task.action in ('command', 'shell'):
+ zuul_filter_result(result._result)
+ super(CallbackModule, self).v2_runner_on_failed(
+ result, ignore_errors=ignore_errors)
+
+ def v2_runner_on_ok(self, result):
+ if result._task.action in ('command', 'shell'):
+ zuul_filter_result(result._result)
+ else:
+ return super(CallbackModule, self).v2_runner_on_ok(result)
+
+ if self._play.strategy == 'free':
+ return super(CallbackModule, self).v2_runner_on_ok(result)
+
+ delegated_vars = result._result.get('_ansible_delegated_vars', None)
+
+ if delegated_vars:
+ msg = "ok: [{host} -> {delegated_host} %s]".format(
+ host=result._host.get_name(),
+ delegated_host=delegated_vars['ansible_host'])
+ else:
+ msg = "ok: [{host}]".format(host=result._host.get_name())
+
+ if result._task.loop and 'results' in result._result:
+ self._process_items(result)
+ else:
+ msg += " Runtime: {delta} Start: {start} End: {end}".format(
+ **result._result)
+
+ self._handle_warnings(result._result)
+
+ self._display.display(msg)
diff --git a/zuul/driver/sql/alembic_reporter/env.py b/zuul/driver/sql/alembic_reporter/env.py
index 56a5b7e..4542a22 100644
--- a/zuul/driver/sql/alembic_reporter/env.py
+++ b/zuul/driver/sql/alembic_reporter/env.py
@@ -64,6 +64,7 @@
with context.begin_transaction():
context.run_migrations()
+
if context.is_offline_mode():
run_migrations_offline()
else:
diff --git a/zuul/lib/connections.py b/zuul/lib/connections.py
index 720299a..9908fff 100644
--- a/zuul/lib/connections.py
+++ b/zuul/lib/connections.py
@@ -105,7 +105,7 @@
# The merger and the reporter only needs source driver.
# This makes sure Reporter like the SQLDriver are only created by
# the scheduler process
- if source_only and not issubclass(driver, SourceInterface):
+ if source_only and not isinstance(driver, SourceInterface):
continue
connection = driver.getConnection(con_name, con_config)
@@ -138,10 +138,11 @@
# Create default connections for drivers which need no
# connection information (e.g., 'timer' or 'zuul').
- for driver in self.drivers.values():
- if not hasattr(driver, 'getConnection'):
- connections[driver.name] = DefaultConnection(
- driver, driver.name, {})
+ if not source_only:
+ for driver in self.drivers.values():
+ if not hasattr(driver, 'getConnection'):
+ connections[driver.name] = DefaultConnection(
+ driver, driver.name, {})
self.connections = connections
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index 5b32e5b..d13a1b4 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -522,6 +522,10 @@
build_set.setConfiguration()
if build_set.merge_state == build_set.NEW:
return self.scheduleMerge(item, ['zuul.yaml', '.zuul.yaml'])
+ if build_set.merge_state == build_set.PENDING:
+ return False
+ if build_set.unable_to_merge:
+ return False
if build_set.config_error:
return False
return True