Send swift upload instructions to workers
Have zuul send signed credentials as part of the job for workers to
consume and upload assets to a defined location.
Zuul currently doesn't care about logs however this change will
suggest a LOG_PATH to builders as a unqiue destination prefix
allowing zuul to know preemptively the destination.
The workers are still required to send a URL of the final location.
Change-Id: I042cdd2dd2407f381cafcabc5c6b83d9b9a9eb00
diff --git a/tests/fixtures/layout-swift.yaml b/tests/fixtures/layout-swift.yaml
new file mode 100644
index 0000000..acaaad8
--- /dev/null
+++ b/tests/fixtures/layout-swift.yaml
@@ -0,0 +1,59 @@
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+ - name: post
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: ref-updated
+ ref: ^(?!refs/).*$
+
+ - name: gate
+ manager: DependentPipelineManager
+ failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
+ trigger:
+ gerrit:
+ - event: comment-added
+ approval:
+ - approved: 1
+ success:
+ gerrit:
+ verified: 2
+ submit: true
+ failure:
+ gerrit:
+ verified: -2
+ start:
+ gerrit:
+ verified: 0
+ precedence: high
+
+jobs:
+ - name: ^.*$
+ swift:
+ - name: logs
+ - name: ^.*-merge$
+ swift:
+ - name: logs
+ container: merge_logs
+ failure-message: Unable to merge change
+ - name: test-test
+ swift:
+ - name: MOSTLY
+ container: stash
+
+projects:
+ - name: org/project
+ gate:
+ - test-merge
+ - test-test
diff --git a/tests/fixtures/layouts/bad_swift.yaml b/tests/fixtures/layouts/bad_swift.yaml
new file mode 100644
index 0000000..d8a8c3f
--- /dev/null
+++ b/tests/fixtures/layouts/bad_swift.yaml
@@ -0,0 +1,29 @@
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+jobs:
+ - name: ^.*$
+ swift:
+ - name: logs
+ - name: ^.*-merge$
+ swift:
+ container: merge_assets
+ failure-message: Unable to merge change
+ - name: test-test
+ swift:
+
+projects:
+ - name: test-org/test
+ check:
+ - test-merge
+ - test-test
diff --git a/tests/fixtures/layouts/good_swift.yaml b/tests/fixtures/layouts/good_swift.yaml
new file mode 100644
index 0000000..913c268
--- /dev/null
+++ b/tests/fixtures/layouts/good_swift.yaml
@@ -0,0 +1,32 @@
+pipelines:
+ - name: check
+ manager: IndependentPipelineManager
+ trigger:
+ gerrit:
+ - event: patchset-created
+ success:
+ gerrit:
+ verified: 1
+ failure:
+ gerrit:
+ verified: -1
+
+jobs:
+ - name: ^.*$
+ swift:
+ - name: logs
+ - name: ^.*-merge$
+ swift:
+ - name: assets
+ container: merge_assets
+ failure-message: Unable to merge change
+ - name: test-test
+ swift:
+ - name: mostly
+ container: stash
+
+projects:
+ - name: test-org/test
+ check:
+ - test-merge
+ - test-test
diff --git a/tests/fixtures/zuul.conf b/tests/fixtures/zuul.conf
index bee06e4..ec76cd0 100644
--- a/tests/fixtures/zuul.conf
+++ b/tests/fixtures/zuul.conf
@@ -21,4 +21,14 @@
server=localhost
port=25
default_from=zuul@example.com
-default_to=you@example.com
\ No newline at end of file
+default_to=you@example.com
+
+[swift]
+authurl=https://identity.api.example.org/v2.0/
+user=username
+key=password
+tenant_name=" "
+
+default_container=logs
+region_name=EXP
+logserver_prefix=http://logs.example.org/server.app/
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 351854d..ade884a 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -30,6 +30,7 @@
import socket
import string
import subprocess
+import swiftclient
import threading
import time
import urllib
@@ -47,6 +48,7 @@
import zuul.rpclistener
import zuul.rpcclient
import zuul.launcher.gearman
+import zuul.lib.swift
import zuul.merger.server
import zuul.merger.client
import zuul.reporter.gerrit
@@ -743,6 +745,18 @@
return True
+class FakeSwiftClientConnection(swiftclient.client.Connection):
+ def post_account(self, headers):
+ # Do nothing
+ pass
+
+ def get_auth(self):
+ # Returns endpoint and (unused) auth token
+ endpoint = os.path.join('https://storage.example.org', 'V1',
+ 'AUTH_account')
+ return endpoint, ''
+
+
class TestScheduler(testtools.TestCase):
log = logging.getLogger("zuul.test")
@@ -823,12 +837,18 @@
self.sched = zuul.scheduler.Scheduler()
+ self.useFixture(fixtures.MonkeyPatch('swiftclient.client.Connection',
+ FakeSwiftClientConnection))
+ self.swift = zuul.lib.swift.Swift(self.config)
+
def URLOpenerFactory(*args, **kw):
args = [self.fake_gerrit] + list(args)
return FakeURLOpener(self.upstream_root, *args, **kw)
urllib2.urlopen = URLOpenerFactory
- self.launcher = zuul.launcher.gearman.Gearman(self.config, self.sched)
+
+ self.launcher = zuul.launcher.gearman.Gearman(self.config, self.sched,
+ self.swift)
self.merge_client = zuul.merger.client.MergeClient(
self.config, self.sched)
@@ -3810,3 +3830,48 @@
self.assertEqual(1, len(self.smtp_messages))
self.assertEqual('The merge failed! For more information...',
self.smtp_messages[0]['body'])
+
+ def test_swift_instructions(self):
+ "Test that the correct swift instructions are sent to the workers"
+ self.config.set('zuul', 'layout_config',
+ 'tests/fixtures/layout-swift.yaml')
+ self.sched.reconfigure(self.config)
+ self.registerJobs()
+
+ self.worker.hold_jobs_in_build = True
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+
+ A.addApproval('CRVW', 2)
+ self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
+ self.waitUntilSettled()
+
+ self.assertEqual(
+ "https://storage.example.org/V1/AUTH_account/merge_logs/1/1/1/"
+ "gate/test-merge/",
+ self.builds[0].parameters['SWIFT_logs_URL'][:-32])
+ self.assertEqual(5,
+ len(self.builds[0].parameters['SWIFT_logs_HMAC_BODY'].
+ split('\n')))
+ self.assertIn('SWIFT_logs_SIGNATURE', self.builds[0].parameters)
+
+ self.assertEqual(
+ "https://storage.example.org/V1/AUTH_account/logs/1/1/1/"
+ "gate/test-test/",
+ self.builds[1].parameters['SWIFT_logs_URL'][:-32])
+ self.assertEqual(5,
+ len(self.builds[1].parameters['SWIFT_logs_HMAC_BODY'].
+ split('\n')))
+ self.assertIn('SWIFT_logs_SIGNATURE', self.builds[1].parameters)
+
+ self.assertEqual(
+ "https://storage.example.org/V1/AUTH_account/stash/1/1/1/"
+ "gate/test-test/",
+ self.builds[1].parameters['SWIFT_MOSTLY_URL'][:-32])
+ self.assertEqual(5,
+ len(self.builds[1].
+ parameters['SWIFT_MOSTLY_HMAC_BODY'].split('\n')))
+ self.assertIn('SWIFT_MOSTLY_SIGNATURE', self.builds[1].parameters)
+
+ self.worker.hold_jobs_in_build = False
+ self.worker.release()
+ self.waitUntilSettled()