Merge branch 'master' into workingv3

This includes forward-porting changes to launcher/server.py with the
exception of the pre/post playbooks changes which will be done in a
follow up commit as they have deviated.

Change-Id: I13aa229c1460b748745babe178c0a745e52f841c
diff --git a/tests/base.py b/tests/base.py
index b343655..9c76d43 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -561,6 +561,7 @@
         self.wait_condition = threading.Condition()
         self.waiting = False
         self.aborted = False
+        self.requeue = False
         self.created = time.time()
         self.run_error = False
         self.changes = None
@@ -616,6 +617,8 @@
             result = 'FAILURE'
         if self.aborted:
             result = 'ABORTED'
+        if self.requeue:
+            result = None
 
         if self.run_error:
             result = 'RUN_ERROR'
@@ -885,8 +888,19 @@
             self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
         if (os.environ.get('OS_LOG_CAPTURE') == 'True' or
             os.environ.get('OS_LOG_CAPTURE') == '1'):
+            log_level = logging.DEBUG
+            if os.environ.get('OS_LOG_LEVEL') == 'DEBUG':
+                log_level = logging.DEBUG
+            elif os.environ.get('OS_LOG_LEVEL') == 'INFO':
+                log_level = logging.INFO
+            elif os.environ.get('OS_LOG_LEVEL') == 'WARNING':
+                log_level = logging.WARNING
+            elif os.environ.get('OS_LOG_LEVEL') == 'ERROR':
+                log_level = logging.ERROR
+            elif os.environ.get('OS_LOG_LEVEL') == 'CRITICAL':
+                log_level = logging.CRITICAL
             self.useFixture(fixtures.FakeLogger(
-                level=logging.DEBUG,
+                level=log_level,
                 format='%(asctime)s %(name)-32s '
                 '%(levelname)-8s %(message)s'))
 
diff --git a/tests/fixtures/layout-abort-attempts.yaml b/tests/fixtures/layout-abort-attempts.yaml
new file mode 100644
index 0000000..86d9d78
--- /dev/null
+++ b/tests/fixtures/layout-abort-attempts.yaml
@@ -0,0 +1,30 @@
+pipelines:
+  - name: check
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        verified: 1
+    failure:
+      gerrit:
+        verified: -1
+
+  - name: post
+    manager: IndependentPipelineManager
+    trigger:
+      gerrit:
+        - event: ref-updated
+          ref: ^(?!refs/).*$
+
+jobs:
+  - name: project-test1
+    attempts: 4
+
+projects:
+  - name: org/project
+    check:
+      - project-merge:
+        - project-test1
+        - project-test2
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 259b7ea..acbc144 100755
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -4607,6 +4607,40 @@
             '- docs-draft-test2 https://server/job/docs-draft-test2/1/',
             body[3])
 
+    @skip("Disabled for early v3 development")
+    def test_rerun_on_abort(self):
+        "Test that if a worker fails to run a job, it is run again"
+
+        self.config.set('zuul', 'layout_config',
+                        'tests/fixtures/layout-abort-attempts.yaml')
+        self.sched.reconfigure(self.config)
+        self.worker.hold_jobs_in_build = True
+        A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+        self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
+        self.waitUntilSettled()
+
+        self.worker.release('.*-merge')
+        self.waitUntilSettled()
+
+        self.assertEqual(len(self.builds), 2)
+        self.builds[0].requeue = True
+        self.worker.release('.*-test*')
+        self.waitUntilSettled()
+
+        for x in range(3):
+            self.assertEqual(len(self.builds), 1)
+            self.builds[0].requeue = True
+            self.worker.release('.*-test1')
+            self.waitUntilSettled()
+
+        self.worker.hold_jobs_in_build = False
+        self.worker.release()
+        self.waitUntilSettled()
+        self.assertEqual(len(self.history), 6)
+        self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2)
+        self.assertEqual(A.reported, 1)
+        self.assertIn('RETRY_LIMIT', A.messages[0])
+
 
 class TestDuplicatePipeline(ZuulTestCase):
     tenant_config_file = 'config/duplicate-pipeline/main.yaml'