diff --git a/tests/base.py b/tests/base.py
index c161b15..c1e7349 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -2069,12 +2069,18 @@
         """
 
         source_path = os.path.join(FIXTURE_DIR, source_name)
-        commit_data = {}
-        with open(source_path, 'r') as nt:
-            commit_data['zuul.yaml'] = nt.read()
+        files = {}
+        with open(source_path, 'r') as f:
+            data = f.read()
+            layout = yaml.safe_load(data)
+            files['zuul.yaml'] = data
+        for item in layout:
+            if 'job' in item:
+                jobname = item['job']['name']
+                files['playbooks/%s.yaml' % jobname] = ''
         before = self.addCommitToRepo(
             project_name, 'Pulling content from %s' % source_name,
-            commit_data)
+            files)
         return before
 
     def addEvent(self, connection, event):