Merge "Remove state_dir from setMountsMap" into feature/zuulv3
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index cc9d181..e37c05c 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -296,10 +296,29 @@
finger_port=79
**git_dir**
- Directory that Zuul should clone local git repositories to::
+ Directory that Zuul should clone local git repositories to. The
+ executor keeps a local copy of every git repository it works with to
+ speed operations and perform speculative merging.
+
+ This should be on the same filesystem as **job_dir** so that when
+ git repos are cloned into the job workspaces, they can be
+ hard-linked to the local git cache. Example::
git_dir=/var/lib/zuul/git
+**job_dir**
+ Directory that Zuul should use to hold temporary job directories.
+ When each job is run, a new entry will be created under this
+ directory to hold the configuration and scratch workspace for that
+ job. It will be deleted at the end of the job (unless the
+ `--keep-jobdir` command line option is specified).
+
+ This should be on the same filesystem as **git_dir** so that when
+ git repos are cloned into the job workspaces, they can be
+ hard-linked to the local git cache. Example::
+
+ job_dir=/var/lib/zuul/jobs
+
**log_config**
Path to log config file for the executor process::
diff --git a/doc/source/admin/tenants.rst b/doc/source/admin/tenants.rst
index 60873a9..18ec381 100644
--- a/doc/source/admin/tenants.rst
+++ b/doc/source/admin/tenants.rst
@@ -33,7 +33,7 @@
config-projects:
- common-config
- shared-jobs:
- include: jobs
+ include: job
untrusted-projects:
- zuul-jobs:
shadow: common-config
diff --git a/tests/base.py b/tests/base.py
index 2c478ad..2b6147a 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1131,8 +1131,9 @@
self.requeue = False
self.created = time.time()
self.changes = None
- if 'ZUUL_CHANGE_IDS' in self.parameters:
- self.changes = self.parameters['ZUUL_CHANGE_IDS']
+ items = self.parameters['zuul']['items']
+ self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset'])
+ for x in items if 'change' in x])
def __repr__(self):
waiting = ''
@@ -1180,7 +1181,7 @@
self.log.debug("Build %s continuing" % self.unique)
result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
- if (('ZUUL_REF' in self.parameters) and self.shouldFail()):
+ if self.shouldFail():
result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
if self.aborted:
result = (RecordingAnsibleJob.RESULT_ABORTED, None)
@@ -1215,8 +1216,7 @@
except NoSuchPathError as e:
self.log.debug('%s' % e)
return False
- ref = self.parameters['ZUUL_REF']
- repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
+ repo_messages = [c.message.strip() for c in repo.iter_commits()]
commit_message = '%s-1' % change.subject
self.log.debug("Checking if build %s has changes; commit_message "
"%s; repo_messages %s" % (self, commit_message,
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 8493570..875a9d7 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -36,8 +36,6 @@
build_params = self.builds[0].parameters
self.assertEqual('master', build_params['ZUUL_BRANCH'])
- self.assertEqual(str(A.number), build_params['ZUUL_CHANGE'])
- self.assertEqual(A.head_sha, build_params['ZUUL_PATCHSET'])
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
@@ -50,8 +48,8 @@
job = self.getJobFromHistory('project-test2')
zuulvars = job.parameters['zuul']
- self.assertEqual(A.number, zuulvars['change'])
- self.assertEqual(A.head_sha, zuulvars['patchset'])
+ self.assertEqual(str(A.number), zuulvars['change'])
+ self.assertEqual(str(A.head_sha), zuulvars['patchset'])
self.assertEqual(1, len(A.comments))
self.assertEqual(2, len(self.history))
@@ -113,10 +111,10 @@
self.waitUntilSettled()
build_params = self.builds[0].parameters
- self.assertEqual('refs/tags/newtag', build_params['ZUUL_REF'])
+ self.assertEqual('refs/tags/newtag', build_params['zuul']['ref'])
self.assertEqual('00000000000000000000000000000000',
- build_params['ZUUL_OLDREV'])
- self.assertEqual(sha, build_params['ZUUL_NEWREV'])
+ build_params['zuul']['oldrev'])
+ self.assertEqual(sha, build_params['zuul']['newrev'])
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
@@ -137,9 +135,9 @@
self.waitUntilSettled()
build_params = self.builds[0].parameters
- self.assertEqual('refs/heads/master', build_params['ZUUL_REF'])
- self.assertEqual(old_sha, build_params['ZUUL_OLDREV'])
- self.assertEqual(new_sha, build_params['ZUUL_NEWREV'])
+ self.assertEqual('refs/heads/master', build_params['zuul']['ref'])
+ self.assertEqual(old_sha, build_params['zuul']['oldrev'])
+ self.assertEqual(new_sha, build_params['zuul']['newrev'])
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index d4290a9..1e0affc 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -1283,7 +1283,7 @@
self.executor_server.hold_jobs_in_build = False
for build in self.builds:
- if build.parameters['ZUUL_CHANGE'] != '1':
+ if build.parameters['zuul']['change'] != '1':
build.release()
self.waitUntilSettled()
@@ -2033,34 +2033,19 @@
self.executor_server.release('.*-merge')
self.waitUntilSettled()
- a_zref = b_zref = c_zref = d_zref = None
a_build = b_build = c_build = d_build = None
for x in self.builds:
- if x.parameters['ZUUL_CHANGE'] == '3':
- a_zref = x.parameters['ZUUL_REF']
+ if x.parameters['zuul']['change'] == '3':
a_build = x
- elif x.parameters['ZUUL_CHANGE'] == '4':
- b_zref = x.parameters['ZUUL_REF']
+ elif x.parameters['zuul']['change'] == '4':
b_build = x
- elif x.parameters['ZUUL_CHANGE'] == '5':
- c_zref = x.parameters['ZUUL_REF']
+ elif x.parameters['zuul']['change'] == '5':
c_build = x
- elif x.parameters['ZUUL_CHANGE'] == '6':
- d_zref = x.parameters['ZUUL_REF']
+ elif x.parameters['zuul']['change'] == '6':
d_build = x
if a_build and b_build and c_build and d_build:
break
- # There are... four... refs.
- self.assertIsNotNone(a_zref)
- self.assertIsNotNone(b_zref)
- self.assertIsNotNone(c_zref)
- self.assertIsNotNone(d_zref)
-
- # And they should all be different
- refs = set([a_zref, b_zref, c_zref, d_zref])
- self.assertEqual(len(refs), 4)
-
# should have a, not b, and should not be in project2
self.assertTrue(a_build.hasChanges(A))
self.assertFalse(a_build.hasChanges(B, M2))
@@ -4776,12 +4761,10 @@
self.waitUntilSettled()
build = self.builds[-1]
- ref = self.getParameter(build, 'ZUUL_REF')
-
path = os.path.join(build.jobdir.src_root, 'review.example.com',
project)
repo = git.Repo(path)
- repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
+ repo_messages = [c.message.strip() for c in repo.iter_commits()]
repo_messages.reverse()
self.executor_server.hold_jobs_in_build = False
@@ -4848,12 +4831,11 @@
build = self.builds[-1]
self.assertEqual(self.getParameter(build, 'ZUUL_BRANCH'), 'mp')
- ref = self.getParameter(build, 'ZUUL_REF')
path = os.path.join(build.jobdir.src_root, 'review.example.com',
'org/project-merge-branches')
repo = git.Repo(path)
- repo_messages = [c.message.strip() for c in repo.iter_commits(ref)]
+ repo_messages = [c.message.strip() for c in repo.iter_commits()]
repo_messages.reverse()
correct_messages = [
'initial commit',
@@ -4889,16 +4871,12 @@
for job in self.builds:
if 'project-merge' in job.name:
job_A = job
- ref_A = self.getParameter(job_A, 'ZUUL_REF')
- commit_A = self.getParameter(job_A, 'ZUUL_COMMIT')
- self.log.debug("Got Zuul ref for change A: %s" % ref_A)
- self.log.debug("Got Zuul commit for change A: %s" % commit_A)
path = os.path.join(job_A.jobdir.src_root, 'review.example.com',
'org/project-merge-branches')
repo = git.Repo(path)
repo_messages = [c.message.strip()
- for c in repo.iter_commits(ref_A)]
+ for c in repo.iter_commits()]
repo_messages.reverse()
correct_messages = [
'initial commit', 'add content from fixture', 'A-1']
@@ -4911,16 +4889,11 @@
for job in self.builds:
if 'project-merge' in job.name:
job_B = job
- ref_B = self.getParameter(job_B, 'ZUUL_REF')
- commit_B = self.getParameter(job_B, 'ZUUL_COMMIT')
- self.log.debug("Got Zuul ref for change B: %s" % ref_B)
- self.log.debug("Got Zuul commit for change B: %s" % commit_B)
path = os.path.join(job_B.jobdir.src_root, 'review.example.com',
'org/project-merge-branches')
repo = git.Repo(path)
- repo_messages = [c.message.strip()
- for c in repo.iter_commits(ref_B)]
+ repo_messages = [c.message.strip() for c in repo.iter_commits()]
repo_messages.reverse()
correct_messages = [
'initial commit', 'add content from fixture', 'mp commit', 'B-1']
@@ -4933,15 +4906,11 @@
for job in self.builds:
if 'project-merge' in job.name:
job_C = job
- ref_C = self.getParameter(job_C, 'ZUUL_REF')
- commit_C = self.getParameter(job_C, 'ZUUL_COMMIT')
- self.log.debug("Got Zuul ref for change C: %s" % ref_C)
- self.log.debug("Got Zuul commit for change C: %s" % commit_C)
+
path = os.path.join(job_C.jobdir.src_root, 'review.example.com',
'org/project-merge-branches')
repo = git.Repo(path)
- repo_messages = [c.message.strip()
- for c in repo.iter_commits(ref_C)]
+ repo_messages = [c.message.strip() for c in repo.iter_commits()]
repo_messages.reverse()
correct_messages = [
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index 6a1a214..06ef0ba 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -82,7 +82,7 @@
self.log.info("Starting log streamer")
streamer = zuul.lib.log_streamer.LogStreamer(
- self.user, '0.0.0.0', self.finger_port, self.jobroot_dir)
+ self.user, '0.0.0.0', self.finger_port, self.job_dir)
# Keep running until the parent dies:
pipe_read = os.fdopen(pipe_read)
@@ -111,15 +111,15 @@
self.user = get_default(self.config, 'executor', 'user', 'zuul')
- if self.config.has_option('executor', 'jobroot_dir'):
- self.jobroot_dir = os.path.expanduser(
- self.config.get('executor', 'jobroot_dir'))
- if not os.path.isdir(self.jobroot_dir):
- print("Invalid jobroot_dir: {jobroot_dir}".format(
- jobroot_dir=self.jobroot_dir))
+ if self.config.has_option('executor', 'job_dir'):
+ self.job_dir = os.path.expanduser(
+ self.config.get('executor', 'job_dir'))
+ if not os.path.isdir(self.job_dir):
+ print("Invalid job_dir: {job_dir}".format(
+ job_dir=self.job_dir))
sys.exit(1)
else:
- self.jobroot_dir = tempfile.gettempdir()
+ self.job_dir = tempfile.gettempdir()
self.setup_logging('executor', 'log_config')
self.log = logging.getLogger("zuul.Executor")
@@ -134,7 +134,7 @@
ExecutorServer = zuul.executor.server.ExecutorServer
self.executor = ExecutorServer(self.config, self.connections,
- jobdir_root=self.jobroot_dir,
+ jobdir_root=self.job_dir,
keep_jobdir=self.args.keep_jobdir,
log_streaming_port=self.finger_port)
self.executor.start()
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 8f8465a..98adc69 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -80,7 +80,7 @@
patchset = data.get('patchSet')
if patchset:
event.patch_number = patchset.get('number')
- event.refspec = patchset.get('ref')
+ event.ref = patchset.get('ref')
event.approvals = data.get('approvals', [])
event.comment = data.get('comment')
refupdate = data.get('refUpdate')
@@ -418,7 +418,7 @@
files = []
for ps in data['patchSets']:
if ps['number'] == change.patchset:
- change.refspec = ps['ref']
+ change.ref = ps['ref']
for f in ps.get('files', []):
files.append(f['file'])
if int(ps['number']) > int(max_ps):
diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py
index ff113ce..c3cafba 100644
--- a/zuul/driver/github/githubconnection.py
+++ b/zuul/driver/github/githubconnection.py
@@ -299,7 +299,7 @@
event.change_number)
event.updated_at = pr_body.get('updated_at')
event.branch = base.get('ref')
- event.refspec = "refs/pull/" + str(pr_body.get('number')) + "/head"
+ event.ref = "refs/pull/" + str(pr_body.get('number')) + "/head"
event.patch_number = head.get('sha')
event.title = pr_body.get('title')
@@ -601,7 +601,7 @@
self.log.info("Updating %s" % (change,))
change.pr = self.getPull(change.project.name, change.number)
- change.refspec = "refs/pull/%s/head" % change.number
+ change.ref = "refs/pull/%s/head" % change.number
change.branch = change.pr.get('base').get('ref')
change.files = change.pr.get('files')
change.title = change.pr.get('title')
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index 214b667..aca1b06 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -41,14 +41,14 @@
with self.connection.engine.begin() as conn:
change = getattr(item.change, 'number', '')
patchset = getattr(item.change, 'patchset', '')
- refspec = getattr(item.change, 'refspec', item.change.newrev)
+ ref = getattr(item.change, 'ref', '')
buildset_ins = self.connection.zuul_buildset_table.insert().values(
zuul_ref=item.current_build_set.ref,
pipeline=item.pipeline.name,
project=item.change.project.name,
change=change,
patchset=patchset,
- ref=refspec,
+ ref=ref,
score=self.result_score,
message=self._formatItemReport(
item, with_jobs=False),
diff --git a/zuul/driver/zuul/__init__.py b/zuul/driver/zuul/__init__.py
index 08612dc..0f6ec7d 100644
--- a/zuul/driver/zuul/__init__.py
+++ b/zuul/driver/zuul/__init__.py
@@ -82,7 +82,7 @@
event.branch = change.branch
event.change_url = change.url
event.patch_number = change.patchset
- event.refspec = change.refspec
+ event.ref = change.ref
self.sched.addEvent(event)
def _createParentChangeEnqueuedEvents(self, change, pipeline):
@@ -104,7 +104,7 @@
event.branch = change.branch
event.change_url = change.url
event.patch_number = change.patchset
- event.refspec = change.refspec
+ event.ref = change.ref
self.sched.addEvent(event)
def getTrigger(self, connection_name, config=None):
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index beb8964..e5fa276 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -168,9 +168,9 @@
if hasattr(item.change, 'tag'):
zuul_params['tag'] = item.change.tag
if hasattr(item.change, 'number'):
- zuul_params['change'] = item.change.number
+ zuul_params['change'] = str(item.change.number)
if hasattr(item.change, 'patchset'):
- zuul_params['patchset'] = item.change.patchset
+ zuul_params['patchset'] = str(item.change.patchset)
if hasattr(item.change, 'oldrev') and item.change.oldrev:
zuul_params['oldrev'] = item.change.oldrev
if hasattr(item.change, 'newrev') and item.change.newrev:
@@ -183,9 +183,9 @@
canonical_hostname=i.change.project.canonical_hostname,
canonical_name=i.change.project.canonical_name)
if hasattr(i.change, 'number'):
- d['change'] = i.change.number
+ d['change'] = str(i.change.number)
if hasattr(i.change, 'patchset'):
- d['patchset'] = i.change.number
+ d['patchset'] = str(i.change.patchset)
if hasattr(i.change, 'branch'):
d['branch'] = i.change.branch
zuul_params['items'].append(d)
@@ -196,52 +196,13 @@
params['ZUUL_PIPELINE'] = pipeline.name
params['ZUUL_URL'] = item.current_build_set.zuul_url
params['ZUUL_VOTING'] = job.voting and '1' or '0'
- if hasattr(item.change, 'refspec'):
+ if hasattr(item.change, 'number'):
changes_str = '^'.join(
['%s:%s:%s' % (i.change.project.name, i.change.branch,
- i.change.refspec)
+ i.change.ref)
for i in all_items])
params['ZUUL_BRANCH'] = item.change.branch
params['ZUUL_CHANGES'] = changes_str
- params['ZUUL_REF'] = ('refs/zuul/%s/%s' %
- (item.change.branch,
- item.current_build_set.ref))
- params['ZUUL_COMMIT'] = item.current_build_set.commit
-
- zuul_changes = ' '.join(['%s,%s' % (i.change.number,
- i.change.patchset)
- for i in all_items])
- params['ZUUL_CHANGE_IDS'] = zuul_changes
- params['ZUUL_CHANGE'] = str(item.change.number)
- params['ZUUL_PATCHSET'] = str(item.change.patchset)
- if hasattr(item.change, 'ref') and item.change.ref is not None:
- params['ZUUL_REFNAME'] = item.change.ref
- params['ZUUL_OLDREV'] = item.change.oldrev
- params['ZUUL_NEWREV'] = item.change.newrev
-
- params['ZUUL_REF'] = item.change.ref
- params['ZUUL_COMMIT'] = item.change.newrev
-
- # This is what we should be heading toward for parameters:
-
- # required:
- # ZUUL_UUID
- # ZUUL_REF (/refs/zuul/..., /refs/tags/foo, master)
- # ZUUL_COMMIT
-
- # optional:
- # ZUUL_PROJECT
- # ZUUL_PIPELINE
-
- # optional (changes only):
- # ZUUL_BRANCH
- # ZUUL_CHANGE
- # ZUUL_CHANGE_IDS
- # ZUUL_PATCHSET
-
- # optional (ref updated only):
- # ZUUL_OLDREV
- # ZUUL_NEWREV
params['job'] = job.name
params['timeout'] = job.timeout
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 9a07b5a..720f503 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -848,7 +848,7 @@
project['name'])
repos[project['canonical_name']] = repo
- merge_items = [i for i in args['items'] if i.get('refspec')]
+ merge_items = [i for i in args['items'] if i.get('number')]
if merge_items:
if not self.doMergeChanges(merger, merge_items,
args['repo_state']):
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index 93340fa..c5d1f2a 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -225,9 +225,9 @@
except AssertionError:
origin.fetch(ref)
- def fetchFrom(self, repository, refspec):
+ def fetchFrom(self, repository, ref):
repo = self.createRepoObject()
- repo.git.fetch(repository, refspec)
+ repo.git.fetch(repository, ref)
def createZuulRef(self, ref, commit='HEAD'):
repo = self.createRepoObject()
@@ -391,11 +391,11 @@
try:
mode = item['merge_mode']
if mode == zuul.model.MERGER_MERGE:
- commit = repo.merge(item['refspec'])
+ commit = repo.merge(item['ref'])
elif mode == zuul.model.MERGER_MERGE_RESOLVE:
- commit = repo.merge(item['refspec'], 'resolve')
+ commit = repo.merge(item['ref'], 'resolve')
elif mode == zuul.model.MERGER_CHERRY_PICK:
- commit = repo.cherryPick(item['refspec'])
+ commit = repo.cherryPick(item['ref'])
else:
raise Exception("Unsupported merge mode: %s" % mode)
except git.GitCommandError:
@@ -410,9 +410,10 @@
return commit
def _mergeItem(self, item, recent, repo_state):
- self.log.debug("Processing refspec %s for project %s/%s / %s ref %s" %
- (item['refspec'], item['connection'],
- item['project'], item['branch'], item['ref']))
+ self.log.debug("Processing ref %s for project %s/%s / %s uuid %s" %
+ (item['ref'], item['connection'],
+ item['project'], item['branch'],
+ item['buildset_uuid']))
repo = self.getRepo(item['connection'], item['project'])
key = (item['connection'], item['project'], item['branch'])
@@ -451,7 +452,7 @@
zuul_ref = None
try:
repo = self.getRepo(connection, project)
- zuul_ref = branch + '/' + item['ref']
+ zuul_ref = branch + '/' + item['buildset_uuid']
if not repo.getCommitFromRef(zuul_ref):
repo.createZuulRef(zuul_ref, mrc)
except Exception:
diff --git a/zuul/model.py b/zuul/model.py
index ed77864..ed50164 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -1841,12 +1841,10 @@
patchset = None
oldrev = None
newrev = None
- refspec = None
branch = None
if hasattr(self.change, 'number'):
number = self.change.number
patchset = self.change.patchset
- refspec = self.change.refspec
if hasattr(self.change, 'newrev'):
oldrev = self.change.oldrev
newrev = self.change.newrev
@@ -1860,9 +1858,9 @@
return dict(project=project.name,
connection=connection_name,
merge_mode=self.current_build_set.getMergeMode(),
- refspec=refspec,
+ ref=self.change.ref,
branch=branch,
- ref=self.current_build_set.ref,
+ buildset_uuid=self.current_build_set.uuid,
number=number,
patchset=patchset,
oldrev=oldrev,
@@ -1951,7 +1949,6 @@
self.number = None
self.url = None
self.patchset = None
- self.refspec = None
self.needs_changes = []
self.needed_by_changes = []
@@ -2008,6 +2005,7 @@
# common
self.type = None
self.branch_updated = False
+ self.ref = None
# For management events (eg: enqueue / promote)
self.tenant_name = None
self.project_hostname = None
@@ -2019,12 +2017,10 @@
self.change_number = None
self.change_url = None
self.patch_number = None
- self.refspec = None
self.branch = None
self.comment = None
self.state = None
# ref-updated
- self.ref = None
self.oldrev = None
self.newrev = None
# For events that arrive with a destination pipeline (eg, from