Merge "Docs: add initial documentation style guide" into feature/zuulv3
diff --git a/tests/base.py b/tests/base.py
index 0f188bd..7209c87 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -139,7 +139,8 @@
'Verified': ('Verified', -2, 2)}
def __init__(self, gerrit, number, project, branch, subject,
- status='NEW', upstream_root=None, files={}):
+ status='NEW', upstream_root=None, files={},
+ parent=None):
self.gerrit = gerrit
self.source = gerrit
self.reported = 0
@@ -174,16 +175,18 @@
'url': 'https://hostname/%s' % number}
self.upstream_root = upstream_root
- self.addPatchset(files=files)
+ self.addPatchset(files=files, parent=parent)
self.data['submitRecords'] = self.getSubmitRecords()
self.open = status == 'NEW'
- def addFakeChangeToRepo(self, msg, files, large):
+ def addFakeChangeToRepo(self, msg, files, large, parent):
path = os.path.join(self.upstream_root, self.project)
repo = git.Repo(path)
+ if parent is None:
+ parent = 'refs/tags/init'
ref = GerritChangeReference.create(
repo, '1/%s/%s' % (self.number, self.latest_patchset),
- 'refs/tags/init')
+ parent)
repo.head.reference = ref
zuul.merger.merger.reset_repo_to_head(repo)
repo.git.clean('-x', '-f', '-d')
@@ -211,7 +214,7 @@
repo.heads['master'].checkout()
return r
- def addPatchset(self, files=None, large=False):
+ def addPatchset(self, files=None, large=False, parent=None):
self.latest_patchset += 1
if not files:
fn = '%s-%s' % (self.branch.replace('/', '_'), self.number)
@@ -219,7 +222,7 @@
(self.branch, self.number, self.latest_patchset))
files = {fn: data}
msg = self.subject + '-' + str(self.latest_patchset)
- c = self.addFakeChangeToRepo(msg, files, large)
+ c = self.addFakeChangeToRepo(msg, files, large, parent)
ps_files = [{'file': '/COMMIT_MSG',
'type': 'ADDED'},
{'file': 'README',
@@ -469,12 +472,12 @@
self.upstream_root = upstream_root
def addFakeChange(self, project, branch, subject, status='NEW',
- files=None):
+ files=None, parent=None):
"""Add a change to the fake Gerrit."""
self.change_number += 1
c = FakeGerritChange(self, self.change_number, project, branch,
subject, upstream_root=self.upstream_root,
- status=status, files=files)
+ status=status, files=files, parent=parent)
self.changes[self.change_number] = c
return c
@@ -563,9 +566,43 @@
def __init__(self, branch='master'):
self.name = branch
+ class FakeStatus(object):
+ def __init__(self, state, url, description, context, user):
+ self._state = state
+ self._url = url
+ self._description = description
+ self._context = context
+ self._user = user
+
+ def as_dict(self):
+ return {
+ 'state': self._state,
+ 'url': self._url,
+ 'description': self._description,
+ 'context': self._context,
+ 'creator': {
+ 'login': self._user
+ }
+ }
+
+ class FakeCommit(object):
+ def __init__(self):
+ self._statuses = []
+
+ def set_status(self, state, url, description, context, user):
+ status = FakeGithub.FakeStatus(
+ state, url, description, context, user)
+ # always insert a status to the front of the list, to represent
+ # the last status provided for a commit.
+ self._statuses.insert(0, status)
+
+ def statuses(self):
+ return self._statuses
+
class FakeRepository(object):
def __init__(self):
self._branches = [FakeGithub.FakeBranch()]
+ self._commits = {}
def branches(self, protected=False):
if protected:
@@ -573,12 +610,40 @@
return []
return self._branches
+ def create_status(self, sha, state, url, description, context,
+ user='zuul'):
+ # Since we're bypassing github API, which would require a user, we
+ # default the user as 'zuul' here.
+ commit = self._commits.get(sha, None)
+ if commit is None:
+ commit = FakeGithub.FakeCommit()
+ self._commits[sha] = commit
+ commit.set_status(state, url, description, context, user)
+
+ def commit(self, sha):
+ commit = self._commits.get(sha, None)
+ if commit is None:
+ commit = FakeGithub.FakeCommit()
+ self._commits[sha] = commit
+ return commit
+
+ def __init__(self):
+ self._repos = {}
+
def user(self, login):
return self.FakeUser(login)
def repository(self, owner, proj):
- repo = self.FakeRepository()
- return repo
+ return self._repos.get((owner, proj), None)
+
+ def repo_from_project(self, project):
+ # This is a convenience method for the tests.
+ owner, proj = project.split('/')
+ return self.repository(owner, proj)
+
+ def addProject(self, project):
+ owner, proj = project.name.split('/')
+ self._repos[(owner, proj)] = self.FakeRepository()
class FakeGithubPullRequest(object):
@@ -893,6 +958,13 @@
}
return (name, data)
+ def setMerged(self, commit_message):
+ self.is_merged = True
+ self.merge_message = commit_message
+
+ repo = self._getRepo()
+ repo.heads[self.branch].commit = repo.commit(self.head_sha)
+
class FakeGithubConnection(githubconnection.GithubConnection):
log = logging.getLogger("zuul.test.FakeGithubConnection")
@@ -929,7 +1001,7 @@
def getPushEvent(self, project, ref, old_rev=None, new_rev=None,
added_files=[], removed_files=[], modified_files=[]):
if not old_rev:
- old_rev = '00000000000000000000000000000000'
+ old_rev = '0' * 40
if not new_rev:
new_rev = random_sha1()
name = 'push'
@@ -964,6 +1036,12 @@
data=payload, headers=headers)
return urllib.request.urlopen(req)
+ def addProject(self, project):
+ # use the original method here and additionally register it in the
+ # fake github
+ super(FakeGithubConnection, self).addProject(project)
+ self.getGithubClient(project).addProject(project)
+
def getPull(self, project, number):
pr = self.pull_requests[number - 1]
data = {
@@ -1034,30 +1112,15 @@
self.merge_not_allowed_count -= 1
raise MergeFailure('Merge was not successful due to mergeability'
' conflict')
- pull_request.is_merged = True
- pull_request.merge_message = commit_message
-
- def getCommitStatuses(self, project, sha):
- return self.statuses.get(project, {}).get(sha, [])
+ pull_request.setMerged(commit_message)
def setCommitStatus(self, project, sha, state, url='', description='',
context='default', user='zuul'):
- # record that this got reported
+ # record that this got reported and call original method
self.reports.append((project, sha, 'status', (user, context, state)))
- # always insert a status to the front of the list, to represent
- # the last status provided for a commit.
- # Since we're bypassing github API, which would require a user, we
- # default the user as 'zuul' here.
- self.statuses.setdefault(project, {}).setdefault(sha, [])
- self.statuses[project][sha].insert(0, {
- 'state': state,
- 'url': url,
- 'description': description,
- 'context': context,
- 'creator': {
- 'login': user
- }
- })
+ super(FakeGithubConnection, self).setCommitStatus(
+ project, sha, state,
+ url=url, description=description, context=context)
def labelPull(self, project, pr_number, label):
# record that this got reported
@@ -2145,15 +2208,15 @@
config = [{'tenant':
{'name': 'tenant-one',
'source': {driver:
- {'config-projects': ['common-config'],
+ {'config-projects': ['org/common-config'],
'untrusted-projects': untrusted_projects}}}}]
f.write(yaml.dump(config).encode('utf8'))
f.close()
self.config.set('scheduler', 'tenant_config',
os.path.join(FIXTURE_DIR, f.name))
- self.init_repo('common-config')
- self.addCommitToRepo('common-config', 'add content from fixture',
+ self.init_repo('org/common-config')
+ self.addCommitToRepo('org/common-config', 'add content from fixture',
files, branch='master', tag='init')
return True
diff --git a/tests/fixtures/config/multi-driver/git/common-config/playbooks/project-gerrit.yaml b/tests/fixtures/config/multi-driver/git/org_common-config/playbooks/project-gerrit.yaml
similarity index 100%
rename from tests/fixtures/config/multi-driver/git/common-config/playbooks/project-gerrit.yaml
rename to tests/fixtures/config/multi-driver/git/org_common-config/playbooks/project-gerrit.yaml
diff --git a/tests/fixtures/config/multi-driver/git/common-config/playbooks/project1-github.yaml b/tests/fixtures/config/multi-driver/git/org_common-config/playbooks/project1-github.yaml
similarity index 100%
rename from tests/fixtures/config/multi-driver/git/common-config/playbooks/project1-github.yaml
rename to tests/fixtures/config/multi-driver/git/org_common-config/playbooks/project1-github.yaml
diff --git a/tests/fixtures/config/multi-driver/git/common-config/zuul.yaml b/tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
similarity index 100%
rename from tests/fixtures/config/multi-driver/git/common-config/zuul.yaml
rename to tests/fixtures/config/multi-driver/git/org_common-config/zuul.yaml
diff --git a/tests/fixtures/config/multi-driver/main.yaml b/tests/fixtures/config/multi-driver/main.yaml
index 301df38..4eed523 100644
--- a/tests/fixtures/config/multi-driver/main.yaml
+++ b/tests/fixtures/config/multi-driver/main.yaml
@@ -3,7 +3,7 @@
source:
github:
config-projects:
- - common-config
+ - org/common-config
untrusted-projects:
- org/project1
gerrit:
diff --git a/tests/fixtures/config/push-reqs/git/common-config/playbooks/job1.yaml b/tests/fixtures/config/push-reqs/git/org_common-config/playbooks/job1.yaml
similarity index 100%
rename from tests/fixtures/config/push-reqs/git/common-config/playbooks/job1.yaml
rename to tests/fixtures/config/push-reqs/git/org_common-config/playbooks/job1.yaml
diff --git a/tests/fixtures/config/push-reqs/git/common-config/zuul.yaml b/tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
similarity index 100%
rename from tests/fixtures/config/push-reqs/git/common-config/zuul.yaml
rename to tests/fixtures/config/push-reqs/git/org_common-config/zuul.yaml
diff --git a/tests/fixtures/config/push-reqs/main.yaml b/tests/fixtures/config/push-reqs/main.yaml
index d9f1a42..b58db73 100644
--- a/tests/fixtures/config/push-reqs/main.yaml
+++ b/tests/fixtures/config/push-reqs/main.yaml
@@ -3,7 +3,7 @@
source:
github:
config-projects:
- - common-config
+ - org/common-config
untrusted-projects:
- org/project1
gerrit:
diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py
index 444d783..3793edc 100755
--- a/tests/unit/test_executor.py
+++ b/tests/unit/test_executor.py
@@ -348,19 +348,31 @@
p1 = "review.example.com/org/project1"
p2 = "review.example.com/org/project2"
projects = [p1, p2]
+ upstream = self.getUpstreamRepos(projects)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
event = A.getRefUpdatedEvent()
A.setMerged()
+ A_commit = str(upstream[p1].commit('master'))
+ self.log.debug("A commit: %s" % A_commit)
+
+ # Add another commit to the repo that merged right after this
+ # one to make sure that our post job runs with the one that we
+ # intended rather than simply the current repo state.
+ B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B',
+ parent='refs/changes/1/1/1')
+ B.setMerged()
+ B_commit = str(upstream[p1].commit('master'))
+ self.log.debug("B commit: %s" % B_commit)
+
self.fake_gerrit.addEvent(event)
self.waitUntilSettled()
- upstream = self.getUpstreamRepos(projects)
states = [
- {p1: dict(commit=str(upstream[p1].commit('master')),
- present=[A], branch='master'),
+ {p1: dict(commit=A_commit,
+ present=[A], absent=[B], branch='master'),
p2: dict(commit=str(upstream[p2].commit('master')),
- absent=[A], branch='master'),
+ absent=[A, B], branch='master'),
},
]
diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py
index 68fbe29..a088236 100644
--- a/tests/unit/test_github_driver.py
+++ b/tests/unit/test_github_driver.py
@@ -110,10 +110,8 @@
build_params = self.builds[0].parameters
self.assertEqual('refs/tags/newtag', build_params['zuul']['ref'])
- self.assertEqual('00000000000000000000000000000000',
- build_params['zuul']['oldrev'])
+ self.assertFalse('oldrev' in build_params['zuul'])
self.assertEqual(sha, build_params['zuul']['newrev'])
-
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
@@ -125,16 +123,20 @@
def test_push_event(self):
self.executor_server.hold_jobs_in_build = True
- old_sha = random_sha1()
- new_sha = random_sha1()
- self.fake_github.emitEvent(
- self.fake_github.getPushEvent('org/project', 'refs/heads/master',
- old_sha, new_sha))
+ A = self.fake_github.openFakePullRequest('org/project', 'master', 'A')
+ old_sha = '0' * 40
+ new_sha = A.head_sha
+ A.setMerged("merging A")
+ pevent = self.fake_github.getPushEvent(project='org/project',
+ ref='refs/heads/master',
+ old_rev=old_sha,
+ new_rev=new_sha)
+ self.fake_github.emitEvent(pevent)
self.waitUntilSettled()
build_params = self.builds[0].parameters
self.assertEqual('refs/heads/master', build_params['zuul']['ref'])
- self.assertEqual(old_sha, build_params['zuul']['oldrev'])
+ self.assertFalse('oldrev' in build_params['zuul'])
self.assertEqual(new_sha, build_params['zuul']['newrev'])
self.executor_server.hold_jobs_in_build = False
@@ -258,14 +260,19 @@
@simple_layout('layouts/reporting-github.yaml', driver='github')
def test_reporting(self):
project = 'org/project'
+ github = self.fake_github.github_client
+
# pipeline reports pull status both on start and success
self.executor_server.hold_jobs_in_build = True
A = self.fake_github.openFakePullRequest(project, 'master', 'A')
self.fake_github.emitEvent(A.getPullRequestOpenedEvent())
self.waitUntilSettled()
+
# We should have a status container for the head sha
- statuses = self.fake_github.statuses[project][A.head_sha]
- self.assertIn(A.head_sha, self.fake_github.statuses[project].keys())
+ self.assertIn(
+ A.head_sha, github.repo_from_project(project)._commits.keys())
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
+
# We should only have one status for the head sha
self.assertEqual(1, len(statuses))
check_status = statuses[0]
@@ -282,7 +289,7 @@
self.executor_server.release()
self.waitUntilSettled()
# We should only have two statuses for the head sha
- statuses = self.fake_github.statuses[project][A.head_sha]
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
self.assertEqual(2, len(statuses))
check_status = statuses[0]
check_url = ('http://zuul.example.com/status/#%s,%s' %
@@ -301,7 +308,7 @@
self.fake_github.emitEvent(
A.getCommentAddedEvent('reporting check'))
self.waitUntilSettled()
- statuses = self.fake_github.statuses[project][A.head_sha]
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
self.assertEqual(2, len(statuses))
# comments increased by one for the start message
self.assertEqual(2, len(A.comments))
@@ -311,7 +318,7 @@
self.executor_server.release()
self.waitUntilSettled()
# pipeline reports success status
- statuses = self.fake_github.statuses[project][A.head_sha]
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
self.assertEqual(3, len(statuses))
report_status = statuses[0]
self.assertEqual('tenant-one/reporting', report_status['context'])
@@ -343,7 +350,7 @@
self.fake_github.emitEvent(
A.getCommentAddedEvent('long pipeline'))
self.waitUntilSettled()
- statuses = self.fake_github.statuses[project][A.head_sha]
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
self.assertEqual(1, len(statuses))
check_status = statuses[0]
# Status is truncated due to long pipeline name
@@ -354,7 +361,7 @@
self.executor_server.release()
self.waitUntilSettled()
# We should only have two statuses for the head sha
- statuses = self.fake_github.statuses[project][A.head_sha]
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
self.assertEqual(2, len(statuses))
check_status = statuses[0]
# Status is truncated due to long pipeline name
@@ -366,9 +373,15 @@
project = 'org/project2'
# pipeline reports pull status both on start and success
self.executor_server.hold_jobs_in_build = True
- pevent = self.fake_github.getPushEvent(project=project,
- ref='refs/heads/master')
+ A = self.fake_github.openFakePullRequest(project, 'master', 'A')
+ old_sha = '0' * 40
+ new_sha = A.head_sha
+ A.setMerged("merging A")
+ pevent = self.fake_github.getPushEvent(project=project,
+ ref='refs/heads/master',
+ old_rev=old_sha,
+ new_rev=new_sha)
self.fake_github.emitEvent(pevent)
self.waitUntilSettled()
@@ -441,6 +454,8 @@
@simple_layout('layouts/reporting-multiple-github.yaml', driver='github')
def test_reporting_multiple_github(self):
project = 'org/project1'
+ github = self.fake_github.github_client
+
# pipeline reports pull status both on start and success
self.executor_server.hold_jobs_in_build = True
A = self.fake_github.openFakePullRequest(project, 'master', 'A')
@@ -451,8 +466,9 @@
self.fake_github.emitEvent(B.getPullRequestOpenedEvent())
self.waitUntilSettled()
# We should have a status container for the head sha
- statuses = self.fake_github.statuses[project][A.head_sha]
- self.assertIn(A.head_sha, self.fake_github.statuses[project].keys())
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
+ self.assertIn(
+ A.head_sha, github.repo_from_project(project)._commits.keys())
# We should only have one status for the head sha
self.assertEqual(1, len(statuses))
check_status = statuses[0]
@@ -468,7 +484,7 @@
self.executor_server.release()
self.waitUntilSettled()
# We should only have two statuses for the head sha
- statuses = self.fake_github.statuses[project][A.head_sha]
+ statuses = self.fake_github.getCommitStatuses(project, A.head_sha)
self.assertEqual(2, len(statuses))
check_status = statuses[0]
check_url = ('http://zuul.example.com/status/#%s,%s' %
@@ -656,7 +672,7 @@
@simple_layout('layouts/basic-github.yaml', driver='github')
def test_push_event_reconfigure(self):
- pevent = self.fake_github.getPushEvent(project='common-config',
+ pevent = self.fake_github.getPushEvent(project='org/common-config',
ref='refs/heads/master',
modified_files=['zuul.yaml'])
diff --git a/tests/unit/test_multi_driver.py b/tests/unit/test_multi_driver.py
index e40591b..1844c33 100644
--- a/tests/unit/test_multi_driver.py
+++ b/tests/unit/test_multi_driver.py
@@ -46,7 +46,8 @@
# Check on reporting results
# github should have a success status (only).
- statuses = self.fake_github.statuses['org/project1'][B.head_sha]
+ statuses = self.fake_github.getCommitStatuses(
+ 'org/project1', B.head_sha)
self.assertEqual(1, len(statuses))
self.assertEqual('success', statuses[0]['state'])
diff --git a/tests/unit/test_push_reqs.py b/tests/unit/test_push_reqs.py
index d3a1feb..80c3be9 100644
--- a/tests/unit/test_push_reqs.py
+++ b/tests/unit/test_push_reqs.py
@@ -25,12 +25,13 @@
def test_push_requirements(self):
self.executor_server.hold_jobs_in_build = True
- # Create a github change, add a change and emit a push event
A = self.fake_github.openFakePullRequest('org/project1', 'master', 'A')
- old_sha = A.head_sha
+ new_sha = A.head_sha
+ A.setMerged("merging A")
pevent = self.fake_github.getPushEvent(project='org/project1',
ref='refs/heads/master',
- old_rev=old_sha)
+ new_rev=new_sha)
+
self.fake_github.emitEvent(pevent)
self.waitUntilSettled()
@@ -43,7 +44,7 @@
# Make a gerrit change, and emit a ref-updated event
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
self.fake_gerrit.addEvent(B.getRefUpdatedEvent())
-
+ B.setMerged()
self.waitUntilSettled()
# All but one pipeline should be skipped, increasing builds by 1
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index e7cc93d..93367b9 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -1103,6 +1103,12 @@
def test_post(self):
"Test that post jobs run"
+ p = "review.example.com/org/project"
+ upstream = self.getUpstreamRepos([p])
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.setMerged()
+ A_commit = str(upstream[p].commit('master'))
+ self.log.debug("A commit: %s" % A_commit)
e = {
"type": "ref-updated",
@@ -1111,7 +1117,7 @@
},
"refUpdate": {
"oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
- "newRev": "d479a0bfcb34da57a31adb2a595c0cf687812543",
+ "newRev": A_commit,
"refName": "master",
"project": "org/project",
}
@@ -1156,7 +1162,7 @@
"refUpdate": {
"oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
"newRev": "0000000000000000000000000000000000000000",
- "refName": "master",
+ "refName": "testbranch",
"project": "org/project",
}
}
@@ -3080,6 +3086,12 @@
def test_client_enqueue_ref(self):
"Test that the RPC client can enqueue a ref"
+ p = "review.example.com/org/project"
+ upstream = self.getUpstreamRepos([p])
+ A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
+ A.setMerged()
+ A_commit = str(upstream[p].commit('master'))
+ self.log.debug("A commit: %s" % A_commit)
client = zuul.rpcclient.RPCClient('127.0.0.1',
self.gearman_server.port)
@@ -3091,7 +3103,7 @@
trigger='gerrit',
ref='master',
oldrev='90f173846e3af9154517b88543ffbd1691f31366',
- newrev='d479a0bfcb34da57a31adb2a595c0cf687812543')
+ newrev=A_commit)
self.waitUntilSettled()
job_names = [x.name for x in self.history]
self.assertEqual(len(self.history), 1)
diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py
index 6bf43d6..de72c69 100644
--- a/zuul/driver/gerrit/gerritconnection.py
+++ b/zuul/driver/gerrit/gerritconnection.py
@@ -274,10 +274,10 @@
self.gerrit_event_connector = None
self.source = driver.getSource(self)
- def getProject(self, name):
+ def getProject(self, name: str) -> Project:
return self.projects.get(name)
- def addProject(self, project):
+ def addProject(self, project: Project) -> None:
self.projects[project.name] = project
def maintainCache(self, relevant):
@@ -542,7 +542,8 @@
return True
return False
- def _waitForRefSha(self, project, ref, old_sha=''):
+ def _waitForRefSha(self, project: Project,
+ ref: str, old_sha: str='') -> bool:
# Wait for the ref to show up in the repo
start = time.time()
while time.time() - start < self.replication_timeout:
@@ -552,8 +553,8 @@
time.sleep(self.replication_retry_interval)
return False
- def getRefSha(self, project, ref):
- refs = {}
+ def getRefSha(self, project: Project, ref: str) -> str:
+ refs = {} # type: Dict[str, str]
try:
refs = self.getInfoRefs(project)
except:
@@ -598,14 +599,14 @@
return False
return True
- def getProjectOpenChanges(self, project):
+ def getProjectOpenChanges(self, project: Project) -> List[GerritChange]:
# This is a best-effort function in case Gerrit is unable to return
# a particular change. It happens.
query = "project:%s status:open" % (project.name,)
self.log.debug("Running query %s to get project open changes" %
(query,))
data = self.simpleQuery(query)
- changes = []
+ changes = [] # type: List[GerritChange]
for record in data:
try:
changes.append(
@@ -793,13 +794,13 @@
ret[ref] = revision
return ret
- def getGitUrl(self, project):
+ def getGitUrl(self, project: Project) -> str:
url = 'ssh://%s@%s:%s/%s' % (self.user, self.server, self.port,
project.name)
return url
- def _getGitwebUrl(self, project, sha=None):
- url = '%s/gitweb?p=%s.git' % (self.baseurl, project)
+ def _getGitwebUrl(self, project: Project, sha: str=None) -> str:
+ url = '%s/gitweb?p=%s.git' % (self.baseurl, project.name)
if sha:
url += ';a=commitdiff;h=' + sha
return url
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index b166111..8d23cb7 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -625,6 +625,7 @@
self.hostname)
self.merger_worker.registerFunction("merger:merge")
self.merger_worker.registerFunction("merger:cat")
+ self.merger_worker.registerFunction("merger:refstate")
def stop(self):
self.log.debug("Stopping")
@@ -721,6 +722,9 @@
elif job.name == 'merger:merge':
self.log.debug("Got merge job: %s" % job.unique)
self.merge(job)
+ elif job.name == 'merger:refstate':
+ self.log.debug("Got refstate job: %s" % job.unique)
+ self.refstate(job)
else:
self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail()
@@ -800,6 +804,14 @@
files=files)
job.sendWorkComplete(json.dumps(result))
+ def refstate(self, job):
+ args = json.loads(job.arguments)
+ with self.merger_lock:
+ success, repo_state = self.merger.getRepoState(args['items'])
+ result = dict(updated=success,
+ repo_state=repo_state)
+ job.sendWorkComplete(json.dumps(result))
+
def merge(self, job):
args = json.loads(job.arguments)
with self.merger_lock:
@@ -954,6 +966,10 @@
# a work complete result, don't run any jobs
return
+ state_items = [i for i in args['items'] if not i.get('number')]
+ if state_items:
+ merger.setRepoState(state_items, args['repo_state'])
+
for project in args['projects']:
repo = repos[project['canonical_name']]
# If this project is the Zuul project and this is a ref
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index dfb3238..8282f86 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -13,6 +13,7 @@
import logging
from zuul import exceptions
+from zuul import model
class DynamicChangeQueueContextManager(object):
@@ -483,20 +484,18 @@
def scheduleMerge(self, item, files=None, dirs=None):
build_set = item.current_build_set
- if not hasattr(item.change, 'branch'):
- self.log.debug("Change %s does not have an associated branch, "
- "not scheduling a merge job for item %s" %
- (item.change, item))
- build_set.merge_state = build_set.COMPLETE
- return True
-
self.log.debug("Scheduling merge for item %s (files: %s, dirs: %s)" %
(item, files, dirs))
build_set = item.current_build_set
build_set.merge_state = build_set.PENDING
- self.sched.merger.mergeChanges(build_set.merger_items,
- item.current_build_set, files, dirs,
- precedence=self.pipeline.precedence)
+ if isinstance(item.change, model.Change):
+ self.sched.merger.mergeChanges(build_set.merger_items,
+ item.current_build_set, files, dirs,
+ precedence=self.pipeline.precedence)
+ else:
+ self.sched.merger.getRepoState(build_set.merger_items,
+ item.current_build_set,
+ precedence=self.pipeline.precedence)
return False
def prepareItem(self, item):
@@ -675,12 +674,13 @@
build_set = event.build_set
item = build_set.item
build_set.merge_state = build_set.COMPLETE
+ build_set.repo_state = event.repo_state
if event.merged:
build_set.commit = event.commit
build_set.files.setFiles(event.files)
- build_set.repo_state = event.repo_state
elif event.updated:
- build_set.commit = item.change.newrev
+ build_set.commit = (item.change.newrev or
+ '0000000000000000000000000000000000000000')
if not build_set.commit:
self.log.info("Unable to merge change %s" % item.change)
item.setUnableToMerge()
diff --git a/zuul/manager/independent.py b/zuul/manager/independent.py
index 06c9a01..7b0a9f5 100644
--- a/zuul/manager/independent.py
+++ b/zuul/manager/independent.py
@@ -44,6 +44,9 @@
if hasattr(change, 'number'):
history = history or []
history.append(change.number)
+ else:
+ # Don't enqueue dependencies ahead of a non-change ref.
+ return True
ret = self.checkForChangesNeededBy(change, change_queue)
if ret in [True, False]:
diff --git a/zuul/merger/client.py b/zuul/merger/client.py
index dd9c8d5..5191a44 100644
--- a/zuul/merger/client.py
+++ b/zuul/merger/client.py
@@ -116,6 +116,11 @@
repo_state=repo_state)
self.submitJob('merger:merge', data, build_set, precedence)
+ def getRepoState(self, items, build_set,
+ precedence=zuul.model.PRECEDENCE_NORMAL):
+ data = dict(items=items)
+ self.submitJob('merger:refstate', data, build_set, precedence)
+
def getFiles(self, connection_name, project_name, branch, files, dirs=[],
precedence=zuul.model.PRECEDENCE_HIGH):
data = dict(connection=connection_name,
diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py
index c5d1f2a..ed98696 100644
--- a/zuul/merger/merger.py
+++ b/zuul/merger/merger.py
@@ -20,6 +20,8 @@
import zuul.model
+NULL_REF = '0000000000000000000000000000000000000000'
+
def reset_repo_to_head(repo):
# This lets us reset the repo even if there is a file in the root
@@ -178,8 +180,13 @@
self.setRef(path, hexsha, repo)
unseen.discard(path)
for path in unseen:
- self.log.debug("Delete reference %s", path)
- git.refs.SymbolicReference.delete(repo, ref.path)
+ self.deleteRef(path, repo)
+
+ def deleteRef(self, path, repo=None):
+ if repo is None:
+ repo = self.createRepoObject()
+ self.log.debug("Delete reference %s", path)
+ git.refs.SymbolicReference.delete(repo, path)
def checkout(self, ref):
repo = self.createRepoObject()
@@ -369,6 +376,16 @@
recent[key] = ref.object
project[ref.path] = ref.object.hexsha
+ def _alterRepoState(self, connection_name, project_name,
+ repo_state, path, hexsha):
+ projects = repo_state.setdefault(connection_name, {})
+ project = projects.setdefault(project_name, {})
+ if hexsha == NULL_REF:
+ if path in project:
+ del project[path]
+ else:
+ project[path] = hexsha
+
def _restoreRepoState(self, connection_name, project_name, repo,
repo_state):
projects = repo_state.get(connection_name, {})
@@ -470,12 +487,8 @@
if repo_state is None:
repo_state = {}
for item in items:
- if item.get("number") and item.get("patchset"):
- self.log.debug("Merging for change %s,%s." %
- (item["number"], item["patchset"]))
- elif item.get("newrev") and item.get("oldrev"):
- self.log.debug("Merging for rev %s with oldrev %s." %
- (item["newrev"], item["oldrev"]))
+ self.log.debug("Merging for change %s,%s" %
+ (item["number"], item["patchset"]))
commit = self._mergeItem(item, recent, repo_state)
if not commit:
return None
@@ -492,6 +505,49 @@
ret_recent[k] = v.hexsha
return commit.hexsha, read_files, repo_state, ret_recent
+ def setRepoState(self, items, repo_state):
+ # Sets the repo state for the items
+ seen = set()
+ for item in items:
+ repo = self.getRepo(item['connection'], item['project'])
+ key = (item['connection'], item['project'], item['branch'])
+
+ if key in seen:
+ continue
+
+ repo.reset()
+ self._restoreRepoState(item['connection'], item['project'], repo,
+ repo_state)
+
+ def getRepoState(self, items):
+ # Gets the repo state for items. Generally this will be
+ # called in any non-change pipeline. We will return the repo
+ # state for each item, but manipulated with any information in
+ # the item (eg, if it creates a ref, that will be in the repo
+ # state regardless of the actual state).
+ seen = set()
+ recent = {}
+ repo_state = {}
+ for item in items:
+ repo = self.getRepo(item['connection'], item['project'])
+ key = (item['connection'], item['project'], item['branch'])
+ if key not in seen:
+ try:
+ repo.reset()
+ except Exception:
+ self.log.exception("Unable to reset repo %s" % repo)
+ return (False, {})
+
+ self._saveRepoState(item['connection'], item['project'], repo,
+ repo_state, recent)
+
+ if item.get('newrev'):
+ # This is a ref update rather than a branch tip, so make sure
+ # our returned state includes this change.
+ self._alterRepoState(item['connection'], item['project'],
+ repo_state, item['ref'], item['newrev'])
+ return (True, repo_state)
+
def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
repo = self.getRepo(connection_name, project_name)
return repo.getFiles(files, dirs, branch=branch)
diff --git a/zuul/merger/server.py b/zuul/merger/server.py
index c342e1a..fc599c1 100644
--- a/zuul/merger/server.py
+++ b/zuul/merger/server.py
@@ -58,6 +58,7 @@
def register(self):
self.worker.registerFunction("merger:merge")
self.worker.registerFunction("merger:cat")
+ self.worker.registerFunction("merger:refstate")
def stop(self):
self.log.debug("Stopping")
@@ -80,6 +81,9 @@
elif job.name == 'merger:cat':
self.log.debug("Got cat job: %s" % job.unique)
self.cat(job)
+ elif job.name == 'merger:refstate':
+ self.log.debug("Got refstate job: %s" % job.unique)
+ self.refstate(job)
else:
self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail()
@@ -104,6 +108,14 @@
recent) = ret
job.sendWorkComplete(json.dumps(result))
+ def refstate(self, job):
+ args = json.loads(job.arguments)
+
+ success, repo_state = self.merger.getItemRepoState(args['items'])
+ result = dict(updated=success,
+ repo_state=repo_state)
+ job.sendWorkComplete(json.dumps(result))
+
def cat(self, job):
args = json.loads(job.arguments)
self.merger.updateRepo(args['connection'], args['project'])