1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
|
# Copyright (C) 2021 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
import os
import pickle
import shutil
from pathlib import Path
from time import sleep
import json
import requests
import git.exc
from git import Git
from git import Repo as GitRepo, exc
from tools import Proposal
from .repo import Repo
class ProposalEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Proposal):
return obj.__dict__
elif isinstance(obj, set):
return list(obj)
return json.JSONEncoder.default(self, obj)
# Note: pushing of state to personal gerrit branch is disabled for now due to unexplained failures.
def fetch_and_checkout(config, repo):
"""Try to fetch the remote ref in the personal gerrit branch for
the running user."""
g = Git(repo.working_tree_dir)
try:
g.fetch(['origin', config._state_ref])
g.checkout('FETCH_HEAD')
except git.exc.GitCommandError as e:
if "couldn't find remote ref refs/personal" in e.stderr:
pass
else:
print(e)
del g
def check_create_local_repo(config) -> GitRepo:
"""Create a local repo for saving state and push it to
the user's personal ref. Checkout any existing version
on the user's personal remote, or create a new commit"""
path = Path(config.cwd, "_state")
if not path.exists():
os.mkdir(path)
try:
repo = GitRepo(path)
if "origin" not in [r.name for r in repo.remotes] and config._state_ref:
repo.create_remote('origin',
f"ssh://{config.GERRIT_HOST[8:]}/{config.GERRIT_STATE_PATH}")
except exc.InvalidGitRepositoryError:
repo = GitRepo.init(path)
if config._state_ref:
repo.create_remote('origin',
f"ssh://{config.GERRIT_HOST[8:]}/{config.GERRIT_STATE_PATH}")
# fetch_and_checkout(config, repo)
state_path = Path(repo.working_tree_dir, "state.bin")
if not state_path.exists():
with open(state_path, 'wb') as state_file:
pickle.dump({}, state_file)
repo.index.add('state.bin')
repo.index.commit("Empty state")
if config._state_ref:
pass
# repo.remotes.origin.push(['-f', f"HEAD:{config._state_ref}"])
if not config._state_ref:
print("\nWARN: Unable to create git remote for state!\n"
"WARN: State will only be saved locally to _state/state.bin.\n"
"INFO: Please configure an ssh user in ~/.ssh/config for your gerrit host\n"
"INFO: as set by 'GERRIT_HOST' in config.yaml in order to save state in gerrit.\n")
return repo
def load_updates_state(config) -> dict[str, Repo]:
"""Load previous state and apply retention policy if not simulating a run."""
if config.args.no_state:
print("Running in no-state mode! No state loaded, and progress will not be saved on exit!")
return {}
print("\nLoading saved update data from codereview...")
if config._state_ref:
pass
# fetch_and_checkout(config, config.state_repo)
state_path = Path(config.state_repo.working_tree_dir, "state.bin")
if not state_path.exists():
with open(state_path, 'wb') as state_file:
pickle.dump(dict(), state_file)
state_data = {}
with open(state_path, mode='rb') as state_file:
state_data = pickle.load(state_file)
print("Done loading state data!")
if state_data.get(config.args.branch):
return state_data[config.args.branch]
else:
return {}
def update_state_data(old_state: dict[str, Repo], new_data: dict[str, Repo]) -> dict[
str, Repo]:
"""Merge two update set dicts"""
updated = old_state
for key in new_data.keys():
if old_state.get(key):
updated[key].merge(new_data[key])
else:
updated[key] = new_data[key]
return updated
def save_updates_state(config, _clear_state: bool = False,
prune_and_keep: str = None) -> None:
"""Save updates to the state file"""
if config.args.no_state:
print("Running in no-state mode. Not saving state!")
return
if not config.args.simulate:
if _clear_state:
clear_state(config)
return
print("Saving update state data to codereview...")
state_path = Path(config.state_repo.working_tree_dir, "state.bin")
data: dict[str, dict[str, Repo]] = {}
with open(state_path, 'rb') as state_file:
data = pickle.load(state_file)
if prune_and_keep:
keep_branches = prune_and_keep.split(",")
print(f"Found branches: {data.keys()}")
print(f"Retaining branches: {keep_branches}")
for branch in list(data.keys()):
if branch not in keep_branches:
del data[branch]
else:
data[config.args.branch] = config.state_data
with open(state_path, 'wb') as state_file:
pickle.dump(data, state_file)
config.state_repo.index.add("state.bin")
config.state_repo.index.commit("Update state")
if config._state_ref:
# config.state_repo.remotes.origin.push(['-f', f"HEAD:{config._state_ref}"])
print("Done saving state.")
def clear_state(config, prune_and_keep: str = None) -> None:
"""Clear state data. Currently operating branch is cleared if not pruning!
Parameters:
prune_and_keep: str: List of branches to keep.
Prune all others. Set to "ALL" clears all branches.
"""
def clear_one():
print(f"Clearing state and resetting updates for {config.args.branch}...")
if config.args.branch:
config.state_data = {}
save_updates_state(config)
print(f"Clearing branch state for {config.args.branch}")
return
def clear_all():
if config._state_ref:
try:
# config.state_repo.remotes.origin.push(['-f', f":{config._state_ref}"])
print("Cleared remote state on codereview...")
except git.exc.GitCommandError:
print(
"WARN: Failed to push an empty commit, probably because the state is already clear.")
del config.state_repo # Need to tear down the instance of PyGit to close the file handle.
sleep(5) # workaround for sometimes slow closing of git handles.
else:
print("\nWARN: No state remote ref set! Only deleting local state.bin file.\n"
"WARN: Run this script again with --reset after configuring an ssh user\n"
"WARN: in ~/.ssh/config for your gerrit host as set by 'GERRIT_HOST' in config.yaml.\n"
"WARN: If a remote state exists next time this script is run, it will likely\n"
"WARN: cause unexpected behavior!")
shutil.rmtree(Path(config.cwd, "_state"), onerror=_unlink_file)
print("Deleted local state files.")
def clear_some():
save_updates_state(config, prune_and_keep=prune_and_keep)
if prune_and_keep:
print("Pruning state data.")
if prune_and_keep == "ALL":
clear_all()
else:
clear_some()
else:
clear_one()
def _unlink_file(function, path, excinfo):
"""In the case that shutil.rmtree fails on a file."""
os.unlink(path)
def print_state_json(config):
state_json = json.dumps([repo.__dict__ for repo in config.state_data.values()], cls=ProposalEncoder)
print("=+=+=+ JSON DUMP +=+=+=")
print(state_json)
print("=+=+=+ END JSON DUMP +=+=+=")
def web_store_state(config):
"""Store or update the state in the database."""
if config.args.simulate:
print("Running in simulate mode. Skipping state storage.")
return
job_name = os.getenv("JOB_NAME")
if not job_name:
print("Not running in Jenkins. Skipping state storage.")
return
state_json = json.dumps([repo.__dict__ for repo in config.state_data.values()], cls=ProposalEncoder)
url = f"{config.SUBMODULE_STATUS_URL}/setDependencyStatus"
print(f"Storing state in database at {url}")
r = requests.post(url,
headers={
"Authorization": "Bearer " + config.SUBMODULE_TOKEN,
"Content-Type": "application/json"
},
json= {
"job_name": job_name,
"state_json": state_json
},
timeout=30
)
print(f"State storage response: {r.status_code}: {r.text}")
|
