Skip to content

Commit 118c696

Browse files
author
epriestley
committed
Separate repository updates from the pull daemon
Summary: Ref T4605. Currently, the PullLocal daemon is responsible for two relatively distinct things: - scheduling repository updates; and - actually updating repositories. Move the "actually updating" part into a new `bin/repository update` command, which basically runs the pull, discover, refs and mirror commands. This will let the parent process focus on scheduling in a more understandable way and update multiple repositories at once. It also makes it easier to debug and understand update behavior since the non-scheduling pipeline can be run separately. Test Plan: - Ran `update --trace` on SVN, Mercurial and Git repos. - Ran PullLocal daemon for a while without issues. Reviewers: btrahan Reviewed By: btrahan Subscribers: epriestley Maniphest Tasks: T4605 Differential Revision: https://secure.phabricator.com/D8780
1 parent 2cf2117 commit 118c696

File tree

3 files changed

+198
-122
lines changed

3 files changed

+198
-122
lines changed

‎src/__phutil_library_map__.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1964,6 +1964,7 @@
19641964
'PhabricatorRepositoryManagementMirrorWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMirrorWorkflow.php',
19651965
'PhabricatorRepositoryManagementPullWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementPullWorkflow.php',
19661966
'PhabricatorRepositoryManagementRefsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php',
1967+
'PhabricatorRepositoryManagementUpdateWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php',
19671968
'PhabricatorRepositoryManagementWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementWorkflow.php',
19681969
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php',
19691970
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php',
@@ -4827,6 +4828,7 @@
48274828
'PhabricatorRepositoryManagementMirrorWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
48284829
'PhabricatorRepositoryManagementPullWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
48294830
'PhabricatorRepositoryManagementRefsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4831+
'PhabricatorRepositoryManagementUpdateWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
48304832
'PhabricatorRepositoryManagementWorkflow' => 'PhabricatorManagementWorkflow',
48314833
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
48324834
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',

‎src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php‎

Lines changed: 18 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@
2929
final class PhabricatorRepositoryPullLocalDaemon
3030
extends PhabricatorDaemon {
3131

32-
private $discoveryEngines = array();
33-
3432

3533
/* -( Pulling Repositories )----------------------------------------------- */
3634

@@ -122,49 +120,28 @@ public function run() {
122120
try {
123121
$this->log("Updating repository '{$callsign}'.");
124122

125-
id(new PhabricatorRepositoryPullEngine())
126-
->setRepository($repository)
127-
->pullRepository();
128-
129-
if (!$no_discovery) {
130-
// TODO: It would be nice to discover only if we pulled something,
131-
// but this isn't totally trivial. It's slightly more complicated
132-
// with hosted repositories, too.
133-
134-
$lock_name = get_class($this).':'.$callsign;
135-
$lock = PhabricatorGlobalLock::newLock($lock_name);
136-
$lock->lock();
137-
138-
try {
139-
$repository->writeStatusMessage(
140-
PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
141-
null);
142-
$this->discoverRepository($repository);
143-
$this->updateRepositoryRefs($repository);
144-
$this->mirrorRepository($repository);
145-
$repository->writeStatusMessage(
146-
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
147-
PhabricatorRepositoryStatusMessage::CODE_OKAY);
148-
} catch (Exception $ex) {
149-
$repository->writeStatusMessage(
150-
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
151-
PhabricatorRepositoryStatusMessage::CODE_ERROR,
152-
array(
153-
'message' => pht(
154-
'Error updating working copy: %s', $ex->getMessage()),
155-
));
156-
$lock->unlock();
157-
throw $ex;
158-
}
159-
160-
$lock->unlock();
123+
$bin_dir = dirname(phutil_get_library_root('phabricator')).'/bin';
124+
$flags = array();
125+
if ($no_discovery) {
126+
$flags[] = '--no-discovery';
127+
}
128+
129+
list($stdout, $stderr) = execx(
130+
'%s/repository update %Ls -- %s',
131+
$bin_dir,
132+
$flags,
133+
$callsign);
134+
135+
if (strlen($stderr)) {
136+
$stderr_msg = pht(
137+
'Unexpected output while updating the %s repository: %s',
138+
$callsign,
139+
$stderr);
140+
phlog($stderr_msg);
161141
}
162142

163143
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
164144
$retry_after[$id] = time() + $sleep_for;
165-
} catch (PhutilLockException $ex) {
166-
$retry_after[$id] = time() + $min_sleep;
167-
$this->log("Failed to acquire lock.");
168145
} catch (Exception $ex) {
169146
$retry_after[$id] = time() + $min_sleep;
170147

@@ -224,85 +201,4 @@ protected function loadRepositories(array $names) {
224201
return $repos;
225202
}
226203

227-
public function discoverRepository(PhabricatorRepository $repository) {
228-
$refs = $this->getDiscoveryEngine($repository)
229-
->discoverCommits();
230-
231-
$this->checkIfRepositoryIsFullyImported($repository);
232-
233-
return (bool)count($refs);
234-
}
235-
236-
private function mirrorRepository(PhabricatorRepository $repository) {
237-
try {
238-
id(new PhabricatorRepositoryMirrorEngine())
239-
->setRepository($repository)
240-
->pushToMirrors();
241-
} catch (Exception $ex) {
242-
// TODO: We should report these into the UI properly, but for
243-
// now just complain. These errors are much less severe than
244-
// pull errors.
245-
$proxy = new PhutilProxyException(
246-
pht(
247-
'Error while pushing "%s" repository to mirrors.',
248-
$repository->getCallsign()),
249-
$ex);
250-
phlog($proxy);
251-
}
252-
}
253-
254-
private function updateRepositoryRefs(PhabricatorRepository $repository) {
255-
id(new PhabricatorRepositoryRefEngine())
256-
->setRepository($repository)
257-
->updateRefs();
258-
}
259-
260-
private function getDiscoveryEngine(PhabricatorRepository $repository) {
261-
$id = $repository->getID();
262-
if (empty($this->discoveryEngines[$id])) {
263-
$engine = id(new PhabricatorRepositoryDiscoveryEngine())
264-
->setRepository($repository)
265-
->setVerbose($this->getVerbose());
266-
267-
$this->discoveryEngines[$id] = $engine;
268-
}
269-
return $this->discoveryEngines[$id];
270-
}
271-
272-
private function checkIfRepositoryIsFullyImported(
273-
PhabricatorRepository $repository) {
274-
275-
// Check if the repository has the "Importing" flag set. We want to clear
276-
// the flag if we can.
277-
$importing = $repository->getDetail('importing');
278-
if (!$importing) {
279-
// This repository isn't marked as "Importing", so we're done.
280-
return;
281-
}
282-
283-
// Look for any commit which hasn't imported.
284-
$unparsed_commit = queryfx_one(
285-
$repository->establishConnection('r'),
286-
'SELECT * FROM %T WHERE repositoryID = %d AND (importStatus & %d) != %d
287-
LIMIT 1',
288-
id(new PhabricatorRepositoryCommit())->getTableName(),
289-
$repository->getID(),
290-
PhabricatorRepositoryCommit::IMPORTED_ALL,
291-
PhabricatorRepositoryCommit::IMPORTED_ALL);
292-
if ($unparsed_commit) {
293-
// We found a commit which still needs to import, so we can't clear the
294-
// flag.
295-
return;
296-
}
297-
298-
// Clear the "importing" flag.
299-
$repository->openTransaction();
300-
$repository->beginReadLocking();
301-
$repository = $repository->reload();
302-
$repository->setDetail('importing', false);
303-
$repository->save();
304-
$repository->endReadLocking();
305-
$repository->saveTransaction();
306-
}
307-
308204
}

‎src/applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
<?php
2+
3+
final class PhabricatorRepositoryManagementUpdateWorkflow
4+
extends PhabricatorRepositoryManagementWorkflow {
5+
6+
private $verbose;
7+
8+
public function setVerbose($verbose) {
9+
$this->verbose = $verbose;
10+
return $this;
11+
}
12+
13+
public function getVerbose() {
14+
return $this->verbose;
15+
}
16+
17+
public function didConstruct() {
18+
$this
19+
->setName('update')
20+
->setExamples('**update** [options] __repository__')
21+
->setSynopsis(
22+
pht(
23+
'Update __repository__, named by callsign. '.
24+
'This performs the __pull__, __discover__, __ref__ and __mirror__ '.
25+
'operations and is primarily an internal workflow.'))
26+
->setArguments(
27+
array(
28+
array(
29+
'name' => 'verbose',
30+
'help' => 'Show additional debugging information.',
31+
),
32+
array(
33+
'name' => 'no-discovery',
34+
'help' => 'Do not perform discovery.',
35+
),
36+
array(
37+
'name' => 'repos',
38+
'wildcard' => true,
39+
),
40+
));
41+
}
42+
43+
public function execute(PhutilArgumentParser $args) {
44+
$this->setVerbose($args->getArg('verbose'));
45+
46+
$repos = $this->loadRepositories($args, 'repos');
47+
if (count($repos) !== 1) {
48+
throw new PhutilArgumentUsageException(
49+
pht('Specify exactly one repository to update, by callsign.'));
50+
}
51+
52+
$repository = head($repos);
53+
$callsign = $repository->getCallsign();
54+
55+
$no_discovery = $args->getArg('no-discovery');
56+
57+
id(new PhabricatorRepositoryPullEngine())
58+
->setRepository($repository)
59+
->setVerbose($this->getVerbose())
60+
->pullRepository();
61+
62+
if ($no_discovery) {
63+
return;
64+
}
65+
66+
// TODO: It would be nice to discover only if we pulled something, but this
67+
// isn't totally trivial. It's slightly more complicated with hosted
68+
// repositories, too.
69+
70+
$lock_name = get_class($this).':'.$callsign;
71+
$lock = PhabricatorGlobalLock::newLock($lock_name);
72+
73+
$lock->lock();
74+
75+
try {
76+
$repository->writeStatusMessage(
77+
PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
78+
null);
79+
80+
$this->discoverRepository($repository);
81+
82+
$this->checkIfRepositoryIsFullyImported($repository);
83+
84+
$this->updateRepositoryRefs($repository);
85+
86+
$this->mirrorRepository($repository);
87+
88+
$repository->writeStatusMessage(
89+
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
90+
PhabricatorRepositoryStatusMessage::CODE_OKAY);
91+
} catch (Exception $ex) {
92+
$repository->writeStatusMessage(
93+
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
94+
PhabricatorRepositoryStatusMessage::CODE_ERROR,
95+
array(
96+
'message' => pht(
97+
'Error updating working copy: %s', $ex->getMessage()),
98+
));
99+
100+
$lock->unlock();
101+
throw $ex;
102+
}
103+
104+
$lock->unlock();
105+
106+
return 0;
107+
}
108+
109+
private function discoverRepository(PhabricatorRepository $repository) {
110+
$refs = id(new PhabricatorRepositoryDiscoveryEngine())
111+
->setRepository($repository)
112+
->setVerbose($this->getVerbose())
113+
->discoverCommits();
114+
115+
return (bool)count($refs);
116+
}
117+
118+
private function mirrorRepository(PhabricatorRepository $repository) {
119+
try {
120+
id(new PhabricatorRepositoryMirrorEngine())
121+
->setRepository($repository)
122+
->pushToMirrors();
123+
} catch (Exception $ex) {
124+
// TODO: We should report these into the UI properly, but for now just
125+
// complain. These errors are much less severe than pull errors.
126+
$proxy = new PhutilProxyException(
127+
pht(
128+
'Error while pushing "%s" repository to mirrors.',
129+
$repository->getCallsign()),
130+
$ex);
131+
phlog($proxy);
132+
}
133+
}
134+
135+
private function updateRepositoryRefs(PhabricatorRepository $repository) {
136+
id(new PhabricatorRepositoryRefEngine())
137+
->setRepository($repository)
138+
->updateRefs();
139+
}
140+
141+
private function checkIfRepositoryIsFullyImported(
142+
PhabricatorRepository $repository) {
143+
144+
// Check if the repository has the "Importing" flag set. We want to clear
145+
// the flag if we can.
146+
$importing = $repository->getDetail('importing');
147+
if (!$importing) {
148+
// This repository isn't marked as "Importing", so we're done.
149+
return;
150+
}
151+
152+
// Look for any commit which hasn't imported.
153+
$unparsed_commit = queryfx_one(
154+
$repository->establishConnection('r'),
155+
'SELECT * FROM %T WHERE repositoryID = %d AND (importStatus & %d) != %d
156+
LIMIT 1',
157+
id(new PhabricatorRepositoryCommit())->getTableName(),
158+
$repository->getID(),
159+
PhabricatorRepositoryCommit::IMPORTED_ALL,
160+
PhabricatorRepositoryCommit::IMPORTED_ALL);
161+
if ($unparsed_commit) {
162+
// We found a commit which still needs to import, so we can't clear the
163+
// flag.
164+
return;
165+
}
166+
167+
// Clear the "importing" flag.
168+
$repository->openTransaction();
169+
$repository->beginReadLocking();
170+
$repository = $repository->reload();
171+
$repository->setDetail('importing', false);
172+
$repository->save();
173+
$repository->endReadLocking();
174+
$repository->saveTransaction();
175+
}
176+
177+
178+
}

0 commit comments

Comments
 (0)