Run one daemon to pull all working copies, not one daemon per working copy

Summary:
Allow the pull daemon to take a list of repositories. By default, pull all repositories.

Make some effort to respect pull frequencies, although we'll necessarily suffer a bit if running with only one process.

NOTE: We still launch one discovery daemon per working copy, so this only cuts the daemon count in half.

Test Plan:
  - Ran `phd debug pulllocal`, verified behavior.
  - Ran `pull.php P MTEST SVNTEST --trace`, verified it pulled the repos and ran the right commands.
  - Ran `phd repository-launch-master`, verified the right daemons launched, checked daemon console.
  - Ran `phd repository-launch-readonly`, verified the right daemon launched, checked daemon console.

Reviewers: btrahan, csilvers, davidreuss

Reviewed By: csilvers

CC: aran

Differential Revision: https://secure.phabricator.com/D2418

Author: epriestley

scripts/daemon/phabricator_daemon_launcher.php

@@ -2,7 +2,7 @@
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -49,35 +49,30 @@ function must_have_extension($ext) {
     exit($err);
 
   case 'repository-launch-readonly':
-    $need_launch = phd_load_tracked_repositories_of_type('git');
+    $need_launch = phd_load_tracked_repositories();
     if (!$need_launch) {
       echo "There are no repositories with tracking enabled.\n";
-    } else {
-      will_launch($control);
-
-      foreach ($need_launch as $repository) {
-        $name = $repository->getName();
-        $callsign = $repository->getCallsign();
-        $desc = "'{$name}' ({$callsign})";
-        $phid = $repository->getPHID();
-
-        echo "Launching 'git fetch' daemon on the {$desc} repository...\n";
-        $control->launchDaemon(
-          'PhabricatorRepositoryGitFetchDaemon',
-          array(
-            $phid,
-          ));
-      }
+      exit(0);
     }
+
+    will_launch($control);
+    $control->launchDaemon(
+      'PhabricatorRepositoryPullLocalDaemon',
+      array());
     break;
 
   case 'repository-launch-master':
     $need_launch = phd_load_tracked_repositories();
     if (!$need_launch) {
       echo "There are no repositories with tracking enabled.\n";
+      exit(1);
     } else {
       will_launch($control);
 
+      $control->launchDaemon(
+        'PhabricatorRepositoryPullLocalDaemon',
+        array());
+
       foreach ($need_launch as $repository) {
         $name = $repository->getName();
         $callsign = $repository->getCallsign();
@@ -86,12 +81,6 @@ function must_have_extension($ext) {
 
         switch ($repository->getVersionControlSystem()) {
           case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
-            echo "Launching 'git fetch' daemon on the {$desc} repository...\n";
-            $control->launchDaemon(
-              'PhabricatorRepositoryGitFetchDaemon',
-              array(
-                $phid,
-              ));
             echo "Launching discovery daemon on the {$desc} repository...\n";
             $control->launchDaemon(
               'PhabricatorRepositoryGitCommitDiscoveryDaemon',
@@ -108,12 +97,6 @@ function must_have_extension($ext) {
               ));
             break;
           case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
-            echo "Launching 'hg pull' daemon on the {$desc} repository...\n";
-            $control->launchDaemon(
-              'PhabricatorRepositoryMercurialPullDaemon',
-              array(
-                $phid,
-              ));
             echo "Launching discovery daemon on the {$desc} repository...\n";
             $control->launchDaemon(
               'PhabricatorRepositoryMercurialCommitDiscoveryDaemon',
@@ -224,18 +207,6 @@ function must_have_extension($ext) {
     exit($err);
 }
 
-function phd_load_tracked_repositories_of_type($type) {
-  $repositories = phd_load_tracked_repositories();
-
-  foreach ($repositories as $key => $repository) {
-    if ($repository->getVersionControlSystem() != $type) {
-      unset($repositories[$key]);
-    }
-  }
-
-  return $repositories;
-}
-
 function phd_load_tracked_repositories() {
   phutil_require_module(
     'phabricator',

scripts/repository/pull.php

@@ -0,0 +1,51 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline('manually pull working copies');
+$args->setSynopsis(<<<EOHELP
+**pull.php** [__options__] __repository-callsign-or-phid ...__
+    Manually pull/fetch working copies for the named repositories.
+EOHELP
+);
+$args->parseStandardArguments();
+$args->parse(
+  array(
+    array(
+      'name'      => 'repositories',
+      'wildcard'  => true,
+    ),
+  ));
+
+$repo_names = $args->getArg('repositories');
+if (!$repo_names) {
+  echo "Specify one or more repositories to pull, by callsign or PHID.\n";
+  exit(1);
+}
+
+$repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names);
+foreach ($repos as $repo) {
+  $callsign = $repo->getCallsign();
+  echo "Pulling '{$callsign}'...\n";
+  PhabricatorRepositoryPullLocalDaemon::pullRepository($repo);
+}
+echo "Done.\n";

src/__phutil_library_map__.php

@@ -857,12 +857,10 @@
     'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/git',
     'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'applications/repository/daemon/commitdiscovery/git/__tests__',
     'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/git',
-    'PhabricatorRepositoryGitFetchDaemon' => 'applications/repository/daemon/gitfetch',
     'PhabricatorRepositoryListController' => 'applications/repository/controller/list',
     'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial',
     'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial',
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial',
-    'PhabricatorRepositoryMercurialPullDaemon' => 'applications/repository/daemon/mercurialpull',
     'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/pulllocal',
     'PhabricatorRepositoryShortcut' => 'applications/repository/storage/shortcut',
     'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/svn',
@@ -1752,13 +1750,11 @@
     'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
     'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'PhabricatorTestCase',
     'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
-    'PhabricatorRepositoryGitFetchDaemon' => 'PhabricatorRepositoryPullLocalDaemon',
     'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
     'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
-    'PhabricatorRepositoryMercurialPullDaemon' => 'PhabricatorRepositoryPullLocalDaemon',
-    'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorRepositoryDaemon',
+    'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
     'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
     'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',