Merge branch 'feature/migrate-import-repository-to-gitaly' into 'master'

Migrate importing repository to Gitaly

Closes gitaly#907

See merge request gitlab-org/gitlab-ce!16431

Author: Robert Speicher

Gemfile

@@ -403,7 +403,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.69.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.73.0', require: 'gitaly'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -284,7 +284,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly-proto (0.69.0)
+    gitaly-proto (0.73.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -1054,7 +1054,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly-proto (~> 0.69.0)
+  gitaly-proto (~> 0.73.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-markup (~> 1.6.2)

lib/gitlab/git/gitlab_projects.rb

@@ -44,29 +44,13 @@ def output
       # Import project via git clone --bare
       # URL must be publicly cloneable
       def import_project(source, timeout)
-        # Skip import if repo already exists
-        return false if File.exist?(repository_absolute_path)
-
-        masked_source = mask_password_in_url(source)
-
-        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
-        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
-
-        success = run_with_timeout(cmd, timeout, nil)
-
-        unless success
-          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
-          FileUtils.rm_rf(repository_absolute_path)
-          return false
+        Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled|
+          if is_enabled
+            gitaly_import_repository(source)
+          else
+            git_import_repository(source, timeout)
+          end
         end
-
-        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
-
-        # The project was imported successfully.
-        # Remove the origin URL since it may contain password.
-        remove_origin_in_repo
-
-        true
       end
 
       def fork_repository(new_shard_path, new_repository_relative_path)
@@ -231,6 +215,42 @@ def shard_name_from_shard_path(shard_path)
           raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'")
       end
 
+      def git_import_repository(source, timeout)
+        # Skip import if repo already exists
+        return false if File.exist?(repository_absolute_path)
+
+        masked_source = mask_password_in_url(source)
+
+        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
+        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
+
+        success = run_with_timeout(cmd, timeout, nil)
+
+        unless success
+          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
+          FileUtils.rm_rf(repository_absolute_path)
+          return false
+        end
+
+        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
+
+        # The project was imported successfully.
+        # Remove the origin URL since it may contain password.
+        remove_origin_in_repo
+
+        true
+      end
+
+      def gitaly_import_repository(source)
+        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
+
+        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
+        true
+      rescue GRPC::BadStatus => e
+        @output << e.message
+        false
+      end
+
       def git_fork_repository(new_shard_path, new_repository_relative_path)
         from_path = repository_absolute_path
         to_path = File.join(new_shard_path, new_repository_relative_path)

lib/gitlab/gitaly_client/repository_service.rb

@@ -100,6 +100,21 @@ def fork_repository(source_repository)
         )
       end
 
+      def import_repository(source)
+        request = Gitaly::CreateRepositoryFromURLRequest.new(
+          repository: @gitaly_repo,
+          url: source
+        )
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :create_repository_from_url,
+          request,
+          timeout: GitalyClient.default_timeout
+        )
+      end
+
       def rebase_in_progress?(rebase_id)
         request = Gitaly::IsRebaseInProgressRequest.new(
           repository: @gitaly_repo,

spec/lib/gitlab/git/gitlab_projects_spec.rb

@@ -158,39 +158,55 @@ def stub_tempfile(name, filename, opts = {})
 
     subject { gl_projects.import_project(import_url, timeout) }
 
-    context 'success import' do
-      it 'imports a repo' do
-        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+    shared_examples 'importing repository' do
+      context 'success import' do
+        it 'imports a repo' do
+          expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
 
-        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
-        expect(logger).to receive(:info).with(message)
+          is_expected.to be_truthy
 
-        is_expected.to be_truthy
+          expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
+        end
+      end
 
-        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_truthy
+      context 'already exists' do
+        it "doesn't import" do
+          FileUtils.mkdir_p(tmp_repo_path)
+
+          is_expected.to be_falsy
+        end
       end
     end
 
-    context 'already exists' do
-      it "doesn't import" do
-        FileUtils.mkdir_p(tmp_repo_path)
+    context 'when Gitaly import_repository feature is enabled' do
+      it_behaves_like 'importing repository'
+    end
+
+    context 'when Gitaly import_repository feature is disabled', :disable_gitaly do
+      describe 'logging' do
+        it 'imports a repo' do
+          message = "Importing project from <#{import_url}> to <#{tmp_repo_path}>."
+          expect(logger).to receive(:info).with(message)
 
-        is_expected.to be_falsy
+          subject
+        end
       end
-    end
 
-    context 'timeout' do
-      it 'does not import a repo' do
-        stub_spawn_timeout(cmd, timeout, nil)
+      context 'timeout' do
+        it 'does not import a repo' do
+          stub_spawn_timeout(cmd, timeout, nil)
 
-        message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
-        expect(logger).to receive(:error).with(message)
+          message = "Importing project from <#{import_url}> to <#{tmp_repo_path}> failed."
+          expect(logger).to receive(:error).with(message)
 
-        is_expected.to be_falsy
+          is_expected.to be_falsy
 
-        expect(gl_projects.output).to eq("Timed out\n")
-        expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+          expect(gl_projects.output).to eq("Timed out\n")
+          expect(File.exist?(File.join(tmp_repo_path, 'HEAD'))).to be_falsy
+        end
       end
+
+      it_behaves_like 'importing repository'
     end
   end