Merge pull request swiftlang#67263 from cachemeifyoucan/eng/PR-explicit-module-interface-check

[ExplicitModule] Allow typecheck-module-from-interface using explicit module

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -105,7 +105,7 @@ WARNING(warning_cannot_find_locale_file,none,
 WARNING(warning_cannot_multithread_batch_mode,none,
       "ignoring -num-threads argument; cannot multithread batch mode", ())
 ERROR(error_cannot_explicit_interface_build_in_mode,none,
-      "'-explicit-interface-module-build' only supported when building a module from interface ('-compile-module-from-interface')'", ())
+      "'-explicit-interface-module-build' only supported when building a module from interface ('-compile-module-from-interface' or '-typecheck-module-from-interface')'", ())
 ERROR(error_cannot_direct_cc1_pcm_build_in_mode,none,
       "'-direct-clang-cc1-module-build' only supported when building a PCM ('-emit-pcm')'", ())
 ERROR(error_unsupported_option_argument,none,

include/swift/Frontend/FrontendOptions.h

@@ -141,6 +141,9 @@ class FrontendOptions {
   /// Clang Include Trees.
   std::vector<std::string> ClangIncludeTrees;
 
+  /// CacheKey for input file.
+  std::string InputFileKey;
+
   /// Number of retry opening an input file if the previous opening returns
   /// bad file descriptor error.
   unsigned BadFileDescriptorRetryCount = 0;

include/swift/Option/FrontendOptions.td

@@ -1225,6 +1225,8 @@ def always_compile_output_files :
 def allow_unstable_cache_key_for_testing: Flag<["-"], "allow-unstable-cache-key-for-testing">,
   HelpText<"Allow compilation caching with unstable inputs for testing purpose">;
 
+def input_file_key : Separate<["-"], "input-file-key">,
+  HelpText<"Cache Key for input file">;
 def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
   HelpText<"Cache Key for bridging header pch">;
 

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -249,6 +249,33 @@ bool ArgsToFrontendOptionsConverter::convert(
   if (checkBuildFromInterfaceOnlyOptions())
     return true;
 
+  Opts.DeterministicCheck = Args.hasArg(OPT_enable_deterministic_check);
+  Opts.EnableCaching = Args.hasArg(OPT_cache_compile_job);
+  Opts.EnableCachingRemarks = Args.hasArg(OPT_cache_remarks);
+  Opts.CacheSkipReplay = Args.hasArg(OPT_cache_disable_replay);
+  Opts.CASOpts.CASPath =
+      Args.getLastArgValue(OPT_cas_path, llvm::cas::getDefaultOnDiskCASPath());
+  Opts.CASOpts.PluginPath = Args.getLastArgValue(OPT_cas_plugin_path);
+  for (StringRef Opt : Args.getAllArgValues(OPT_cas_plugin_option)) {
+    StringRef Name, Value;
+    std::tie(Name, Value) = Opt.split('=');
+    Opts.CASOpts.PluginOptions.emplace_back(std::string(Name),
+                                            std::string(Value));
+  }
+
+  Opts.CASFSRootIDs = Args.getAllArgValues(OPT_cas_fs);
+  Opts.ClangIncludeTrees = Args.getAllArgValues(OPT_clang_include_tree_root);
+  Opts.InputFileKey = Args.getLastArgValue(OPT_input_file_key);
+
+  if (Opts.EnableCaching && Opts.CASFSRootIDs.empty() &&
+      Opts.ClangIncludeTrees.empty() &&
+      FrontendOptions::supportCompilationCaching(Opts.RequestedAction)) {
+    if (!Args.hasArg(OPT_allow_unstable_cache_key_for_testing)) {
+        Diags.diagnose(SourceLoc(), diag::error_caching_no_cas_fs);
+        return true;
+    }
+  }
+
   if (FrontendOptions::doesActionGenerateIR(Opts.RequestedAction)) {
     if (Args.hasArg(OPT_experimental_skip_non_inlinable_function_bodies) ||
         Args.hasArg(OPT_experimental_skip_all_function_bodies) ||
@@ -346,36 +373,10 @@ bool ArgsToFrontendOptionsConverter::convert(
     Opts.serializedPathObfuscator.addMapping(SplitMap.first, SplitMap.second);
   }
   Opts.emptyABIDescriptor = Args.hasArg(OPT_empty_abi_descriptor);
-  Opts.DeterministicCheck = Args.hasArg(OPT_enable_deterministic_check);
   for (auto A : Args.getAllArgValues(options::OPT_block_list_file)) {
     Opts.BlocklistConfigFilePaths.push_back(A);
   }
 
-  Opts.EnableCaching = Args.hasArg(OPT_cache_compile_job);
-  Opts.EnableCachingRemarks = Args.hasArg(OPT_cache_remarks);
-  Opts.CacheSkipReplay = Args.hasArg(OPT_cache_disable_replay);
-  Opts.CASOpts.CASPath =
-      Args.getLastArgValue(OPT_cas_path, llvm::cas::getDefaultOnDiskCASPath());
-  Opts.CASOpts.PluginPath = Args.getLastArgValue(OPT_cas_plugin_path);
-  for (StringRef Opt : Args.getAllArgValues(OPT_cas_plugin_option)) {
-    StringRef Name, Value;
-    std::tie(Name, Value) = Opt.split('=');
-    Opts.CASOpts.PluginOptions.emplace_back(std::string(Name),
-                                            std::string(Value));
-  }
-
-  Opts.CASFSRootIDs = Args.getAllArgValues(OPT_cas_fs);
-  Opts.ClangIncludeTrees = Args.getAllArgValues(OPT_clang_include_tree_root);
-
-  if (Opts.EnableCaching && Opts.CASFSRootIDs.empty() &&
-      Opts.ClangIncludeTrees.empty() &&
-      FrontendOptions::supportCompilationCaching(Opts.RequestedAction)) {
-    if (!Args.hasArg(OPT_allow_unstable_cache_key_for_testing)) {
-        Diags.diagnose(SourceLoc(), diag::error_caching_no_cas_fs);
-        return true;
-    }
-  }
-
   return false;
 }
 
@@ -706,7 +707,10 @@ bool ArgsToFrontendOptionsConverter::
 
 bool ArgsToFrontendOptionsConverter::checkBuildFromInterfaceOnlyOptions()
     const {
-  if (Opts.RequestedAction != FrontendOptions::ActionType::CompileModuleFromInterface &&
+  if (Opts.RequestedAction !=
+          FrontendOptions::ActionType::CompileModuleFromInterface &&
+      Opts.RequestedAction !=
+          FrontendOptions::ActionType::TypecheckModuleFromInterface &&
       Opts.ExplicitInterfaceBuild) {
     Diags.diagnose(SourceLoc(),
                    diag::error_cannot_explicit_interface_build_in_mode);

lib/Frontend/Frontend.cpp

@@ -520,38 +520,44 @@ bool CompilerInstance::setup(const CompilerInvocation &Invoke,
 }
 
 bool CompilerInstance::setUpVirtualFileSystemOverlays() {
-  if (Invocation.getFrontendOptions().EnableCaching &&
-      (!Invocation.getFrontendOptions().CASFSRootIDs.empty() ||
-       !Invocation.getFrontendOptions().ClangIncludeTrees.empty())) {
-    // Set up CASFS as BaseFS.
+  if (Invocation.getFrontendOptions().EnableCaching) {
     const auto &Opts = getInvocation().getFrontendOptions();
-    auto FS =
-        createCASFileSystem(*CAS, Opts.CASFSRootIDs, Opts.ClangIncludeTrees);
-    if (!FS) {
-      Diagnostics.diagnose(SourceLoc(), diag::error_cas,
-                           toString(FS.takeError()));
-      return true;
+    if (!Invocation.getFrontendOptions().CASFSRootIDs.empty() ||
+        !Invocation.getFrontendOptions().ClangIncludeTrees.empty()) {
+      // Set up CASFS as BaseFS.
+      auto FS =
+          createCASFileSystem(*CAS, Opts.CASFSRootIDs, Opts.ClangIncludeTrees);
+      if (!FS) {
+        Diagnostics.diagnose(SourceLoc(), diag::error_cas,
+                             toString(FS.takeError()));
+        return true;
+      }
+      SourceMgr.setFileSystem(std::move(*FS));
+    }
+
+    // If we need to load any files from CAS, try load it now and overlay it.
+    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> MemFS =
+        new llvm::vfs::InMemoryFileSystem();
+    const auto &ClangOpts = getInvocation().getClangImporterOptions();
+
+    if (!ClangOpts.BridgingHeaderPCHCacheKey.empty()) {
+      if (auto loadedBuffer = loadCachedCompileResultFromCacheKey(
+              getObjectStore(), getActionCache(), Diagnostics,
+              ClangOpts.BridgingHeaderPCHCacheKey, ClangOpts.BridgingHeader))
+        MemFS->addFile(Invocation.getClangImporterOptions().BridgingHeader, 0,
+                       std::move(loadedBuffer));
     }
-    SourceMgr.setFileSystem(std::move(*FS));
-  }
-
-  // If we have a bridging header cache key, try load it now and overlay it.
-  if (!Invocation.getClangImporterOptions().BridgingHeaderPCHCacheKey.empty() &&
-      Invocation.getFrontendOptions().EnableCaching) {
-    auto loadedBridgingBuffer = loadCachedCompileResultFromCacheKey(
-        getObjectStore(), getActionCache(), Diagnostics,
-        Invocation.getClangImporterOptions().BridgingHeaderPCHCacheKey,
-        Invocation.getClangImporterOptions().BridgingHeader);
-    if (loadedBridgingBuffer) {
-      llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> PCHFS =
-          new llvm::vfs::InMemoryFileSystem();
-      PCHFS->addFile(Invocation.getClangImporterOptions().BridgingHeader, 0,
-                     std::move(loadedBridgingBuffer));
-      llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayVFS =
-          new llvm::vfs::OverlayFileSystem(SourceMgr.getFileSystem());
-      OverlayVFS->pushOverlay(PCHFS);
-      SourceMgr.setFileSystem(std::move(OverlayVFS));
+    if (!Opts.InputFileKey.empty()) {
+      auto InputPath = Opts.InputsAndOutputs.getFilenameOfFirstInput();
+      if (auto loadedBuffer = loadCachedCompileResultFromCacheKey(
+              getObjectStore(), getActionCache(), Diagnostics,
+              Opts.InputFileKey, InputPath))
+        MemFS->addFile(InputPath, 0, std::move(loadedBuffer));
     }
+    llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayVFS =
+        new llvm::vfs::OverlayFileSystem(SourceMgr.getFileSystem());
+    OverlayVFS->pushOverlay(MemFS);
+    SourceMgr.setFileSystem(std::move(OverlayVFS));
   }
 
   auto ExpectedOverlay =

lib/Frontend/FrontendOptions.cpp

@@ -229,7 +229,6 @@ bool FrontendOptions::supportCompilationCaching(ActionType action) {
   case ActionType::DumpInterfaceHash:
   case ActionType::EmitImportedModules:
   case ActionType::ScanDependencies:
-  case ActionType::TypecheckModuleFromInterface:
   case ActionType::ResolveImports:
   case ActionType::Typecheck:
   case ActionType::DumpAST:
@@ -241,6 +240,7 @@ bool FrontendOptions::supportCompilationCaching(ActionType action) {
   case ActionType::Immediate:
   case ActionType::DumpTypeInfo:
     return false;
+  case ActionType::TypecheckModuleFromInterface:
   case ActionType::CompileModuleFromInterface:
   case ActionType::EmitPCH:
   case ActionType::EmitPCM:
@@ -769,14 +769,14 @@ bool FrontendOptions::doesActionProduceOutput(ActionType action) {
   case ActionType::EmitImportedModules:
   case ActionType::MergeModules:
   case ActionType::CompileModuleFromInterface:
-  case ActionType::TypecheckModuleFromInterface:
   case ActionType::DumpTypeInfo:
   case ActionType::EmitPCM:
   case ActionType::DumpPCM:
   case ActionType::ScanDependencies:
   case ActionType::PrintFeature:
     return true;
 
+  case ActionType::TypecheckModuleFromInterface:
   case ActionType::NoneAction:
   case ActionType::Immediate:
   case ActionType::REPL:
@@ -800,12 +800,12 @@ bool FrontendOptions::doesActionProduceTextualOutput(ActionType action) {
   case ActionType::Immediate:
   case ActionType::REPL:
   case ActionType::EmitPCM:
+  case ActionType::TypecheckModuleFromInterface:
     return false;
 
   case ActionType::Parse:
   case ActionType::ResolveImports:
   case ActionType::Typecheck:
-  case ActionType::TypecheckModuleFromInterface:
   case ActionType::DumpParse:
   case ActionType::DumpInterfaceHash:
   case ActionType::DumpAST:

lib/Frontend/Serialization.cpp

@@ -65,8 +65,10 @@ void swift::serializeToBuffers(
     std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
     const SILModule *M) {
+  // Serialization output is disabled.
+  if (options.OutputPath.empty())
+    return;
 
-  assert(!options.OutputPath.empty());
   {
     FrontendStatsTracer tracer(getContext(DC).Stats,
                                "Serialization, swiftmodule, to buffer");

test/CAS/Inputs/ExtractOutputKey.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+#
+# Usage: ExtractOutputKey.py file.json OutputPath
+
+import json
+import sys
+
+input_json = sys.argv[1]
+output_path = sys.argv[2]
+
+
+with open(input_json, 'r') as file:
+    outputs = json.load(file)
+    for output in outputs:
+        if output['OutputPath'] != output_path:
+            continue
+        print(output['CacheKey'])

test/CAS/cas-explicit-module-map.swift

@@ -110,13 +110,33 @@
 
 // RUN: %target-swift-frontend -emit-module -emit-module-path %t/Foo.swiftmodule -disable-implicit-swift-modules -module-cache-path %t.module-cache -explicit-swift-module-map-file @%t/map.casid -Rmodule-loading -Xcc -Rmodule-import %s -cache-compile-job -cas-path %t/cas -allow-unstable-cache-key-for-testing 2>&1 | %FileCheck %s
 
+// RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/Foo.swiftinterface -disable-implicit-swift-modules -module-cache-path %t.module-cache -explicit-swift-module-map-file @%t/map.casid %s -cache-compile-job -cas-path %t/cas -allow-unstable-cache-key-for-testing -swift-version 5 -enable-library-evolution
+// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-output-keys -- \
+// RUN:   %target-swift-frontend -typecheck -emit-module-interface-path %t/Foo.swiftinterface -disable-implicit-swift-modules \
+// RUN:   -module-cache-path %t.module-cache -explicit-swift-module-map-file @%t/map.casid %s -cache-compile-job \
+// RUN:   -cas-path %t/cas -allow-unstable-cache-key-for-testing  -swift-version 5 -enable-library-evolution > %t/keys.json
+
+// RUN: %S/Inputs/ExtractOutputKey.py %t/keys.json %t/Foo.swiftinterface > %t/interface.casid
+// RUN: %target-swift-frontend -typecheck-module-from-interface %t/Foo.swiftinterface -disable-implicit-swift-modules \
+// RUN:   -module-cache-path %t.module-cache -explicit-swift-module-map-file @%t/map.casid  \
+// RUN:   -cache-compile-job -cas-path %t/cas -allow-unstable-cache-key-for-testing -swift-version 5 -enable-library-evolution \
+// RUN:   -explicit-interface-module-build -Rcache-compile-job 2>&1 | %FileCheck %s --check-prefix=VERIFY-OUTPUT --check-prefix=CACHE-MISS
+// RUN: %target-swift-frontend -typecheck-module-from-interface %t/Foo.swiftinterface -disable-implicit-swift-modules \
+// RUN:   -module-cache-path %t.module-cache -explicit-swift-module-map-file @%t/map.casid  \
+// RUN:   -cache-compile-job -cas-path %t/cas -allow-unstable-cache-key-for-testing -swift-version 5 -enable-library-evolution \
+// RUN:   -explicit-interface-module-build -Rcache-compile-job 2>&1 | %FileCheck %s --check-prefix=VERIFY-OUTPUT --check-prefix=CACHE-HIT
+
 // CHECK-DAG: loaded module 'A'
 // CHECK-DAG: loaded module 'B'
 // CHECK-DAG: loaded module 'Swift'
 // CHECK-DAG: loaded module '_StringProcessing'
 // CHECK-DAG: loaded module '_Concurrency'
 // CHECK-DAG: loaded module 'SwiftOnoneSupport'
 
+// CACHE-MISS: remark: cache miss output file
+// VERIFY-OUTPUT: warning: module 'A' was not compiled with library evolution support
+// CACHE-HIT: remark: replay output file
+
 //--- A.swift
 func test() {}
 

test/ScanDependencies/explicit-module-map-typecheck-module-from-interface.swift

@@ -0,0 +1,42 @@
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/clang-module-cache
+// RUN: mkdir -p %t/inputs
+// RUN: echo "/// Some cool comments" > %t/foo.swift
+// RUN: echo "public func foo() {}" >> %t/foo.swift
+// RUN: %target-swift-frontend -emit-module -emit-module-path %t/inputs/Foo.swiftmodule -emit-module-interface-path %t/Foo.swiftinterface \
+// RUN:   -swift-version 5 -enable-library-evolution -module-cache-path %t.module-cache %t/foo.swift -module-name Foo
+
+// RUN: echo "[{" > %/t/inputs/map.json
+// RUN: echo "\"moduleName\": \"Foo\"," >> %/t/inputs/map.json
+// RUN: echo "\"modulePath\": \"%/t/inputs/Foo.swiftmodule\"," >> %/t/inputs/map.json
+// RUN: echo "\"docPath\": \"%/t/inputs/Foo.swiftdoc\"," >> %/t/inputs/map.json
+// RUN: echo "\"sourceInfoPath\": \"%/t/inputs/Foo.swiftsourceinfo\"," >> %/t/inputs/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/inputs/map.json
+// RUN: echo "}," >> %/t/inputs/map.json
+// RUN: echo "{" >> %/t/inputs/map.json
+// RUN: echo "\"moduleName\": \"Swift\"," >> %/t/inputs/map.json
+// RUN: echo "\"modulePath\": \"%/stdlib_module\"," >> %/t/inputs/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/inputs/map.json
+// RUN: echo "}," >> %/t/inputs/map.json
+// RUN: echo "{" >> %/t/inputs/map.json
+// RUN: echo "\"moduleName\": \"SwiftOnoneSupport\"," >> %/t/inputs/map.json
+// RUN: echo "\"modulePath\": \"%/ononesupport_module\"," >> %/t/inputs/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/inputs/map.json
+// RUN: echo "}," >> %/t/inputs/map.json
+// RUN: echo "{" >> %/t/inputs/map.json
+// RUN: echo "\"moduleName\": \"_Concurrency\"," >> %/t/inputs/map.json
+// RUN: echo "\"modulePath\": \"%/concurrency_module\"," >> %/t/inputs/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/inputs/map.json
+// RUN: echo "}," >> %/t/inputs/map.json
+// RUN: echo "{" >> %/t/inputs/map.json
+// RUN: echo "\"moduleName\": \"_StringProcessing\"," >> %/t/inputs/map.json
+// RUN: echo "\"modulePath\": \"%/string_processing_module\"," >> %/t/inputs/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/inputs/map.json
+// RUN: echo "}]" >> %/t/inputs/map.json
+
+// RUN: %target-swift-frontend -typecheck-module-from-interface %t/Foo.swiftinterface -module-cache-path %t.module-cache \
+// RUN:   -explicit-interface-module-build -explicit-swift-module-map-file %t/inputs/map.json -Rmodule-loading -Xcc -Rmodule-import 2>&1 | %FileCheck %s
+
+// CHECK-DAG: loaded module 'Swift'
+// CHECK-DAG: loaded module '_StringProcessing'
+// CHECK-DAG: loaded module '_Concurrency'