Selectively load rules in the same TU as the translated code (#451)

Part of #447. This only loads the rules into cpp2rust, AST matching
comes in a future PR:

> To implement this, rules are included in the translated TU. Only the
rules that parse are kept. If the TU never included <map> then there are
no rules for map. This is faster than the string matching approach.

cpp_rule_indexer runs at build time to build an index of rules. This
index is used by rule_loader to lazily load the rules into the converter
for doing AST matching.

There are 2 index files: one for C code and one for C++ code. Both live
in build/rules_index. C translated code only loads the C index, C++
translated code loads both indexes.

Example of index:

```
key:      std/vector/push_back
requires: std/vector
dir:      vector
text:
        template <typename T1> void f14(std::vector<T1> &o, T1 &&value) {
          return o.push_back(std::move(value));
        }

key:      std/basic_ifstream/rdbuf
requires: std/basic_ifstream, std/basic_filebuf
dir:      ifstream
text:
        std::filebuf *f4(const std::ifstream &o) { return o.rdbuf(); }
```

`key` is generated using the dependent name of the rules for templates
and using the concrete name for non-templates. `text` is extracted from
the file and appended into the translated TU if it needs the rule.
`requires` only loads the rule in the TU if the translated code uses all
the dependencies, basic_ifstream and basic_fielbuf in the example.

This makes the translation fast. With 100K dummy rules, it takes on
avergage 20ms per invocation to selectively load the rules. With full
loading of all the 100K dummy rules, it takes 5 seconds on average.

Author: lucic71

CMakeLists.txt

@@ -219,7 +219,17 @@ add_custom_command(
 )
 
 file(GLOB rule_subdirs CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/rules/*)
+file(GLOB rule_common_headers CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/rules/common/*.h)
+file(GLOB rule_indexer_sources CONFIGURE_DEPENDS
+     ${PROJECT_SOURCE_DIR}/cpp2rust/rule_preprocessing/*
+     ${PROJECT_SOURCE_DIR}/cpp2rust/converter/rules/rules_loader.*)
+set(_common_headers)
+foreach(_header IN LISTS rule_common_headers)
+  list(APPEND _common_headers --common-header=${_header})
+endforeach()
 set(cpp_rules_ir_outputs)
+set(cpp_rules_index_inputs)
+set(cpp_rules_index_outputs ${CMAKE_BINARY_DIR}/rules_index/c.json ${CMAKE_BINARY_DIR}/rules_index/cpp.json)
 set(rust_rules_ir_outputs)
 foreach(_rule_dir IN LISTS rule_subdirs)
   file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
@@ -229,7 +239,7 @@ foreach(_rule_dir IN LISTS rule_subdirs)
   get_filename_component(_rule_name ${_rule_dir} NAME)
   set(_out_dir ${RULES_IR_DIR}/${_rule_name})
   set(_cpp_out ${_out_dir}/ir_src.json)
-  set(_cxxflags --cxxflags=-isystem${_rule_dir})
+  set(_cxxflags)
   set(_cxxflags_file)
   if(EXISTS ${_rule_dir}/cxxflags)
     set(_cxxflags_file ${_rule_dir}/cxxflags)
@@ -242,11 +252,15 @@ foreach(_rule_dir IN LISTS rule_subdirs)
     OUTPUT ${_cpp_out}
     COMMAND ${CMAKE_COMMAND} -E make_directory ${_out_dir}
     COMMAND $<TARGET_FILE:cpp-rule-preprocessor> --dir ${_rule_dir} --out ${_cpp_out} ${_cxxflags}
-    DEPENDS ${_cpp_srcs} ${_cxxflags_file} ${PROJECT_SOURCE_DIR}/cpp2rust/cpp_rule_preprocessor.cpp ${RUSTLS_HEADER} ${BROTLI_HEADERS}
+    DEPENDS ${_cpp_srcs} ${_cxxflags_file} ${rule_common_headers} ${PROJECT_SOURCE_DIR}/cpp2rust/rule_preprocessing/cpp_rule_preprocessor.cpp ${RUSTLS_HEADER} ${BROTLI_HEADERS}
     VERBATIM
   )
   list(APPEND cpp_rules_ir_outputs ${_cpp_out})
 
+  if(NOT _rule_name STREQUAL "builtin_types")
+    list(APPEND cpp_rules_index_inputs ${_cpp_srcs} ${_cxxflags_file})
+  endif()
+
   set(_rust_srcs)
   set(_rust_out)
   if(EXISTS ${_rule_dir}/tgt_unsafe.rs)
@@ -277,6 +291,15 @@ endforeach()
 
 add_custom_target("preprocess-cpp-rules" ALL DEPENDS ${cpp_rules_ir_outputs})
 
+add_custom_command(
+  OUTPUT ${cpp_rules_index_outputs}
+  COMMAND $<TARGET_FILE:cpp-rule-indexer> --exclude builtin_types ${_common_headers}
+  DEPENDS ${cpp_rules_index_inputs} ${rule_common_headers} ${rule_indexer_sources} ${RUSTLS_HEADER} ${BROTLI_HEADERS}
+  VERBATIM
+)
+
+add_custom_target("index-cpp-rules" ALL DEPENDS ${cpp_rules_index_outputs})
+
 add_custom_target("preprocess-rust-rules" ALL
                   DEPENDS ${rust_rules_ir_outputs})
 

cpp2rust/CMakeLists.txt

@@ -1,7 +1,8 @@
 file(GLOB_RECURSE CORE_SOURCES "*.cpp")
+file(GLOB RULE_PREPROCESSING_SOURCES "rule_preprocessing/*.cpp")
 list(REMOVE_ITEM CORE_SOURCES
     "${CMAKE_CURRENT_SOURCE_DIR}/cpp2rust.cpp"
-    "${CMAKE_CURRENT_SOURCE_DIR}/cpp_rule_preprocessor.cpp"
+    ${RULE_PREPROCESSING_SOURCES}
 )
 
 add_library(cpp2rust_core STATIC ${CORE_SOURCES})
@@ -17,6 +18,8 @@ target_compile_definitions(cpp2rust_core PUBLIC "-DCLANG_C_COMPILER=\"${CMAKE_C_
 target_compile_definitions(cpp2rust_core PUBLIC "-DCLANG_CXX_COMPILER=\"${CMAKE_CXX_COMPILER}\"")
 target_compile_definitions(cpp2rust_core PUBLIC "-DCLANG_RESOURCE_DIR=\"${LLVM_LIBRARY_DIR}/clang/${LLVM_VERSION_MAJOR}\"")
 target_compile_definitions(cpp2rust_core PUBLIC "-DCOMPAT_INCLUDE_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/compat\"")
+target_compile_definitions(cpp2rust_core PUBLIC "-DRULES_SRC_DIR=\"${PROJECT_SOURCE_DIR}/rules\"")
+target_compile_definitions(cpp2rust_core PUBLIC "-DRULES_INDEX_DIR=\"${CMAKE_BINARY_DIR}/rules_index\"")
 target_compile_definitions(cpp2rust_core PUBLIC "-DRUST_STABLE_VERSION=\"${RUST_STABLE_VERSION}\"")
 
 if (APPLE)
@@ -34,5 +37,10 @@ target_link_libraries(cpp2rust_core PUBLIC ${llvm_libs})
 add_clang_executable(cpp2rust PARTIAL_SOURCES_INTENDED cpp2rust.cpp)
 target_link_libraries(cpp2rust PRIVATE cpp2rust_core)
 
-add_clang_executable(cpp-rule-preprocessor PARTIAL_SOURCES_INTENDED cpp_rule_preprocessor.cpp)
+add_clang_executable(cpp-rule-preprocessor PARTIAL_SOURCES_INTENDED
+    rule_preprocessing/cpp_rule_preprocessor.cpp)
 target_link_libraries(cpp-rule-preprocessor PRIVATE cpp2rust_core)
+
+add_clang_executable(cpp-rule-indexer PARTIAL_SOURCES_INTENDED
+    rule_preprocessing/cpp_rule_indexer.cpp)
+target_link_libraries(cpp-rule-indexer PRIVATE cpp2rust_core)

cpp2rust/ast_consumer.cpp

@@ -3,10 +3,16 @@
 
 #include "ast_consumer.h"
 
+#include <llvm/Support/ErrorHandling.h>
+
 #include "converter/converter.h"
 
 namespace cpp2rust {
 void ASTConsumer::HandleTranslationUnit(clang::ASTContext &ctx) {
+  if (CI_.getDiagnostics().hasErrorOccurred()) {
+    llvm::report_fatal_error(
+        "the translation unit and its loaded rules do not compile");
+  }
   auto converter = CreateConverter(rs_code_, ctx, model_, rules_dir_);
   converter->SetSema(CI_.getSema());
   if (first_) {

cpp2rust/converter/converter.cpp

@@ -1030,49 +1030,7 @@ bool Converter::VisitCXXRecordDecl(clang::CXXRecordDecl *decl) {
 }
 
 void Converter::DefineImplicitMembers(clang::CXXRecordDecl *decl) {
-  clang::Scope tu_scope(nullptr, clang::Scope::DeclScope,
-                        sema_->getDiagnostics());
-  tu_scope.setEntity(ctx_.getTranslationUnitDecl());
-  auto *saved_tu_scope = std::exchange(sema_->TUScope, &tu_scope);
-  sema_->ForceDeclarationOfImplicitMembers(decl);
-  for (auto ctor : decl->ctors()) {
-    if (ctor->isCopyConstructor() && ctor->isImplicit() &&
-        !ctor->doesThisDeclarationHaveABody() && !ctor->isDeleted()) {
-      sema_->DefineImplicitCopyConstructor(decl->getLocation(), ctor);
-    }
-    if (ctor->isMoveConstructor() && !ctor->isUserProvided() &&
-        !ctor->doesThisDeclarationHaveABody() && !ctor->isDeleted() &&
-        !HasDefaultedCopyConstructor(decl)) {
-      sema_->DefineImplicitMoveConstructor(decl->getLocation(), ctor);
-    }
-  }
-  for (auto *method : decl->methods()) {
-    if (method->isMoveAssignmentOperator() && !method->isUserProvided() &&
-        !method->doesThisDeclarationHaveABody() && !method->isDeleted() &&
-        !HasDefaultedCopyAssignment(decl)) {
-      sema_->DefineImplicitMoveAssignment(decl->getLocation(), method);
-    }
-  }
-  auto define_defaulted_comparison = [&](clang::FunctionDecl *fn) {
-    if (!fn || !IsComparisonOperator(fn) || !fn->isDefaulted() ||
-        fn->doesThisDeclarationHaveABody()) {
-      return;
-    }
-#if CLANG_VERSION_MAJOR >= 24
-    auto kind = fn->getDefaultedComparisonKind();
-#else
-    auto kind = sema_->getDefaultedComparisonKind(fn);
-#endif
-    sema_->DefineDefaultedComparison(decl->getLocation(), fn, kind);
-  };
-  for (auto *method : decl->methods()) {
-    define_defaulted_comparison(method);
-  }
-  for (auto *friend_decl : decl->friends()) {
-    define_defaulted_comparison(clang::dyn_cast_or_null<clang::FunctionDecl>(
-        friend_decl->getFriendDecl()));
-  }
-  sema_->TUScope = saved_tu_scope;
+  cpp2rust::DefineImplicitMembers(*sema_, decl);
 }
 
 bool Converter::VisitCXXMethodDecl(clang::CXXMethodDecl *decl) {

cpp2rust/converter/converter_lib.cpp

@@ -3,6 +3,7 @@
 
 #include "converter/converter_lib.h"
 
+#include <clang/AST/DeclFriend.h>
 #include <clang/AST/DeclTemplate.h>
 #include <clang/AST/ExprCXX.h>
 #include <clang/AST/ExprConcepts.h>
@@ -1460,6 +1461,52 @@ bool IsComparisonOperator(const clang::FunctionDecl *fn) {
   }
 }
 
+void DefineImplicitMembers(clang::Sema &sema, clang::CXXRecordDecl *decl) {
+  clang::Scope tu_scope(nullptr, clang::Scope::DeclScope,
+                        sema.getDiagnostics());
+  tu_scope.setEntity(sema.Context.getTranslationUnitDecl());
+  auto *saved_tu_scope = std::exchange(sema.TUScope, &tu_scope);
+  sema.ForceDeclarationOfImplicitMembers(decl);
+  for (auto ctor : decl->ctors()) {
+    if (ctor->isCopyConstructor() && ctor->isImplicit() &&
+        !ctor->doesThisDeclarationHaveABody() && !ctor->isDeleted()) {
+      sema.DefineImplicitCopyConstructor(decl->getLocation(), ctor);
+    }
+    if (ctor->isMoveConstructor() && !ctor->isUserProvided() &&
+        !ctor->doesThisDeclarationHaveABody() && !ctor->isDeleted() &&
+        !HasDefaultedCopyConstructor(decl)) {
+      sema.DefineImplicitMoveConstructor(decl->getLocation(), ctor);
+    }
+  }
+  for (auto *method : decl->methods()) {
+    if (method->isMoveAssignmentOperator() && !method->isUserProvided() &&
+        !method->doesThisDeclarationHaveABody() && !method->isDeleted() &&
+        !HasDefaultedCopyAssignment(decl)) {
+      sema.DefineImplicitMoveAssignment(decl->getLocation(), method);
+    }
+  }
+  auto define_defaulted_comparison = [&](clang::FunctionDecl *fn) {
+    if (!fn || !IsComparisonOperator(fn) || !fn->isDefaulted() ||
+        fn->doesThisDeclarationHaveABody()) {
+      return;
+    }
+#if CLANG_VERSION_MAJOR >= 24
+    auto kind = fn->getDefaultedComparisonKind();
+#else
+    auto kind = sema.getDefaultedComparisonKind(fn);
+#endif
+    sema.DefineDefaultedComparison(decl->getLocation(), fn, kind);
+  };
+  for (auto *method : decl->methods()) {
+    define_defaulted_comparison(method);
+  }
+  for (auto *friend_decl : decl->friends()) {
+    define_defaulted_comparison(clang::dyn_cast_or_null<clang::FunctionDecl>(
+        friend_decl->getFriendDecl()));
+  }
+  sema.TUScope = saved_tu_scope;
+}
+
 bool IsEmittableMethod(clang::CXXMethodDecl *method) {
   if (method->isDeleted()) {
     return false;

cpp2rust/converter/converter_lib.h

@@ -133,6 +133,9 @@ bool IsConvertibleCXXRecordDecl(const clang::CXXRecordDecl *decl);
 bool IsConvertibleCXXMethodDecl(const clang::CXXMethodDecl *decl);
 
 bool IsComparisonOperator(const clang::FunctionDecl *fn);
+
+void DefineImplicitMembers(clang::Sema &sema, clang::CXXRecordDecl *decl);
+
 bool IsEmittableMethod(clang::CXXMethodDecl *method);
 
 bool IsMethodOnPtr(const clang::CXXMethodDecl *method);

cpp2rust/converter/printer.cpp

@@ -134,7 +134,20 @@ std::string ToString(clang::ASTContext &ctx, clang::QualType qual_type,
     }
     if (const auto *typedef_type = t->getAs<clang::TypedefType>()) {
       if (t.getCanonicalType()->isBuiltinType()) {
-        return typedef_type->getDecl()->getNameAsString();
+        auto decl = typedef_type->getDecl();
+        if (auto record =
+                clang::dyn_cast<clang::RecordDecl>(decl->getDeclContext());
+            record && !record->isDependentContext()) {
+          return ToString(ctx, ctx.getCanonicalTagType(record)) +
+                 "::" + decl->getNameAsString();
+        }
+        if (decl->getDeclContext()->getRedeclContext()->isNamespace()) {
+          std::string name;
+          llvm::raw_string_ostream os(name);
+          decl->printQualifiedName(os, getPrintPolicy(ctx));
+          return name;
+        }
+        return decl->getNameAsString();
       }
     } else if (const auto *predef = t->getAs<clang::PredefinedSugarType>()) {
       return predef->getIdentifier()->getName().str();