Don't load rules from an unsupported standard version

CMakeLists.txt

@@ -228,11 +228,11 @@ foreach(_header IN LISTS rule_common_headers)
   list(APPEND _common_headers --common-header=${_header})
 endforeach()
 set(cpp_rules_ir_outputs)
-set(cpp_rules_index_inputs)
-set(cpp_rules_index_outputs ${CMAKE_BINARY_DIR}/rules_index/c.json ${CMAKE_BINARY_DIR}/rules_index/cpp.json)
+set(rules_index_parts)
+set(rules_index_parts_dir ${CMAKE_BINARY_DIR}/rules_index/parts)
 set(rust_rules_ir_outputs)
 foreach(_rule_dir IN LISTS rule_subdirs)
-  file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
+  file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp ${_rule_dir}/src_cxx*.cpp)
   if(NOT _cpp_srcs)
     continue()
   endif()
@@ -258,7 +258,15 @@ foreach(_rule_dir IN LISTS rule_subdirs)
   list(APPEND cpp_rules_ir_outputs ${_cpp_out})
 
   if(NOT _rule_name STREQUAL "builtin_types")
-    list(APPEND cpp_rules_index_inputs ${_cpp_srcs} ${_cxxflags_file})
+    set(_part ${rules_index_parts_dir}/${_rule_name}.json)
+    add_custom_command(
+      OUTPUT ${_part}
+      COMMAND ${CMAKE_COMMAND} -E make_directory ${rules_index_parts_dir}
+      COMMAND $<TARGET_FILE:cpp-rule-indexer> --dir ${_rule_dir} -o ${_part}
+      DEPENDS ${_cpp_srcs} ${rule_common_headers} ${rule_indexer_sources} ${RUSTLS_HEADER} ${BROTLI_HEADERS}
+      VERBATIM
+    )
+    list(APPEND rules_index_parts ${_part})
   endif()
 
   set(_rust_srcs)
@@ -291,10 +299,11 @@ endforeach()
 
 add_custom_target("preprocess-cpp-rules" ALL DEPENDS ${cpp_rules_ir_outputs})
 
+set(cpp_rules_index_outputs ${CMAKE_BINARY_DIR}/rules_index/c.json ${CMAKE_BINARY_DIR}/rules_index/cpp.json)
 add_custom_command(
   OUTPUT ${cpp_rules_index_outputs}
-  COMMAND $<TARGET_FILE:cpp-rule-indexer> --exclude builtin_types ${_common_headers}
-  DEPENDS ${cpp_rules_index_inputs} ${rule_common_headers} ${rule_indexer_sources} ${RUSTLS_HEADER} ${BROTLI_HEADERS}
+  COMMAND $<TARGET_FILE:cpp-rule-indexer> ${_common_headers} -o ${CMAKE_BINARY_DIR}/rules_index ${rules_index_parts}
+  DEPENDS ${rules_index_parts} ${rule_common_headers} ${rule_indexer_sources}
   VERBATIM
 )
 

cpp2rust/converter/rules/rules_loader.cpp

@@ -5,6 +5,7 @@
 
 #include <clang/AST/ASTContext.h>
 #include <clang/AST/RecursiveASTVisitor.h>
+#include <clang/Basic/LangOptions.h>
 #include <clang/Lex/Lexer.h>
 #include <clang/Lex/Preprocessor.h>
 #include <clang/Sema/Sema.h>
@@ -296,9 +297,30 @@ std::string WrapRule(const llvm::json::Object &rule, bool is_cxx) {
       dir, text);
 }
 
+unsigned CxxStandard(const clang::LangOptions &opts) {
+  if (opts.CPlusPlus26) {
+    return 26;
+  }
+  if (opts.CPlusPlus23) {
+    return 23;
+  }
+  if (opts.CPlusPlus20) {
+    return 20;
+  }
+  if (opts.CPlusPlus17) {
+    return 17;
+  }
+  if (opts.CPlusPlus14) {
+    return 14;
+  }
+  return 11;
+}
+
 std::string BuildRulesEpilogue(const std::unordered_set<std::string> &keys,
-                               bool is_cxx) {
+                               const clang::LangOptions &opts) {
+  bool is_cxx = opts.CPlusPlus;
   const auto &index = is_cxx ? CxxIndex() : CIndex();
+  unsigned standard = is_cxx ? CxxStandard(opts) : 0;
   auto common = index.getString("common");
   std::string out;
   if (!common->empty()) {
@@ -312,6 +334,9 @@ std::string BuildRulesEpilogue(const std::unordered_set<std::string> &keys,
     }
     for (const auto &entry : *entries) {
       auto rule = entry.getAsObject();
+      if (auto since = rule->getInteger("since"); since && *since > standard) {
+        continue;
+      }
       if (auto required = rule->getArray("requires");
           required &&
           !llvm::all_of(*required, [&](const llvm::json::Value &key) {
@@ -482,7 +507,7 @@ void PragmaHandler::HandlePragma(clang::Preprocessor &PP,
   RuleUsageCollector collector(CI_.getSema(), keys);
   collector.TraverseDecl(ctx.getTranslationUnitDecl());
 
-  auto text = BuildRulesEpilogue(keys, ctx.getLangOpts().CPlusPlus);
+  auto text = BuildRulesEpilogue(keys, ctx.getLangOpts());
   log() << "rules loaded for this translation unit:\n" << text;
   auto rules_file = src_mgr.createFileID(
       llvm::MemoryBuffer::getMemBufferCopy(text, "<cpp2rust-rules>"),

cpp2rust/rule_preprocessing/cpp_rule_indexer.cpp

@@ -21,7 +21,6 @@
 #include <cstdlib>
 #include <filesystem>
 #include <format>
-#include <fstream>
 #include <string>
 #include <unordered_set>
 #include <vector>
@@ -309,54 +308,91 @@ void WriteJson(const fs::path &path, llvm::json::Object object) {
   os << llvm::json::Value(std::move(object));
 }
 
-std::vector<std::string> ReadCXXFlags(const fs::path &rule_dir) {
-  std::vector<std::string> flags;
-  std::ifstream file(rule_dir / "cxxflags");
-  for (std::string flag; std::getline(file, flag);) {
-    if (!flag.empty()) {
-      flags.push_back(flag);
-    }
+llvm::json::Object ReadJson(const fs::path &path) {
+  auto buffer = llvm::MemoryBuffer::getFile(path.string());
+  if (!buffer) {
+    llvm::errs() << "ERROR: cannot read " << path.string() << '\n';
+    std::exit(EXIT_FAILURE);
+  }
+  auto parsed = llvm::json::parse((*buffer)->getBuffer());
+  if (!parsed || !parsed->getAsObject()) {
+    llvm::errs() << "ERROR: cannot parse " << path.string() << '\n';
+    std::exit(EXIT_FAILURE);
+  }
+  return std::move(*parsed->getAsObject());
+}
+
+unsigned SinceStandard(const fs::path &src_path) {
+  auto stem_string = src_path.stem().string();
+  llvm::StringRef stem = stem_string;
+  unsigned standard = 11;
+  if (stem.consume_front("src_cxx") && stem.getAsInteger(10, standard)) {
+    llvm::errs() << "ERROR: unexpected rule file " << src_path.string() << '\n';
+    std::exit(EXIT_FAILURE);
   }
-  return flags;
+  return standard;
 }
 
-void IndexRules(const fs::path &rules_dir, const fs::path &index_dir,
-                const std::vector<std::string> &excluded,
-                const std::vector<std::string> &common_headers) {
-  std::vector<fs::path> rule_dirs;
-  for (const auto &entry : fs::directory_iterator(rules_dir)) {
-    if (entry.is_directory() &&
-        !llvm::is_contained(excluded, entry.path().filename().string())) {
-      rule_dirs.push_back(entry.path());
+void MergeRules(llvm::json::Object &into, llvm::json::Object &&from) {
+  for (auto &[key, entries] : from) {
+    for (auto &entry : *entries.getAsArray()) {
+      AddRule(into, key.str(), std::move(entry));
     }
   }
-  std::ranges::sort(rule_dirs);
-  fs::create_directories(index_dir);
-  for (const char *lang : {"c", "cpp"}) {
-    std::string common;
-    if (lang == std::string("cpp")) {
-      for (const auto &header : common_headers) {
-        common += std::format("#include \"{}\"\n", header);
-      }
+}
+
+void IndexDir(const fs::path &rule_dir, const fs::path &out) {
+  auto dir_name = rule_dir.filename().string();
+  llvm::json::Object c_rules;
+  if (fs::exists(rule_dir / "src.c")) {
+    Index(rule_dir / "src.c", /*is_c=*/true, dir_name, {}, c_rules);
+  }
+  std::vector<fs::path> cxx_srcs;
+  for (const auto &entry : fs::directory_iterator(rule_dir)) {
+    auto name = entry.path().filename().string();
+    if (name == "src.c" || name == "src.cpp" ||
+        (name.starts_with("src_cxx") && name.ends_with(".cpp"))) {
+      cxx_srcs.push_back(entry.path());
     }
-    bool is_c = lang == std::string("c");
+  }
+  std::ranges::sort(cxx_srcs);
+  llvm::json::Object cxx_rules;
+  for (const auto &src_path : cxx_srcs) {
+    auto standard = SinceStandard(src_path);
     llvm::json::Object rules;
-    for (const auto &rule_dir : rule_dirs) {
-      for (const char *name : {"src.c", "src.cpp"}) {
-        auto src_path = rule_dir / name;
-        if (!fs::exists(src_path) || (is_c && src_path.extension() != ".c")) {
-          continue;
-        }
-        llvm::errs() << "Indexing " << src_path.string() << " as " << lang
-                     << '\n';
-        Index(src_path, is_c, rule_dir.filename().string(),
-              ReadCXXFlags(rule_dir), rules);
+    Index(src_path, /*is_c=*/false, dir_name,
+          {std::format("-std=c++{}", standard)}, rules);
+    for (auto &[key, entries] : rules) {
+      for (auto &entry : *entries.getAsArray()) {
+        (*entry.getAsObject())["since"] = standard;
       }
     }
-    WriteJson(index_dir / (std::string(lang) + ".json"),
-              llvm::json::Object{{"common", std::move(common)},
-                                 {"rules", std::move(rules)}});
+    MergeRules(cxx_rules, std::move(rules));
+  }
+  WriteJson(out, llvm::json::Object{{"c", std::move(c_rules)},
+                                    {"cpp", std::move(cxx_rules)}});
+}
+
+void MergeIndexes(const std::vector<std::string> &parts,
+                  const std::vector<std::string> &common_headers,
+                  const fs::path &index_dir) {
+  llvm::json::Object c_rules;
+  llvm::json::Object cxx_rules;
+  for (const auto &part : parts) {
+    auto json = ReadJson(part);
+    MergeRules(c_rules, std::move(*json.getObject("c")));
+    MergeRules(cxx_rules, std::move(*json.getObject("cpp")));
+  }
+  std::string common;
+  for (const auto &header : common_headers) {
+    common += std::format("#include \"{}\"\n", header);
   }
+  fs::create_directories(index_dir);
+  WriteJson(index_dir / "c.json",
+            llvm::json::Object{{"common", ""}, {"rules", std::move(c_rules)}});
+  WriteJson(index_dir / "cpp.json",
+            llvm::json::Object{{"common", std::move(common)},
+                               {"rules", std::move(cxx_rules)}});
 }
 
 } // namespace cpp2rust
@@ -365,23 +401,35 @@ namespace {
 
 llvm::cl::OptionCategory cat("cpp-rule-indexer options");
 
-llvm::cl::list<std::string>
-    Excluded("exclude", llvm::cl::desc("Rule directory that is not indexed"),
-             llvm::cl::value_desc("rule-dir"), llvm::cl::ZeroOrMore,
-             llvm::cl::cat(cat));
+llvm::cl::opt<std::string> Dir("dir", llvm::cl::desc("Rule directory to index"),
+                               llvm::cl::value_desc("rule-dir"),
+                               llvm::cl::cat(cat));
+
+llvm::cl::list<std::string> Parts(llvm::cl::Positional,
+                                  llvm::cl::desc("<indexes to merge>"),
+                                  llvm::cl::ZeroOrMore, llvm::cl::cat(cat));
 
 llvm::cl::list<std::string> CommonHeaders(
     "common-header", llvm::cl::desc("Header the C++ rules depend on"),
     llvm::cl::value_desc("header"), llvm::cl::ZeroOrMore, llvm::cl::cat(cat));
 
+llvm::cl::opt<std::string>
+    Out("o",
+        llvm::cl::desc("Output part of a rule directory, or index directory"),
+        llvm::cl::value_desc("path"), llvm::cl::Required, llvm::cl::cat(cat));
+
 } // namespace
 
 int main(int argc, char *argv[]) {
   llvm::cl::HideUnrelatedOptions(cat);
   llvm::cl::ParseCommandLineOptions(argc, argv);
 
-  cpp2rust::IndexRules(RULES_SRC_DIR, RULES_INDEX_DIR,
-                       {Excluded.begin(), Excluded.end()},
-                       {CommonHeaders.begin(), CommonHeaders.end()});
+  if (!Dir.empty()) {
+    cpp2rust::IndexDir(Dir.getValue(), Out.getValue());
+  } else {
+    cpp2rust::MergeIndexes({Parts.begin(), Parts.end()},
+                           {CommonHeaders.begin(), CommonHeaders.end()},
+                           Out.getValue());
+  }
   return EXIT_SUCCESS;
 }

cpp2rust/rule_preprocessing/cpp_rule_preprocessor.cpp

@@ -27,9 +27,11 @@
 #include <llvm/Support/MemoryBuffer.h>
 #include <llvm/Support/raw_ostream.h>
 
+#include <algorithm>
 #include <cstdlib>
 #include <filesystem>
 #include <string>
+#include <vector>
 
 #include "compat/platform_flags.h"
 #include "converter/converter_lib.h"
@@ -1120,14 +1122,26 @@ int main(int argc, char *argv[]) {
                                                   CXXFlags.end());
   fs::path dir = SrcDir.getValue();
   llvm::json::Object root;
-  for (const char *name : {"src.c", "src.cpp"}) {
-    auto path = dir / name;
-    if (!fs::exists(path)) {
-      continue;
+  std::vector<fs::path> paths;
+  for (const auto &entry : fs::directory_iterator(dir)) {
+    auto name = entry.path().filename().string();
+    if (name == "src.c" || name == "src.cpp" ||
+        (name.starts_with("src_cxx") && name.ends_with(".cpp"))) {
+      paths.push_back(entry.path());
     }
+  }
+  std::ranges::sort(paths);
+  for (const auto &path : paths) {
     llvm::errs() << "Preprocessing " << path.string() << '\n';
+    llvm::SmallVector<llvm::StringRef, 4> file_flags(cxx_flags);
+    auto stem = path.stem().string();
+    std::string standard_flag;
+    if (llvm::StringRef standard = stem; standard.consume_front("src_cxx")) {
+      standard_flag = "-std=c++" + standard.str();
+      file_flags.push_back(standard_flag);
+    }
     llvm::json::Object file_root;
-    cpp2rust::Extract(path, file_root, cxx_flags);
+    cpp2rust::Extract(path, file_root, file_flags);
     for (auto &[k, v] : file_root) {
       if (!root.try_emplace(k, std::move(v)).second) {
         llvm::errs() << "ERROR: rule name " << k.str()

docs/src/rules/format.md

@@ -5,6 +5,8 @@ library it covers (`rules/unistd`, `rules/vector`, `rules/string`, ...). It
 contains:
 
 - `src.cpp` and/or `src.c`: the C++ (or C) side of each rule.
+- `src_cxxNN.cpp` (optional): C++ rules that exist from C++NN on, for example
+  `src_cxx17.cpp`; see below.
 - `tgt_unsafe.rs`: the Rust targets for the unsafe model.
 - `tgt_refcount.rs`: the Rust targets for the reference counting model
   (optional, see below).
@@ -141,6 +143,21 @@ for the C++ `__builtin_mul_overflow` (returning `bool`) while `src.c` defines
 `f12`/`f13` for the C version (returning `int`); their Rust bodies are
 identical.
 
+## Standard versions
+
+`src.cpp` is parsed as C++11 and its rules are available to every C++
+translation unit. A rule that needs a later standard goes into `src_cxxNN.cpp`,
+which is parsed as C++NN and whose rules are only loaded into translation units
+compiled as C++NN or later:
+
+```cpp
+// rules/new/src_cxx17.cpp
+template <typename T1> T1 *f5(T1 *a0) { return std::launder(a0); }
+```
+
+Rule names are shared across all the files of a module. Platform `#if`s are fine
+in any file; `#if __cplusplus` is not needed.
+
 ## The `rules` crate
 
 The whole `rules/` tree is a single Rust crate. `rules/build.rs` walks the tree,

rules/builtin_types/src.cpp

@@ -13,9 +13,6 @@
 typedef decltype(nullptr) t0;
 
 BUILTIN_TYPE(1, bool)
-#if __cplusplus >= 202002L
-BUILTIN_TYPE(5, char8_t)
-#endif
 BUILTIN_TYPE(8, char16_t)
 BUILTIN_TYPE(11, wchar_t)
 BUILTIN_TYPE(12, char32_t)

rules/builtin_types/src_cxx20.cpp

@@ -0,0 +1,8 @@
+// Copyright (c) 2022-present INESC-ID.
+// Distributed under the MIT license that can be found in the LICENSE file.
+
+typedef char8_t t50;
+typedef char8_t *t51;
+typedef const char8_t *t52;
+typedef volatile char8_t *t53;
+typedef const volatile char8_t *t54;

rules/cstddef/src.cpp → rules/cstddef/src_cxx17.cpp

@@ -3,7 +3,6 @@
 
 #include <cstddef>
 
-#if __cplusplus >= 201703L
 using t1 = std::byte;
 typedef std::size_t t9;
 typedef std::size_t *t10;
@@ -16,4 +15,3 @@ std::byte f2(const std::byte &a0, unsigned a1) { return operator>>(a0, a1); }
 std::byte f3(std::byte &a0, unsigned a1) { return operator<<=(a0, a1); }
 
 std::byte f4(std::byte &a0, unsigned a1) { return operator>>=(a0, a1); }
-#endif