Skip to content

Commit d84269b

Browse files
authored
Don't load rules from an unsupported standard version (#529)
In rules/, standard specific rules are guarded by `#if __cplusplus >= 202002L`. We will also have cases where a rule is only available up to a specific standard version, then deprecated. This adds a new attribute in the rules index: supported standards. The loader only loads rules which have the same standard as the current TU.
1 parent ca7ee7e commit d84269b

17 files changed

Lines changed: 209 additions & 87 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -228,11 +228,11 @@ foreach(_header IN LISTS rule_common_headers)
228228
list(APPEND _common_headers --common-header=${_header})
229229
endforeach()
230230
set(cpp_rules_ir_outputs)
231-
set(cpp_rules_index_inputs)
232-
set(cpp_rules_index_outputs ${CMAKE_BINARY_DIR}/rules_index/c.json ${CMAKE_BINARY_DIR}/rules_index/cpp.json)
231+
set(rules_index_parts)
232+
set(rules_index_parts_dir ${CMAKE_BINARY_DIR}/rules_index/parts)
233233
set(rust_rules_ir_outputs)
234234
foreach(_rule_dir IN LISTS rule_subdirs)
235-
file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
235+
file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp ${_rule_dir}/src_cxx*.cpp)
236236
if(NOT _cpp_srcs)
237237
continue()
238238
endif()
@@ -258,7 +258,15 @@ foreach(_rule_dir IN LISTS rule_subdirs)
258258
list(APPEND cpp_rules_ir_outputs ${_cpp_out})
259259

260260
if(NOT _rule_name STREQUAL "builtin_types")
261-
list(APPEND cpp_rules_index_inputs ${_cpp_srcs} ${_cxxflags_file})
261+
set(_part ${rules_index_parts_dir}/${_rule_name}.json)
262+
add_custom_command(
263+
OUTPUT ${_part}
264+
COMMAND ${CMAKE_COMMAND} -E make_directory ${rules_index_parts_dir}
265+
COMMAND $<TARGET_FILE:cpp-rule-indexer> --dir ${_rule_dir} -o ${_part}
266+
DEPENDS ${_cpp_srcs} ${rule_common_headers} ${rule_indexer_sources} ${RUSTLS_HEADER} ${BROTLI_HEADERS}
267+
VERBATIM
268+
)
269+
list(APPEND rules_index_parts ${_part})
262270
endif()
263271

264272
set(_rust_srcs)
@@ -291,10 +299,11 @@ endforeach()
291299

292300
add_custom_target("preprocess-cpp-rules" ALL DEPENDS ${cpp_rules_ir_outputs})
293301

302+
set(cpp_rules_index_outputs ${CMAKE_BINARY_DIR}/rules_index/c.json ${CMAKE_BINARY_DIR}/rules_index/cpp.json)
294303
add_custom_command(
295304
OUTPUT ${cpp_rules_index_outputs}
296-
COMMAND $<TARGET_FILE:cpp-rule-indexer> --exclude builtin_types ${_common_headers}
297-
DEPENDS ${cpp_rules_index_inputs} ${rule_common_headers} ${rule_indexer_sources} ${RUSTLS_HEADER} ${BROTLI_HEADERS}
305+
COMMAND $<TARGET_FILE:cpp-rule-indexer> ${_common_headers} -o ${CMAKE_BINARY_DIR}/rules_index ${rules_index_parts}
306+
DEPENDS ${rules_index_parts} ${rule_common_headers} ${rule_indexer_sources}
298307
VERBATIM
299308
)
300309

‎cpp2rust/converter/rules/rules_loader.cpp‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
#include <clang/AST/ASTContext.h>
77
#include <clang/AST/RecursiveASTVisitor.h>
8+
#include <clang/Basic/LangOptions.h>
89
#include <clang/Lex/Lexer.h>
910
#include <clang/Lex/Preprocessor.h>
1011
#include <clang/Sema/Sema.h>
@@ -296,9 +297,30 @@ std::string WrapRule(const llvm::json::Object &rule, bool is_cxx) {
296297
dir, text);
297298
}
298299

300+
unsigned CxxStandard(const clang::LangOptions &opts) {
301+
if (opts.CPlusPlus26) {
302+
return 26;
303+
}
304+
if (opts.CPlusPlus23) {
305+
return 23;
306+
}
307+
if (opts.CPlusPlus20) {
308+
return 20;
309+
}
310+
if (opts.CPlusPlus17) {
311+
return 17;
312+
}
313+
if (opts.CPlusPlus14) {
314+
return 14;
315+
}
316+
return 11;
317+
}
318+
299319
std::string BuildRulesEpilogue(const std::unordered_set<std::string> &keys,
300-
bool is_cxx) {
320+
const clang::LangOptions &opts) {
321+
bool is_cxx = opts.CPlusPlus;
301322
const auto &index = is_cxx ? CxxIndex() : CIndex();
323+
unsigned standard = is_cxx ? CxxStandard(opts) : 0;
302324
auto common = index.getString("common");
303325
std::string out;
304326
if (!common->empty()) {
@@ -312,6 +334,9 @@ std::string BuildRulesEpilogue(const std::unordered_set<std::string> &keys,
312334
}
313335
for (const auto &entry : *entries) {
314336
auto rule = entry.getAsObject();
337+
if (auto since = rule->getInteger("since"); since && *since > standard) {
338+
continue;
339+
}
315340
if (auto required = rule->getArray("requires");
316341
required &&
317342
!llvm::all_of(*required, [&](const llvm::json::Value &key) {
@@ -484,7 +509,7 @@ void PragmaHandler::HandlePragma(clang::Preprocessor &PP,
484509
RuleUsageCollector collector(CI_.getSema(), keys);
485510
collector.TraverseDecl(ctx.getTranslationUnitDecl());
486511

487-
auto text = BuildRulesEpilogue(keys, ctx.getLangOpts().CPlusPlus);
512+
auto text = BuildRulesEpilogue(keys, ctx.getLangOpts());
488513
log() << "rules loaded for this translation unit:\n" << text;
489514
auto rules_file = src_mgr.createFileID(
490515
llvm::MemoryBuffer::getMemBufferCopy(text, "<cpp2rust-rules>"),

‎cpp2rust/rule_preprocessing/cpp_rule_indexer.cpp‎

Lines changed: 94 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
#include <cstdlib>
2222
#include <filesystem>
2323
#include <format>
24-
#include <fstream>
2524
#include <string>
2625
#include <unordered_set>
2726
#include <vector>
@@ -309,54 +308,91 @@ void WriteJson(const fs::path &path, llvm::json::Object object) {
309308
os << llvm::json::Value(std::move(object));
310309
}
311310

312-
std::vector<std::string> ReadCXXFlags(const fs::path &rule_dir) {
313-
std::vector<std::string> flags;
314-
std::ifstream file(rule_dir / "cxxflags");
315-
for (std::string flag; std::getline(file, flag);) {
316-
if (!flag.empty()) {
317-
flags.push_back(flag);
318-
}
311+
llvm::json::Object ReadJson(const fs::path &path) {
312+
auto buffer = llvm::MemoryBuffer::getFile(path.string());
313+
if (!buffer) {
314+
llvm::errs() << "ERROR: cannot read " << path.string() << '\n';
315+
std::exit(EXIT_FAILURE);
316+
}
317+
auto parsed = llvm::json::parse((*buffer)->getBuffer());
318+
if (!parsed || !parsed->getAsObject()) {
319+
llvm::errs() << "ERROR: cannot parse " << path.string() << '\n';
320+
std::exit(EXIT_FAILURE);
321+
}
322+
return std::move(*parsed->getAsObject());
323+
}
324+
325+
unsigned SinceStandard(const fs::path &src_path) {
326+
auto stem_string = src_path.stem().string();
327+
llvm::StringRef stem = stem_string;
328+
unsigned standard = 11;
329+
if (stem.consume_front("src_cxx") && stem.getAsInteger(10, standard)) {
330+
llvm::errs() << "ERROR: unexpected rule file " << src_path.string() << '\n';
331+
std::exit(EXIT_FAILURE);
319332
}
320-
return flags;
333+
return standard;
321334
}
322335

323-
void IndexRules(const fs::path &rules_dir, const fs::path &index_dir,
324-
const std::vector<std::string> &excluded,
325-
const std::vector<std::string> &common_headers) {
326-
std::vector<fs::path> rule_dirs;
327-
for (const auto &entry : fs::directory_iterator(rules_dir)) {
328-
if (entry.is_directory() &&
329-
!llvm::is_contained(excluded, entry.path().filename().string())) {
330-
rule_dirs.push_back(entry.path());
336+
void MergeRules(llvm::json::Object &into, llvm::json::Object &&from) {
337+
for (auto &[key, entries] : from) {
338+
for (auto &entry : *entries.getAsArray()) {
339+
AddRule(into, key.str(), std::move(entry));
331340
}
332341
}
333-
std::ranges::sort(rule_dirs);
334-
fs::create_directories(index_dir);
335-
for (const char *lang : {"c", "cpp"}) {
336-
std::string common;
337-
if (lang == std::string("cpp")) {
338-
for (const auto &header : common_headers) {
339-
common += std::format("#include \"{}\"\n", header);
340-
}
342+
}
343+
344+
void IndexDir(const fs::path &rule_dir, const fs::path &out) {
345+
auto dir_name = rule_dir.filename().string();
346+
llvm::json::Object c_rules;
347+
if (fs::exists(rule_dir / "src.c")) {
348+
Index(rule_dir / "src.c", /*is_c=*/true, dir_name, {}, c_rules);
349+
}
350+
std::vector<fs::path> cxx_srcs;
351+
for (const auto &entry : fs::directory_iterator(rule_dir)) {
352+
auto name = entry.path().filename().string();
353+
if (name == "src.c" || name == "src.cpp" ||
354+
(name.starts_with("src_cxx") && name.ends_with(".cpp"))) {
355+
cxx_srcs.push_back(entry.path());
341356
}
342-
bool is_c = lang == std::string("c");
357+
}
358+
std::ranges::sort(cxx_srcs);
359+
llvm::json::Object cxx_rules;
360+
for (const auto &src_path : cxx_srcs) {
361+
auto standard = SinceStandard(src_path);
343362
llvm::json::Object rules;
344-
for (const auto &rule_dir : rule_dirs) {
345-
for (const char *name : {"src.c", "src.cpp"}) {
346-
auto src_path = rule_dir / name;
347-
if (!fs::exists(src_path) || (is_c && src_path.extension() != ".c")) {
348-
continue;
349-
}
350-
llvm::errs() << "Indexing " << src_path.string() << " as " << lang
351-
<< '\n';
352-
Index(src_path, is_c, rule_dir.filename().string(),
353-
ReadCXXFlags(rule_dir), rules);
363+
Index(src_path, /*is_c=*/false, dir_name,
364+
{std::format("-std=c++{}", standard)}, rules);
365+
for (auto &[key, entries] : rules) {
366+
for (auto &entry : *entries.getAsArray()) {
367+
(*entry.getAsObject())["since"] = standard;
354368
}
355369
}
356-
WriteJson(index_dir / (std::string(lang) + ".json"),
357-
llvm::json::Object{{"common", std::move(common)},
358-
{"rules", std::move(rules)}});
370+
MergeRules(cxx_rules, std::move(rules));
371+
}
372+
WriteJson(out, llvm::json::Object{{"c", std::move(c_rules)},
373+
{"cpp", std::move(cxx_rules)}});
374+
}
375+
376+
void MergeIndexes(const std::vector<std::string> &parts,
377+
const std::vector<std::string> &common_headers,
378+
const fs::path &index_dir) {
379+
llvm::json::Object c_rules;
380+
llvm::json::Object cxx_rules;
381+
for (const auto &part : parts) {
382+
auto json = ReadJson(part);
383+
MergeRules(c_rules, std::move(*json.getObject("c")));
384+
MergeRules(cxx_rules, std::move(*json.getObject("cpp")));
385+
}
386+
std::string common;
387+
for (const auto &header : common_headers) {
388+
common += std::format("#include \"{}\"\n", header);
359389
}
390+
fs::create_directories(index_dir);
391+
WriteJson(index_dir / "c.json",
392+
llvm::json::Object{{"common", ""}, {"rules", std::move(c_rules)}});
393+
WriteJson(index_dir / "cpp.json",
394+
llvm::json::Object{{"common", std::move(common)},
395+
{"rules", std::move(cxx_rules)}});
360396
}
361397

362398
} // namespace cpp2rust
@@ -365,23 +401,35 @@ namespace {
365401

366402
llvm::cl::OptionCategory cat("cpp-rule-indexer options");
367403

368-
llvm::cl::list<std::string>
369-
Excluded("exclude", llvm::cl::desc("Rule directory that is not indexed"),
370-
llvm::cl::value_desc("rule-dir"), llvm::cl::ZeroOrMore,
371-
llvm::cl::cat(cat));
404+
llvm::cl::opt<std::string> Dir("dir", llvm::cl::desc("Rule directory to index"),
405+
llvm::cl::value_desc("rule-dir"),
406+
llvm::cl::cat(cat));
407+
408+
llvm::cl::list<std::string> Parts(llvm::cl::Positional,
409+
llvm::cl::desc("<indexes to merge>"),
410+
llvm::cl::ZeroOrMore, llvm::cl::cat(cat));
372411

373412
llvm::cl::list<std::string> CommonHeaders(
374413
"common-header", llvm::cl::desc("Header the C++ rules depend on"),
375414
llvm::cl::value_desc("header"), llvm::cl::ZeroOrMore, llvm::cl::cat(cat));
376415

416+
llvm::cl::opt<std::string>
417+
Out("o",
418+
llvm::cl::desc("Output part of a rule directory, or index directory"),
419+
llvm::cl::value_desc("path"), llvm::cl::Required, llvm::cl::cat(cat));
420+
377421
} // namespace
378422

379423
int main(int argc, char *argv[]) {
380424
llvm::cl::HideUnrelatedOptions(cat);
381425
llvm::cl::ParseCommandLineOptions(argc, argv);
382426

383-
cpp2rust::IndexRules(RULES_SRC_DIR, RULES_INDEX_DIR,
384-
{Excluded.begin(), Excluded.end()},
385-
{CommonHeaders.begin(), CommonHeaders.end()});
427+
if (!Dir.empty()) {
428+
cpp2rust::IndexDir(Dir.getValue(), Out.getValue());
429+
} else {
430+
cpp2rust::MergeIndexes({Parts.begin(), Parts.end()},
431+
{CommonHeaders.begin(), CommonHeaders.end()},
432+
Out.getValue());
433+
}
386434
return EXIT_SUCCESS;
387435
}

‎cpp2rust/rule_preprocessing/cpp_rule_preprocessor.cpp‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@
2727
#include <llvm/Support/MemoryBuffer.h>
2828
#include <llvm/Support/raw_ostream.h>
2929

30+
#include <algorithm>
3031
#include <cstdlib>
3132
#include <filesystem>
3233
#include <string>
34+
#include <vector>
3335

3436
#include "compat/platform_flags.h"
3537
#include "converter/converter_lib.h"
@@ -1120,14 +1122,26 @@ int main(int argc, char *argv[]) {
11201122
CXXFlags.end());
11211123
fs::path dir = SrcDir.getValue();
11221124
llvm::json::Object root;
1123-
for (const char *name : {"src.c", "src.cpp"}) {
1124-
auto path = dir / name;
1125-
if (!fs::exists(path)) {
1126-
continue;
1125+
std::vector<fs::path> paths;
1126+
for (const auto &entry : fs::directory_iterator(dir)) {
1127+
auto name = entry.path().filename().string();
1128+
if (name == "src.c" || name == "src.cpp" ||
1129+
(name.starts_with("src_cxx") && name.ends_with(".cpp"))) {
1130+
paths.push_back(entry.path());
11271131
}
1132+
}
1133+
std::ranges::sort(paths);
1134+
for (const auto &path : paths) {
11281135
llvm::errs() << "Preprocessing " << path.string() << '\n';
1136+
llvm::SmallVector<llvm::StringRef, 4> file_flags(cxx_flags);
1137+
auto stem = path.stem().string();
1138+
std::string standard_flag;
1139+
if (llvm::StringRef standard = stem; standard.consume_front("src_cxx")) {
1140+
standard_flag = "-std=c++" + standard.str();
1141+
file_flags.push_back(standard_flag);
1142+
}
11291143
llvm::json::Object file_root;
1130-
cpp2rust::Extract(path, file_root, cxx_flags);
1144+
cpp2rust::Extract(path, file_root, file_flags);
11311145
for (auto &[k, v] : file_root) {
11321146
if (!root.try_emplace(k, std::move(v)).second) {
11331147
llvm::errs() << "ERROR: rule name " << k.str()

‎docs/src/rules/format.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ library it covers (`rules/unistd`, `rules/vector`, `rules/string`, ...). It
55
contains:
66

77
- `src.cpp` and/or `src.c`: the C++ (or C) side of each rule.
8+
- `src_cxxNN.cpp` (optional): C++ rules that exist from C++NN on, for example
9+
`src_cxx17.cpp`; see below.
810
- `tgt_unsafe.rs`: the Rust targets for the unsafe model.
911
- `tgt_refcount.rs`: the Rust targets for the reference counting model
1012
(optional, see below).
@@ -141,6 +143,21 @@ for the C++ `__builtin_mul_overflow` (returning `bool`) while `src.c` defines
141143
`f12`/`f13` for the C version (returning `int`); their Rust bodies are
142144
identical.
143145

146+
## Standard versions
147+
148+
`src.cpp` is parsed as C++11 and its rules are available to every C++
149+
translation unit. A rule that needs a later standard goes into `src_cxxNN.cpp`,
150+
which is parsed as C++NN and whose rules are only loaded into translation units
151+
compiled as C++NN or later:
152+
153+
```cpp
154+
// rules/new/src_cxx17.cpp
155+
template <typename T1> T1 *f5(T1 *a0) { return std::launder(a0); }
156+
```
157+
158+
Rule names are shared across all the files of a module. Platform `#if`s are fine
159+
in any file; `#if __cplusplus` is not needed.
160+
144161
## The `rules` crate
145162
146163
The whole `rules/` tree is a single Rust crate. `rules/build.rs` walks the tree,

‎rules/builtin_types/cxxflags‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎rules/builtin_types/src.cpp‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
typedef decltype(nullptr) t0;
1414

1515
BUILTIN_TYPE(1, bool)
16-
#if __cplusplus >= 202002L
17-
BUILTIN_TYPE(5, char8_t)
18-
#endif
1916
BUILTIN_TYPE(8, char16_t)
2017
BUILTIN_TYPE(11, wchar_t)
2118
BUILTIN_TYPE(12, char32_t)

‎rules/builtin_types/src_cxx20.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
// Copyright (c) 2022-present INESC-ID.
2+
// Distributed under the MIT license that can be found in the LICENSE file.
3+
4+
typedef char8_t t50;
5+
typedef char8_t *t51;
6+
typedef const char8_t *t52;
7+
typedef volatile char8_t *t53;
8+
typedef const volatile char8_t *t54;

‎rules/compare/cxxflags‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

0 commit comments

Comments
 (0)