Skip to content

Commit 4b39f92

Browse files
joaotgouveiajoaotgouveia
authored
Make rule-preprocessor process a single directory at a time (#372)
Changes `rule-preprocessor` to process a single directory per invocation, making it possible to parallelize. This matches what is already the case for `cpp-rule-preprocessor`. Additionally, this drops `modules.rs` from source control.
1 parent 67b0540 commit 4b39f92

8 files changed

Lines changed: 175 additions & 271 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 66 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -141,18 +141,53 @@ set(RULES_IR_DIR "${CMAKE_BINARY_DIR}/rules")
141141

142142
set(RUSTLS_HEADER "${PROJECT_SOURCE_DIR}/rules/rustls/rustls.h")
143143

144+
set(PREPROCESSOR_BIN "${CMAKE_BINARY_DIR}/target_preprocessor/release/rule-preprocessor")
145+
set(PREPROCESSOR_STAMP "${CMAKE_BINARY_DIR}/.rule-preprocessor.stamp")
146+
147+
file(GLOB_RECURSE rule_preprocessor_sources
148+
${PROJECT_SOURCE_DIR}/rule-preprocessor/src/*.rs)
149+
150+
execute_process(
151+
COMMAND ${CMAKE_COMMAND} -E env RUSTUP_TOOLCHAIN=${RUST_NIGHTLY_VERSION}
152+
rustc --print sysroot
153+
OUTPUT_VARIABLE NIGHTLY_SYSROOT
154+
OUTPUT_STRIP_TRAILING_WHITESPACE
155+
)
156+
157+
add_custom_command(
158+
OUTPUT ${RUSTLS_HEADER} ${PREPROCESSOR_STAMP}
159+
COMMAND cargo +${RUST_STABLE_VERSION} build --release
160+
--manifest-path "${PROJECT_SOURCE_DIR}/rules/Cargo.toml"
161+
COMMAND ${CMAKE_COMMAND} -E touch "${PREPROCESSOR_STAMP}"
162+
DEPENDS ${rule_preprocessor_sources}
163+
"${PROJECT_SOURCE_DIR}/rules/build.rs"
164+
"${PROJECT_SOURCE_DIR}/rules/Cargo.toml"
165+
"${RUST_STAMP_FILE}"
166+
VERBATIM
167+
)
168+
169+
add_custom_command(
170+
OUTPUT ${PREPROCESSOR_BIN}
171+
COMMAND ${CMAKE_COMMAND} -E env
172+
CARGO_TARGET_DIR=${CMAKE_BINARY_DIR}/target_preprocessor
173+
cargo +${RUST_NIGHTLY_VERSION} build --release
174+
--manifest-path "${PROJECT_SOURCE_DIR}/rule-preprocessor/Cargo.toml"
175+
DEPENDS ${rule_preprocessor_sources}
176+
"${RUST_STAMP_FILE}"
177+
VERBATIM
178+
)
179+
144180
file(GLOB rule_subdirs ${PROJECT_SOURCE_DIR}/rules/*)
145181
set(cpp_rules_ir_outputs)
146182
set(rust_rules_ir_outputs)
147-
set(rust_rules_inputs)
148183
foreach(_rule_dir IN LISTS rule_subdirs)
149-
file(GLOB _srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
150-
if(NOT _srcs)
184+
file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
185+
if(NOT _cpp_srcs)
151186
continue()
152187
endif()
153188
get_filename_component(_rule_name ${_rule_dir} NAME)
154189
set(_out_dir ${RULES_IR_DIR}/${_rule_name})
155-
set(_out ${_out_dir}/ir_src.json)
190+
set(_cpp_out ${_out_dir}/ir_src.json)
156191
set(_cxxflags)
157192
set(_cxxflags_file)
158193
if(EXISTS ${_rule_dir}/cxxflags)
@@ -163,39 +198,44 @@ foreach(_rule_dir IN LISTS rule_subdirs)
163198
endforeach()
164199
endif()
165200
add_custom_command(
166-
OUTPUT ${_out}
201+
OUTPUT ${_cpp_out}
167202
COMMAND ${CMAKE_COMMAND} -E make_directory ${_out_dir}
168-
COMMAND $<TARGET_FILE:cpp-rule-preprocessor> --dir ${_rule_dir} --out ${_out} ${_cxxflags}
169-
DEPENDS ${_srcs} ${_cxxflags_file} ${PROJECT_SOURCE_DIR}/cpp2rust/cpp_rule_preprocessor.cpp ${RUSTLS_HEADER}
203+
COMMAND $<TARGET_FILE:cpp-rule-preprocessor> --dir ${_rule_dir} --out ${_cpp_out} ${_cxxflags}
204+
DEPENDS ${_cpp_srcs} ${_cxxflags_file} ${PROJECT_SOURCE_DIR}/cpp2rust/cpp_rule_preprocessor.cpp ${RUSTLS_HEADER}
170205
VERBATIM
171206
)
172-
list(APPEND cpp_rules_ir_outputs ${_out})
207+
list(APPEND cpp_rules_ir_outputs ${_cpp_out})
208+
209+
set(_rust_srcs)
210+
set(_rust_out)
173211
if(EXISTS ${_rule_dir}/tgt_unsafe.rs)
174-
list(APPEND rust_rules_ir_outputs ${_out_dir}/ir_unsafe.json)
175-
list(APPEND rust_rules_inputs ${_rule_dir}/tgt_unsafe.rs)
212+
list(APPEND _rust_out ${_out_dir}/ir_unsafe.json)
213+
list(APPEND _rust_srcs ${_rule_dir}/tgt_unsafe.rs)
176214
endif()
177215
if(EXISTS ${_rule_dir}/tgt_refcount.rs)
178-
list(APPEND rust_rules_ir_outputs ${_out_dir}/ir_refcount.json)
179-
list(APPEND rust_rules_inputs ${_rule_dir}/tgt_refcount.rs)
216+
list(APPEND _rust_out ${_out_dir}/ir_refcount.json)
217+
list(APPEND _rust_srcs ${_rule_dir}/tgt_refcount.rs)
180218
endif()
219+
if(NOT _rust_srcs)
220+
continue()
221+
endif()
222+
add_custom_command(
223+
OUTPUT ${_rust_out}
224+
COMMAND ${CMAKE_COMMAND} -E make_directory ${_out_dir}
225+
COMMAND ${CMAKE_COMMAND} -E env
226+
CARGO_TARGET_DIR=${CMAKE_BINARY_DIR}/target_preprocessor
227+
RUSTUP_TOOLCHAIN=${RUST_NIGHTLY_VERSION}
228+
LD_LIBRARY_PATH=${NIGHTLY_SYSROOT}/lib
229+
DYLD_LIBRARY_PATH=${NIGHTLY_SYSROOT}/lib
230+
${PREPROCESSOR_BIN} ${_out_dir} ${_rule_dir}
231+
DEPENDS ${_rust_srcs} ${PREPROCESSOR_BIN} ${PREPROCESSOR_STAMP}
232+
VERBATIM
233+
)
234+
list(APPEND rust_rules_ir_outputs ${_rust_out})
181235
endforeach()
182236

183237
add_custom_target("preprocess-cpp-rules" ALL DEPENDS ${cpp_rules_ir_outputs})
184238

185-
file(GLOB_RECURSE rule_preprocessor_sources
186-
${PROJECT_SOURCE_DIR}/rule-preprocessor/src/*.rs)
187-
188-
add_custom_command(
189-
OUTPUT ${rust_rules_ir_outputs} ${RUSTLS_HEADER}
190-
COMMAND cargo +${RUST_STABLE_VERSION} build --release --manifest-path "${PROJECT_SOURCE_DIR}/rules/Cargo.toml"
191-
COMMAND ${CMAKE_COMMAND} -E env
192-
CARGO_TARGET_DIR=${CMAKE_BINARY_DIR}/target_preprocessor
193-
cargo +${RUST_NIGHTLY_VERSION} run --release --manifest-path "${PROJECT_SOURCE_DIR}/rule-preprocessor/Cargo.toml"
194-
-- "${RULES_IR_DIR}"
195-
DEPENDS ${rust_rules_inputs} ${rule_preprocessor_sources} "${RUST_STAMP_FILE}"
196-
VERBATIM
197-
)
198-
199239
add_custom_target("preprocess-rust-rules" ALL
200240
DEPENDS ${rust_rules_ir_outputs})
201241

‎rule-preprocessor/src/ir.rs‎

Lines changed: 54 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,32 @@
33

44
use serde::{Deserialize, Serialize};
55
use std::collections::{BTreeMap, HashMap};
6+
use std::iter::Flatten;
67
use std::path::{Path, PathBuf};
78

9+
pub enum Model {
10+
Refcount,
11+
Unsafe,
12+
}
13+
14+
impl Model {
15+
pub const ALL: [Model; 2] = [Model::Refcount, Model::Unsafe];
16+
17+
pub fn src_filename(self) -> &'static str {
18+
match self {
19+
Model::Refcount => "tgt_refcount.rs",
20+
Model::Unsafe => "tgt_unsafe.rs",
21+
}
22+
}
23+
24+
pub fn ir_filename(self) -> &'static str {
25+
match self {
26+
Model::Refcount => "ir_refcount.json",
27+
Model::Unsafe => "ir_unsafe.json",
28+
}
29+
}
30+
}
31+
832
fn validate_consecutive_keys<'a>(
933
keys: impl Iterator<Item = &'a String>,
1034
prefix: char,
@@ -231,30 +255,44 @@ pub enum RuleIr {
231255
/// Per-file IR: rule name -> FnIr or TypeIr
232256
pub type FileIr = BTreeMap<String, RuleIr>;
233257

234-
/// All IR for all rule files.
258+
/// Per-dir IR
235259
pub struct RulesIR {
236-
pub all_ir: HashMap<String, FileIr>,
237-
pub crate_root: PathBuf,
260+
pub dir: PathBuf,
261+
pub refcount_ir: Option<FileIr>,
262+
pub unsafe_ir: Option<FileIr>,
263+
}
264+
265+
impl<'a> IntoIterator for &'a RulesIR {
266+
type Item = (Model, &'a FileIr);
267+
type IntoIter = Flatten<std::array::IntoIter<Option<Self::Item>, 2>>;
268+
269+
fn into_iter(self) -> Self::IntoIter {
270+
[
271+
self.refcount_ir.as_ref().map(|ir| (Model::Refcount, ir)),
272+
self.unsafe_ir.as_ref().map(|ir| (Model::Unsafe, ir)),
273+
]
274+
.into_iter()
275+
.flatten()
276+
}
238277
}
239278

240279
impl RulesIR {
241280
pub fn write_ir(&self, out_dir: &Path) {
242-
for (rule_path, file_ir) in &self.all_ir {
243-
let rule_path = Path::new(rule_path);
244-
let rule_name = rule_path.parent().unwrap().file_name().unwrap();
245-
let json_name = rule_path
246-
.file_name()
247-
.unwrap()
248-
.to_str()
249-
.unwrap()
250-
.replace("tgt_", "ir_")
251-
.replace(".rs", ".json");
252-
let json_path = out_dir.join(rule_name).join(json_name);
253-
254-
std::fs::create_dir_all(json_path.parent().unwrap()).unwrap();
281+
for (model, file_ir) in self {
282+
let json_path = out_dir.join(model.ir_filename());
255283
let json = serde_json::to_string_pretty(file_ir).unwrap();
256284
std::fs::write(&json_path, format!("{json}\n")).unwrap();
257285
println!("{}", json_path.display());
258286
}
259287
}
288+
289+
pub fn get_mut(&mut self, file: &str) -> Option<&mut FileIr> {
290+
if file.ends_with(Model::Refcount.src_filename()) {
291+
self.refcount_ir.as_mut()
292+
} else if file.ends_with(Model::Unsafe.src_filename()) {
293+
self.unsafe_ir.as_mut()
294+
} else {
295+
None
296+
}
297+
}
260298
}

‎rule-preprocessor/src/main.rs‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,15 @@ fn main() {
2222
let mut args = std::env::args().skip(1);
2323
let out_dir = args
2424
.next()
25-
.expect("usage: rule-preprocessor <out-dir> [rules-crate-dir]");
26-
let in_dir = args.next().unwrap_or_else(|| "../rules".to_string());
27-
SemanticAnalysis::run(SyntacticAnalysis::run(
28-
&std::fs::canonicalize(&in_dir).unwrap(),
29-
))
30-
.write_ir(&std::path::PathBuf::from(out_dir));
25+
.expect("usage: rule-preprocessor <out-dir> <rule-dir>");
26+
let in_dir = args
27+
.next()
28+
.expect("usage: rule-preprocessor <out-dir> <rule-dir>");
29+
30+
let out = std::path::PathBuf::from(out_dir);
31+
SemanticAnalysis::run(
32+
SyntacticAnalysis::run(&std::fs::canonicalize(&in_dir).unwrap()),
33+
&out,
34+
)
35+
.write_ir(&out);
3136
}

‎rule-preprocessor/src/semantic.rs‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
// Copyright (c) 2022-present INESC-ID.
22
// Distributed under the MIT license that can be found in the LICENSE file.
33

4-
use crate::ir::RulesIR;
5-
use crate::ir::{Access, FnIr, RuleIr};
4+
use crate::ir::{Access, FnIr, RuleIr, RulesIR};
65
use std::collections::HashMap;
76
use std::path::{Path, PathBuf};
87

98
pub struct SemanticAnalysis;
109

1110
impl SemanticAnalysis {
12-
pub fn run(ir: RulesIR) -> RulesIR {
13-
let args = build_rustc_args(&ir.crate_root);
11+
pub fn run(ir: RulesIR, out_dir: &Path) -> RulesIR {
12+
let crate_root = write_crate_root(&ir, out_dir);
13+
let args = build_rustc_args(&crate_root);
1414
let mut resolver = MethodResolver { ir };
1515

1616
if rustc_driver::catch_fatal_errors(|| {
@@ -26,14 +26,30 @@ impl SemanticAnalysis {
2626
}
2727
}
2828

29+
fn write_crate_root(ir: &RulesIR, out_dir: &Path) -> PathBuf {
30+
let mut buf = String::from("#![allow(warnings)]\n");
31+
for (model, _) in ir {
32+
let filename = model.src_filename();
33+
let stem = filename.strip_suffix(".rs").unwrap();
34+
let source = ir.dir.join(filename);
35+
buf.push_str(&format!(
36+
"#[path = r#\"{}\"#]\npub mod rule_{stem};\n",
37+
source.display()
38+
));
39+
}
40+
41+
let path = out_dir.join("crate_root.rs");
42+
std::fs::write(&path, buf).unwrap();
43+
path
44+
}
45+
2946
fn build_rustc_args(crate_root: &Path) -> Vec<String> {
3047
let sysroot = get_sysroot();
31-
let lib_path = crate_root.join("src").join("lib.rs");
3248
let build_dir = find_build_dir();
3349

3450
let mut args = vec![
3551
"rustc".to_string(),
36-
lib_path.to_string_lossy().to_string(),
52+
crate_root.to_string_lossy().to_string(),
3753
"--crate-name".to_string(),
3854
"rules".to_string(),
3955
"--crate-type".to_string(),
@@ -217,7 +233,7 @@ struct MethodResolver {
217233

218234
impl MethodResolver {
219235
fn resolve_rule<'tcx>(&mut self, tcx: rustc_middle::ty::TyCtxt<'tcx>, f: &FnDecl<'tcx>) {
220-
let Some(file_ir) = self.ir.all_ir.get_mut(&f.source_file) else {
236+
let Some(file_ir) = self.ir.get_mut(&f.source_file) else {
221237
return;
222238
};
223239
match file_ir.get_mut(&f.name) {
@@ -231,14 +247,14 @@ impl MethodResolver {
231247
}
232248

233249
fn assert_no_unknowns(&self) {
234-
for (source_file, file_ir) in &self.ir.all_ir {
250+
for (model, file_ir) in &self.ir {
235251
for (rule_name, rule) in file_ir {
236252
let RuleIr::Fn(fn_ir) = rule else { continue };
237253
assert!(
238254
!fn_ir.has_unknowns(),
239255
"unresolved access=\"unknown\" in {} ({})",
240256
rule_name,
241-
source_file
257+
self.ir.dir.join(model.src_filename()).display()
242258
);
243259
}
244260
}

‎rule-preprocessor/src/syntactic.rs‎

Lines changed: 14 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ use cfg_expr::Expression;
55
use cfg_expr::expr::{Predicate, TargetPredicate};
66
use ra_ap_syntax::ast::{HasAttrs, HasGenericParams, HasName, HasTypeBounds};
77
use ra_ap_syntax::{AstNode, SyntaxKind, ast, match_ast};
8-
use std::collections::{BTreeMap, HashMap};
8+
use std::collections::BTreeMap;
99
use std::path::{Path, PathBuf};
1010

1111
use crate::ir::{
12-
Access, BodyFragment, FileIr, FnIr, MethodCallInner, PlaceholderInner, RuleIr, RulesIR,
12+
Access, BodyFragment, FileIr, FnIr, MethodCallInner, Model, PlaceholderInner, RuleIr, RulesIR,
1313
TypeInfo, TypeIr,
1414
};
1515

@@ -87,46 +87,23 @@ fn cfg_matches_host(fn_item: &ast::Fn) -> bool {
8787
pub struct SyntacticAnalysis;
8888

8989
impl SyntacticAnalysis {
90-
pub fn run(crate_root: &Path) -> RulesIR {
91-
let rule_files = Self::collect_rule_files(crate_root);
92-
let mut all_ir = HashMap::new();
93-
94-
for rule_file in &rule_files {
95-
let source = std::fs::read_to_string(rule_file).unwrap();
96-
let file_ir = Self::parse_rule_file(&source, rule_file);
97-
98-
let canonical = rule_file
99-
.canonicalize()
100-
.unwrap_or_else(|_| rule_file.clone())
101-
.to_string_lossy()
102-
.to_string();
103-
all_ir.insert(canonical, file_ir);
104-
}
90+
pub fn run(rule_dir: &Path) -> RulesIR {
91+
let [refcount_ir, unsafe_ir] = Self::collect_rule_files(rule_dir).map(|opt| {
92+
opt.map(|f| Self::parse_rule_file(&std::fs::read_to_string(&f).unwrap(), &f))
93+
});
10594

10695
RulesIR {
107-
all_ir,
108-
crate_root: crate_root.to_path_buf(),
96+
dir: rule_dir.to_path_buf(),
97+
refcount_ir,
98+
unsafe_ir,
10999
}
110100
}
111101

112-
fn collect_rule_files(dir: &Path) -> Vec<PathBuf> {
113-
let mut out = Vec::new();
114-
let Ok(entries) = std::fs::read_dir(dir) else {
115-
return out;
116-
};
117-
for entry in entries.flatten() {
118-
let path = entry.path();
119-
if path.is_dir() {
120-
out.extend(Self::collect_rule_files(&path));
121-
} else if let Some(name) = path.file_name().and_then(|s| s.to_str())
122-
&& name.starts_with("tgt_")
123-
&& name.ends_with(".rs")
124-
{
125-
out.push(path);
126-
}
127-
}
128-
out.sort();
129-
out
102+
fn collect_rule_files(dir: &Path) -> [Option<PathBuf>; 2] {
103+
Model::ALL.map(|m| {
104+
let p = dir.join(m.src_filename());
105+
p.exists().then_some(p)
106+
})
130107
}
131108

132109
fn parse_rule_file(source: &str, path: &Path) -> FileIr {

0 commit comments

Comments
 (0)