diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 020c6668fb9d3..dfff90d9702cb 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -720,6 +720,10 @@ pub(crate) fn optimize_and_codegen_thin_module( { let target = &*module.module_llvm.tm; let llmod = module.module_llvm.llmod(); + + let reloc_model = cgcx.relocation_model; + llvm::set_module_pic_and_pie_levels(llmod, reloc_model, &cgcx.crate_types); + save_temp_bitcode(cgcx, &module, "thin-lto-input"); // Up next comes the per-module local analyses that we do for Thin LTO. diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 64e1ae03bde76..15f754f2c8ec1 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -23,12 +23,11 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::Session; use rustc_session::config::{ - BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet, + BranchProtection, CFGuard, CFProtection, DebugInfo, FunctionReturn, PAuthKey, PacRet, }; use rustc_span::{DUMMY_SP, Span, Spanned, Symbol, sym}; use rustc_target::spec::{ - Arch, CfgAbi, Env, FramePointer, HasTargetSpec, Os, RelocModel, SmallDataThresholdSupport, - Target, TlsModel, + Arch, CfgAbi, Env, FramePointer, HasTargetSpec, Os, SmallDataThresholdSupport, Target, TlsModel, }; use smallvec::SmallVec; @@ -256,20 +255,7 @@ pub(crate) unsafe fn create_module<'ll>( } let reloc_model = sess.relocation_model(); - if matches!(reloc_model, RelocModel::Pic | RelocModel::Pie) { - unsafe { - llvm::LLVMRustSetModulePICLevel(llmod); - } - // PIE is potentially more effective than PIC, but can only be used in executables. - // If all our outputs are executables, then we can relax PIC to PIE. - if reloc_model == RelocModel::Pie - || tcx.crate_types().iter().all(|ty| *ty == CrateType::Executable) - { - unsafe { - llvm::LLVMRustSetModulePIELevel(llmod); - } - } - } + llvm::set_module_pic_and_pie_levels(llmod, reloc_model, tcx.crate_types()); // Linking object files with different code models is undefined behavior // because the compiler would have to generate additional code (to span @@ -519,6 +505,15 @@ pub(crate) unsafe fn create_module<'ll>( ); } + if let Some(direct_access_external_data) = sess.direct_access_external_data() { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Max, + "direct-access-external-data", + direct_access_external_data as u32, + ); + } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) { // Set up the small-data optimization limit for architectures that use diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 2ec19b1795b5a..6ae4d6b0da4f0 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -8,6 +8,7 @@ use std::string::FromUtf8Error; use libc::c_uint; use rustc_abi::{AddressSpace, Align, Size, WrappingRange}; use rustc_llvm::RustString; +use rustc_session::config::CrateType; pub(crate) use self::CallConv::*; pub(crate) use self::CodeGenOptSize::*; @@ -475,3 +476,27 @@ pub(crate) fn add_alias<'ll>( ) -> &'ll Value { unsafe { LLVMAddAlias2(module, ty, address_space.0, aliasee, name.as_ptr()) } } + +/// Safe wrapper for setting the PIC and PIE levels via +/// `LLVMRustSetModulePICLevel` and `LLVMRustSetModulePIELevel`. +pub(crate) fn set_module_pic_and_pie_levels( + llmod: &Module, + reloc_model: rustc_target::spec::RelocModel, + crate_types: &[CrateType], +) { + if matches!( + reloc_model, + rustc_target::spec::RelocModel::Pic | rustc_target::spec::RelocModel::Pie + ) { + unsafe { + LLVMRustSetModulePICLevel(llmod); + } + if reloc_model == rustc_target::spec::RelocModel::Pie + || crate_types.iter().all(|ty| *ty == CrateType::Executable) + { + unsafe { + LLVMRustSetModulePIELevel(llmod); + } + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 112cf45ebbf2e..1b392d9e80ff0 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -30,7 +30,7 @@ use rustc_session::config::{ }; use rustc_span::source_map::SourceMap; use rustc_span::{FileName, InnerSpan, Span, SpanData}; -use rustc_target::spec::{MergeFunctions, SanitizerSet}; +use rustc_target::spec::{MergeFunctions, RelocModel, SanitizerSet}; use tracing::debug; use crate::back::link::ensure_removed; @@ -340,6 +340,7 @@ pub struct CodegenContext { pub split_debuginfo: rustc_target::spec::SplitDebuginfo, pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, pub pointer_size: Size, + pub relocation_model: RelocModel, /// LLVM optimizations for which we want to print remarks. pub remark: Passes, @@ -1281,6 +1282,7 @@ fn start_executing_work( split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind, parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend, pointer_size: tcx.data_layout.pointer_size(), + relocation_model: sess.relocation_model(), }; // This is the "main loop" of parallel work happening for parallel codegen. diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 5dcaa5f6f84b8..6564bb0858c26 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1113,11 +1113,17 @@ LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, } extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { - unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); + Module *Mod = unwrap(M); + if (!Mod->getModuleFlag("PIC Level")) { + Mod->setPICLevel(PICLevel::Level::BigPIC); + } } extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { - unwrap(M)->setPIELevel(PIELevel::Level::Large); + Module *Mod = unwrap(M); + if (!Mod->getModuleFlag("PIE Level")) { + Mod->setPIELevel(PIELevel::Level::Large); + } } extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 87c40fa588c03..6e09dfdee2eb2 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -934,6 +934,7 @@ crate::target_spec_enum! { } crate::target_spec_enum! { + #[derive(Encodable, Decodable)] pub enum RelocModel { Static = "static", Pic = "pic", diff --git a/tests/codegen-llvm/direct-access-external-data.rs b/tests/codegen-llvm/direct-access-external-data.rs index a151bb6012e1e..092d8a692c7f2 100644 --- a/tests/codegen-llvm/direct-access-external-data.rs +++ b/tests/codegen-llvm/direct-access-external-data.rs @@ -47,3 +47,10 @@ pub fn refer() { core::hint::black_box(EXTERNAL); core::hint::black_box(WEAK); } + +// DIRECT: !{{[0-9]+}} = !{i32 7, !"direct-access-external-data", i32 1} + +// INDIRECT: !{{[0-9]+}} = !{i32 7, !"direct-access-external-data", i32 0} + +// DEFAULT-NOT: direct-access-external-data +// PIE-NOT: direct-access-external-data diff --git a/tests/run-make/lto-direct-access-external-data/dep.rs b/tests/run-make/lto-direct-access-external-data/dep.rs new file mode 100644 index 0000000000000..02ca2d23ddb00 --- /dev/null +++ b/tests/run-make/lto-direct-access-external-data/dep.rs @@ -0,0 +1,25 @@ +#![feature(no_core, lang_items)] +#![no_std] +#![no_core] +#![crate_type = "lib"] + +#[lang = "pointee_sized"] +trait PointeeSized {} +#[lang = "meta_sized"] +trait MetaSized: PointeeSized {} +#[lang = "sized"] +trait Sized: MetaSized {} + +#[lang = "copy"] +pub trait Copy {} + +impl Copy for i32 {} + +unsafe extern "C" { + pub safe static VAR: i32; +} + +#[no_mangle] +pub fn refer_dep() -> i32 { + unsafe { VAR } +} diff --git a/tests/run-make/lto-direct-access-external-data/lib.rs b/tests/run-make/lto-direct-access-external-data/lib.rs new file mode 100644 index 0000000000000..618fcaed03a5f --- /dev/null +++ b/tests/run-make/lto-direct-access-external-data/lib.rs @@ -0,0 +1,65 @@ +#![feature(no_core, lang_items)] +#![no_std] +#![no_core] + +extern crate dep; + +unsafe extern "C" { + pub safe static VAR: i32; +} + +pub mod mod_0 { + use super::VAR; + #[no_mangle] + pub fn refer_0() -> i32 { + VAR + } +} + +pub mod mod_1 { + use super::VAR; + #[no_mangle] + pub fn refer_1() -> i32 { + VAR + } +} + +#[no_mangle] +pub fn call_dep() -> i32 { + dep::refer_dep() +} + +// DEFAULT: @VAR = {{.*}}dso_local{{.*}}global i32 +// DEFAULT-NOT: direct-access-external-data +// DEFAULT-NOT: PIE Level + +// PIE: @VAR = external +// PIE-NOT: dso_local +// PIE-SAME: global i32 +// PIE-NOT: direct-access-external-data +// PIE: !{{[0-9]+}} = !{i32 7, !"PIE Level", i32 2} + +// DIRECT: @VAR = {{.*}}dso_local{{.*}}global i32 +// DIRECT-DAG: !{{[0-9]+}} = !{i32 7, !"direct-access-external-data", i32 1} +// DIRECT-DAG: !{{[0-9]+}} = !{i32 7, !"PIE Level", i32 2} + +// INDIRECT: @VAR = external +// INDIRECT-NOT: dso_local +// INDIRECT-SAME: global i32 +// INDIRECT: !{{[0-9]+}} = !{i32 7, !"direct-access-external-data", i32 0} +// INDIRECT-NOT: PIE Level + +// DEP-PIE: !{{[0-9]+}} = !{i32 7, !"PIE Level", i32 2} + +// For Full & Thin LTO we expect direct PC-relative access. +// DIRECT-RELOC-NOT: R_X86_64_GOTPCREL{{.*}}VAR +// DIRECT-RELOC: R_X86_64_PC32{{.*}}VAR + +// For indirect cases, we always expect GOT indirection. +// INDIRECT-RELOC: R_X86_64_GOTPCREL{{.*}}VAR +// INDIRECT-RELOC-NOT: R_X86_64_PC32{{.*}}VAR + +// Default PIE without direct-access enabled should use GOT indirection. +// This is correct and is not changed by setting PIE Level consistently. +// PIE-RELOC: R_X86_64_GOTPCREL{{.*}}VAR +// PIE-RELOC-NOT: R_X86_64_PC32{{.*}}VAR diff --git a/tests/run-make/lto-direct-access-external-data/rmake.rs b/tests/run-make/lto-direct-access-external-data/rmake.rs new file mode 100644 index 0000000000000..9f6958ddf9c48 --- /dev/null +++ b/tests/run-make/lto-direct-access-external-data/rmake.rs @@ -0,0 +1,176 @@ +//@ needs-llvm-components: x86 +//@ ignore-loongarch64 (handles dso_local differently) +//@ ignore-powerpc64 (handles dso_local differently) +//@ ignore-apple (handles dso_local differently) + +// Test the various interleavings of LTO and relocation model. +// We know that in some cases the -Zdirect-access-external-data option will not +// result in the correct code generation due to the module having +// inconsistently set PIE Level. Test those cases so that the result is clear +// when future patches fix the problem. + +use run_make_support::{ + cwd, has_extension, llvm_dis, llvm_filecheck, llvm_readobj, rfs, rustc, shallow_find_files, +}; + +struct TestCase { + lto: &'static str, + reloc: &'static str, + direct_access: Option<&'static str>, + expected_prefix: &'static str, + dep_prefixes: &'static [&'static str], +} + +fn main() { + let test_cases = [ + TestCase { + lto: "thin", + reloc: "static", + direct_access: None, + expected_prefix: "DEFAULT", + dep_prefixes: &[], + }, + TestCase { + lto: "fat", + reloc: "static", + direct_access: None, + expected_prefix: "DEFAULT", + dep_prefixes: &[], + }, + TestCase { + lto: "thin", + reloc: "pie", + direct_access: None, + expected_prefix: "PIE", + dep_prefixes: &["DEP-PIE"], + }, + TestCase { + lto: "fat", + reloc: "pie", + direct_access: None, + expected_prefix: "PIE", + dep_prefixes: &[], + }, + TestCase { + lto: "thin", + reloc: "pie", + direct_access: Some("yes"), + expected_prefix: "DIRECT", + dep_prefixes: &["DEP-PIE"], + }, + TestCase { + lto: "fat", + reloc: "pie", + direct_access: Some("yes"), + expected_prefix: "DIRECT", + dep_prefixes: &[], + }, + TestCase { + lto: "thin", + reloc: "static", + direct_access: Some("no"), + expected_prefix: "INDIRECT", + dep_prefixes: &[], + }, + TestCase { + lto: "fat", + reloc: "static", + direct_access: Some("no"), + expected_prefix: "INDIRECT", + dep_prefixes: &[], + }, + ]; + + for case in test_cases { + // Remove all output files that are not source Rust code for cleanup. + for file in shallow_find_files(cwd(), |path| !has_extension(path, "rs")) { + rfs::remove_file(file); + } + + let mut cmd = rustc(); + cmd.input("dep.rs") + .target("x86_64-unknown-linux-gnu") + .crate_type("rlib") + .crate_name("dep") + .arg("-Cpanic=abort"); + + if let Some(da) = case.direct_access { + cmd.arg(format!("-Zdirect-access-external-data={da}")); + } + cmd.run(); + + cmd = rustc(); + cmd.input("lib.rs") + .target("x86_64-unknown-linux-gnu") + .crate_type("staticlib") + .crate_name("lto_direct_access_test") + .extern_("dep", "libdep.rlib") + .arg("-Cpanic=abort") + .arg("-Csave-temps") + .arg(format!("-Clto={}", case.lto)) + .arg(format!("-Crelocation-model={}", case.reloc)) + .arg("-Ccodegen-units=2"); + + if let Some(da) = case.direct_access { + cmd.arg(format!("-Zdirect-access-external-data={da}")); + } + cmd.run(); + + let suffix = if case.lto == "thin" { + ".rcgu.thin-lto-after-pm.bc" + } else { + ".rcgu.lto.after-restriction.bc" + }; + + let bc_files = shallow_find_files(".", |path| { + let name = path.file_name().unwrap().to_str().unwrap(); + name.starts_with("lto_direct_access_test.lto_direct_access_test.") + && name.ends_with(suffix) + }); + assert!(!bc_files.is_empty(), "No expected bitcode files were generated"); + + for bc_file in &bc_files { + llvm_dis().input(bc_file).run(); + let ll_file = bc_file.with_extension("ll"); + llvm_filecheck() + .input_file(&ll_file) + .patterns("lib.rs") + .check_prefix(case.expected_prefix) + .run(); + } + + let relocs = llvm_readobj().arg("--relocations").input("liblto_direct_access_test.a").run(); + let relocs_out = "relocs.txt"; + rfs::write(relocs_out, relocs.stdout_utf8()); + + let reloc_prefix = match case.expected_prefix { + "DIRECT" => Some("DIRECT-RELOC"), + "PIE" => Some("PIE-RELOC"), + "INDIRECT" => Some("INDIRECT-RELOC"), + _ => None, + }; + + if let Some(prefix) = reloc_prefix { + llvm_filecheck().input_file(relocs_out).patterns("lib.rs").check_prefix(prefix).run(); + } + + if !case.dep_prefixes.is_empty() { + let dep_bc_files = shallow_find_files(".", |path| { + let name = path.file_name().unwrap().to_str().unwrap(); + name.contains("dep") && name.ends_with(suffix) + }); + assert!(!dep_bc_files.is_empty(), "dep bitcode files not found"); + for dep_bc in &dep_bc_files { + llvm_dis().input(dep_bc).run(); + let dep_ll = dep_bc.with_extension("ll"); + for dep_cp in case.dep_prefixes { + llvm_filecheck() + .input_file(&dep_ll) + .patterns("lib.rs") + .check_prefix(dep_cp) + .run(); + } + } + } + } +}