Split exported_symbols for generic and non-generic symbols

This reduces metadata decoder overhead during the monomorphization collector.

Author: bjorn3

compiler/rustc_codegen_ssa/src/back/linker.rs

@@ -1794,7 +1794,10 @@ fn for_each_exported_symbols_include_dep<'tcx>(
     for (cnum, dep_format) in deps.iter_enumerated() {
         // For each dependency that we are linking to statically ...
         if *dep_format == Linkage::Static {
-            for &(symbol, info) in tcx.exported_symbols(cnum).iter() {
+            for &(symbol, info) in tcx.exported_non_generic_symbols(cnum).iter() {
+                callback(symbol, info, cnum);
+            }
+            for &(symbol, info) in tcx.exported_generic_symbols(cnum).iter() {
                 callback(symbol, info, cnum);
             }
         }

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -164,7 +164,8 @@ fn is_reachable_non_generic_provider_extern(tcx: TyCtxt<'_>, def_id: DefId) -> b
     tcx.reachable_non_generics(def_id.krate).contains_key(&def_id)
 }
 
-fn exported_symbols_provider_local<'tcx>(
+// FIXME maybe split into two functions
+fn exported_symbols_provider_local<'tcx, const GENERIC: bool>(
     tcx: TyCtxt<'tcx>,
     _: LocalCrate,
 ) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
@@ -174,15 +175,19 @@ fn exported_symbols_provider_local<'tcx>(
 
     // FIXME: Sorting this is unnecessary since we are sorting later anyway.
     //        Can we skip the later sorting?
-    let sorted = tcx.with_stable_hashing_context(|hcx| {
-        tcx.reachable_non_generics(LOCAL_CRATE).to_sorted(&hcx, true)
-    });
+    let sorted = if GENERIC {
+        vec![]
+    } else {
+        tcx.with_stable_hashing_context(|hcx| {
+            tcx.reachable_non_generics(LOCAL_CRATE).to_sorted(&hcx, true)
+        })
+    };
 
     let mut symbols: Vec<_> =
         sorted.iter().map(|&(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect();
 
     // Export TLS shims
-    if !tcx.sess.target.dll_tls_export {
+    if !GENERIC && !tcx.sess.target.dll_tls_export {
         symbols.extend(sorted.iter().filter_map(|&(&def_id, &info)| {
             tcx.needs_thread_local_shim(def_id).then(|| {
                 (
@@ -197,7 +202,7 @@ fn exported_symbols_provider_local<'tcx>(
         }))
     }
 
-    if tcx.entry_fn(()).is_some() {
+    if !GENERIC && tcx.entry_fn(()).is_some() {
         let exported_symbol =
             ExportedSymbol::NoDefId(SymbolName::new(tcx, tcx.sess.target.entry_name.as_ref()));
 
@@ -212,7 +217,7 @@ fn exported_symbols_provider_local<'tcx>(
     }
 
     // Mark allocator shim symbols as exported only if they were generated.
-    if allocator_kind_for_codegen(tcx).is_some() {
+    if !GENERIC && allocator_kind_for_codegen(tcx).is_some() {
         for symbol_name in ALLOCATOR_METHODS
             .iter()
             .map(|method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str()))
@@ -235,7 +240,7 @@ fn exported_symbols_provider_local<'tcx>(
         }
     }
 
-    if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() {
+    if !GENERIC && tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() {
         // These are weak symbols that point to the profile version and the
         // profile name, which need to be treated as exported so LTO doesn't nix
         // them.
@@ -255,7 +260,7 @@ fn exported_symbols_provider_local<'tcx>(
         }));
     }
 
-    if tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) {
+    if !GENERIC && tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) {
         let mut msan_weak_symbols = Vec::new();
 
         // Similar to profiling, preserve weak msan symbol during LTO.
@@ -280,8 +285,9 @@ fn exported_symbols_provider_local<'tcx>(
         }));
     }
 
-    if tcx.crate_types().contains(&CrateType::Dylib)
-        || tcx.crate_types().contains(&CrateType::ProcMacro)
+    if !GENERIC
+        && (tcx.crate_types().contains(&CrateType::Dylib)
+            || tcx.crate_types().contains(&CrateType::ProcMacro))
     {
         let symbol_name = metadata_symbol_name(tcx);
         let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name));
@@ -296,7 +302,7 @@ fn exported_symbols_provider_local<'tcx>(
         ));
     }
 
-    if tcx.local_crate_exports_generics() {
+    if GENERIC && tcx.local_crate_exports_generics() {
         use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility};
         use rustc_middle::ty::InstanceKind;
 
@@ -458,7 +464,7 @@ fn upstream_monomorphizations_provider(
     let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn();
 
     for &cnum in cnums.iter() {
-        for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() {
+        for (exported_symbol, _) in tcx.exported_generic_symbols(cnum).iter() {
             let (def_id, args) = match *exported_symbol {
                 ExportedSymbol::Generic(def_id, args) => (def_id, args),
                 ExportedSymbol::DropGlue(ty) => {
@@ -480,10 +486,7 @@ fn upstream_monomorphizations_provider(
                 ExportedSymbol::AsyncDropGlue(def_id, ty) => (def_id, tcx.mk_args(&[ty.into()])),
                 ExportedSymbol::NonGeneric(..)
                 | ExportedSymbol::ThreadLocalShim(..)
-                | ExportedSymbol::NoDefId(..) => {
-                    // These are no monomorphizations
-                    continue;
-                }
+                | ExportedSymbol::NoDefId(..) => unreachable!(),
             };
 
             let args_map = instances.entry(def_id).or_default();
@@ -538,7 +541,8 @@ fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId)
 pub(crate) fn provide(providers: &mut Providers) {
     providers.reachable_non_generics = reachable_non_generics_provider;
     providers.is_reachable_non_generic = is_reachable_non_generic_provider_local;
-    providers.exported_symbols = exported_symbols_provider_local;
+    providers.exported_non_generic_symbols = exported_symbols_provider_local::<false>;
+    providers.exported_generic_symbols = exported_symbols_provider_local::<true>;
     providers.upstream_monomorphizations = upstream_monomorphizations_provider;
     providers.is_unreachable_local_definition = is_unreachable_local_definition_provider;
     providers.upstream_drop_glue_for = upstream_drop_glue_for_provider;

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -1124,8 +1124,9 @@ fn start_executing_work<B: ExtraBackendMethods>(
 
         let copy_symbols = |cnum| {
             let symbols = tcx
-                .exported_symbols(cnum)
+                .exported_non_generic_symbols(cnum)
                 .iter()
+                .chain(tcx.exported_generic_symbols(cnum))
                 .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
                 .collect();
             Arc::new(symbols)

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -1496,11 +1496,18 @@ impl<'a> CrateMetadataRef<'a> {
             .map(move |v| (self.local_def_id(v.0), v.1))
     }
 
-    fn exported_symbols<'tcx>(
+    fn exported_non_generic_symbols<'tcx>(
         self,
         tcx: TyCtxt<'tcx>,
     ) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
-        tcx.arena.alloc_from_iter(self.root.exported_symbols.decode((self, tcx)))
+        tcx.arena.alloc_from_iter(self.root.exported_non_generic_symbols.decode((self, tcx)))
+    }
+
+    fn exported_generic_symbols<'tcx>(
+        self,
+        tcx: TyCtxt<'tcx>,
+    ) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
+        tcx.arena.alloc_from_iter(self.root.exported_generic_symbols.decode((self, tcx)))
     }
 
     fn get_macro(self, id: DefIndex, sess: &Session) -> ast::MacroDef {

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -358,7 +358,7 @@ provide! { tcx, def_id, other, cdata,
     specialization_enabled_in => { cdata.root.specialization_enabled_in }
     reachable_non_generics => {
         let reachable_non_generics = tcx
-            .exported_symbols(cdata.cnum)
+            .exported_non_generic_symbols(cdata.cnum)
             .iter()
             .filter_map(|&(exported_symbol, export_info)| {
                 if let ExportedSymbol::NonGeneric(def_id) = exported_symbol {
@@ -408,15 +408,8 @@ provide! { tcx, def_id, other, cdata,
 
     exportable_items => { tcx.arena.alloc_from_iter(cdata.get_exportable_items()) }
     stable_order_of_exportable_impls => { tcx.arena.alloc(cdata.get_stable_order_of_exportable_impls().collect()) }
-    exported_symbols => {
-        let syms = cdata.exported_symbols(tcx);
-
-        // FIXME rust-lang/rust#64319, rust-lang/rust#64872: We want
-        // to block export of generics from dylibs, but we must fix
-        // rust-lang/rust#65890 before we can do that robustly.
-
-        syms
-    }
+    exported_non_generic_symbols => { cdata.exported_non_generic_symbols(tcx) }
+    exported_generic_symbols => { cdata.exported_generic_symbols(tcx) }
 
     crate_extern_paths => { cdata.source().paths().cloned().collect() }
     expn_that_defined => { cdata.get_expn_that_defined(def_id.index, tcx.sess) }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -677,9 +677,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             stat!("exportable-items", || self.encode_stable_order_of_exportable_impls());
 
         // Encode exported symbols info. This is prefetched in `encode_metadata`.
-        let exported_symbols = stat!("exported-symbols", || {
-            self.encode_exported_symbols(tcx.exported_symbols(LOCAL_CRATE))
-        });
+        let (exported_non_generic_symbols, exported_generic_symbols) =
+            stat!("exported-symbols", || {
+                (
+                    self.encode_exported_symbols(tcx.exported_non_generic_symbols(LOCAL_CRATE)),
+                    self.encode_exported_symbols(tcx.exported_generic_symbols(LOCAL_CRATE)),
+                )
+            });
 
         // Encode the hygiene data.
         // IMPORTANT: this *must* be the last thing that we encode (other than `SourceMap`). The
@@ -745,7 +749,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 incoherent_impls,
                 exportable_items,
                 stable_order_of_exportable_impls,
-                exported_symbols,
+                exported_non_generic_symbols,
+                exported_generic_symbols,
                 interpret_alloc_index,
                 tables,
                 syntax_contexts,
@@ -2356,7 +2361,13 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) {
         // Prefetch some queries used by metadata encoding.
         // This is not necessary for correctness, but is only done for performance reasons.
         // It can be removed if it turns out to cause trouble or be detrimental to performance.
-        join(|| prefetch_mir(tcx), || tcx.exported_symbols(LOCAL_CRATE));
+        join(
+            || prefetch_mir(tcx),
+            || {
+                let _ = tcx.exported_non_generic_symbols(LOCAL_CRATE);
+                let _ = tcx.exported_generic_symbols(LOCAL_CRATE);
+            },
+        );
     }
 
     with_encode_metadata_header(tcx, path, |ecx| {

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -282,7 +282,8 @@ pub(crate) struct CrateRoot {
 
     exportable_items: LazyArray<DefIndex>,
     stable_order_of_exportable_impls: LazyArray<(DefIndex, usize)>,
-    exported_symbols: LazyArray<(ExportedSymbol<'static>, SymbolExportInfo)>,
+    exported_non_generic_symbols: LazyArray<(ExportedSymbol<'static>, SymbolExportInfo)>,
+    exported_generic_symbols: LazyArray<(ExportedSymbol<'static>, SymbolExportInfo)>,
 
     syntax_contexts: SyntaxContextTable,
     expn_data: ExpnDataTable,

compiler/rustc_middle/src/query/mod.rs

@@ -2312,13 +2312,28 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    /// The list of symbols exported from the given crate.
+    /// The list of non-generic symbols exported from the given crate.
+    ///
+    /// - All names contained in `exported_non_generic_symbols(cnum)` are
+    ///   guaranteed to correspond to a publicly visible symbol in `cnum`
+    ///   machine code.
+    /// - The `exported_non_generic_symbols` and `exported_generic_symbols`
+    ///   sets of different crates do not intersect.
+    query exported_non_generic_symbols(cnum: CrateNum) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
+        desc { "collecting exported non-generic symbols for crate `{}`", cnum}
+        cache_on_disk_if { *cnum == LOCAL_CRATE }
+        separate_provide_extern
+    }
+
+    /// The list of generic symbols exported from the given crate.
     ///
-    /// - All names contained in `exported_symbols(cnum)` are guaranteed to
-    ///   correspond to a publicly visible symbol in `cnum` machine code.
-    /// - The `exported_symbols` sets of different crates do not intersect.
-    query exported_symbols(cnum: CrateNum) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
-        desc { "collecting exported symbols for crate `{}`", cnum}
+    /// - All names contained in `exported_generic_symbols(cnum)` are
+    ///   guaranteed to correspond to a publicly visible symbol in `cnum`
+    ///   machine code.
+    /// - The `exported_non_generic_symbols` and `exported_generic_symbols`
+    ///   sets of different crates do not intersect.
+    query exported_generic_symbols(cnum: CrateNum) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
+        desc { "collecting exported generic symbols for crate `{}`", cnum}
         cache_on_disk_if { *cnum == LOCAL_CRATE }
         separate_provide_extern
     }

src/tools/miri/src/bin/miri.rs

@@ -85,7 +85,7 @@ fn entry_fn(tcx: TyCtxt<'_>) -> (DefId, MiriEntryFnType) {
         return (def_id, MiriEntryFnType::Rustc(entry_type));
     }
     // Look for a symbol in the local crate named `miri_start`, and treat that as the entry point.
-    let sym = tcx.exported_symbols(LOCAL_CRATE).iter().find_map(|(sym, _)| {
+    let sym = tcx.exported_non_generic_symbols(LOCAL_CRATE).iter().find_map(|(sym, _)| {
         if sym.symbol_name_for_local_instance(tcx).name == "miri_start" { Some(sym) } else { None }
     });
     if let Some(ExportedSymbol::NonGeneric(id)) = sym {
@@ -249,10 +249,10 @@ impl rustc_driver::Callbacks for MiriBeRustCompilerCalls {
             // Queries overridden here affect the data stored in `rmeta` files of dependencies,
             // which will be used later in non-`MIRI_BE_RUSTC` mode.
             config.override_queries = Some(|_, local_providers| {
-                // `exported_symbols` and `reachable_non_generics` provided by rustc always returns
+                // `exported_non_generic_symbols` and `reachable_non_generics` provided by rustc always returns
                 // an empty result if `tcx.sess.opts.output_types.should_codegen()` is false.
                 // In addition we need to add #[used] symbols to exported_symbols for `lookup_link_section`.
-                local_providers.exported_symbols = |tcx, LocalCrate| {
+                local_providers.exported_non_generic_symbols = |tcx, LocalCrate| {
                     let reachable_set = tcx.with_stable_hashing_context(|hcx| {
                         tcx.reachable_set(()).to_sorted(&hcx, true)
                     });

src/tools/miri/src/helpers.rs

@@ -162,7 +162,7 @@ pub fn iter_exported_symbols<'tcx>(
 
         // We can ignore `_export_info` here: we are a Rust crate, and everything is exported
         // from a Rust crate.
-        for &(symbol, _export_info) in tcx.exported_symbols(cnum) {
+        for &(symbol, _export_info) in tcx.exported_non_generic_symbols(cnum) {
             if let ExportedSymbol::NonGeneric(def_id) = symbol {
                 f(cnum, def_id)?;
             }