1use std::assert_matches::assert_matches;
2use std::cmp::Ordering;
3
4use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size};
5use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
6use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
7use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
8use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
9use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
10use rustc_codegen_ssa::traits::*;
11use rustc_hir as hir;
12use rustc_middle::mir::BinOp;
13use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
14use rustc_middle::ty::{self, GenericArgsRef, Ty};
15use rustc_middle::{bug, span_bug};
16use rustc_span::{Span, Symbol, sym};
17use rustc_symbol_mangling::mangle_internal_symbol;
18use rustc_target::spec::{HasTargetSpec, PanicStrategy};
19use tracing::debug;
20
21use crate::abi::FnAbiLlvmExt;
22use crate::builder::Builder;
23use crate::context::CodegenCx;
24use crate::llvm::{self, Metadata};
25use crate::type_::Type;
26use crate::type_of::LayoutLlvmExt;
27use crate::va_arg::emit_va_arg;
28use crate::value::Value;
29
30fn get_simple_intrinsic<'ll>(
31 cx: &CodegenCx<'ll, '_>,
32 name: Symbol,
33) -> Option<(&'ll Type, &'ll Value)> {
34 let llvm_name = match name {
35 sym::sqrtf16 => "llvm.sqrt.f16",
36 sym::sqrtf32 => "llvm.sqrt.f32",
37 sym::sqrtf64 => "llvm.sqrt.f64",
38 sym::sqrtf128 => "llvm.sqrt.f128",
39
40 sym::powif16 => "llvm.powi.f16.i32",
41 sym::powif32 => "llvm.powi.f32.i32",
42 sym::powif64 => "llvm.powi.f64.i32",
43 sym::powif128 => "llvm.powi.f128.i32",
44
45 sym::sinf16 => "llvm.sin.f16",
46 sym::sinf32 => "llvm.sin.f32",
47 sym::sinf64 => "llvm.sin.f64",
48 sym::sinf128 => "llvm.sin.f128",
49
50 sym::cosf16 => "llvm.cos.f16",
51 sym::cosf32 => "llvm.cos.f32",
52 sym::cosf64 => "llvm.cos.f64",
53 sym::cosf128 => "llvm.cos.f128",
54
55 sym::powf16 => "llvm.pow.f16",
56 sym::powf32 => "llvm.pow.f32",
57 sym::powf64 => "llvm.pow.f64",
58 sym::powf128 => "llvm.pow.f128",
59
60 sym::expf16 => "llvm.exp.f16",
61 sym::expf32 => "llvm.exp.f32",
62 sym::expf64 => "llvm.exp.f64",
63 sym::expf128 => "llvm.exp.f128",
64
65 sym::exp2f16 => "llvm.exp2.f16",
66 sym::exp2f32 => "llvm.exp2.f32",
67 sym::exp2f64 => "llvm.exp2.f64",
68 sym::exp2f128 => "llvm.exp2.f128",
69
70 sym::logf16 => "llvm.log.f16",
71 sym::logf32 => "llvm.log.f32",
72 sym::logf64 => "llvm.log.f64",
73 sym::logf128 => "llvm.log.f128",
74
75 sym::log10f16 => "llvm.log10.f16",
76 sym::log10f32 => "llvm.log10.f32",
77 sym::log10f64 => "llvm.log10.f64",
78 sym::log10f128 => "llvm.log10.f128",
79
80 sym::log2f16 => "llvm.log2.f16",
81 sym::log2f32 => "llvm.log2.f32",
82 sym::log2f64 => "llvm.log2.f64",
83 sym::log2f128 => "llvm.log2.f128",
84
85 sym::fmaf16 => "llvm.fma.f16",
86 sym::fmaf32 => "llvm.fma.f32",
87 sym::fmaf64 => "llvm.fma.f64",
88 sym::fmaf128 => "llvm.fma.f128",
89
90 sym::fmuladdf16 => "llvm.fmuladd.f16",
91 sym::fmuladdf32 => "llvm.fmuladd.f32",
92 sym::fmuladdf64 => "llvm.fmuladd.f64",
93 sym::fmuladdf128 => "llvm.fmuladd.f128",
94
95 sym::fabsf16 => "llvm.fabs.f16",
96 sym::fabsf32 => "llvm.fabs.f32",
97 sym::fabsf64 => "llvm.fabs.f64",
98 sym::fabsf128 => "llvm.fabs.f128",
99
100 sym::minnumf16 => "llvm.minnum.f16",
101 sym::minnumf32 => "llvm.minnum.f32",
102 sym::minnumf64 => "llvm.minnum.f64",
103 sym::minnumf128 => "llvm.minnum.f128",
104
105 sym::minimumf16 => "llvm.minimum.f16",
106 sym::minimumf32 => "llvm.minimum.f32",
107 sym::minimumf64 => "llvm.minimum.f64",
108 sym::maxnumf16 => "llvm.maxnum.f16",
112 sym::maxnumf32 => "llvm.maxnum.f32",
113 sym::maxnumf64 => "llvm.maxnum.f64",
114 sym::maxnumf128 => "llvm.maxnum.f128",
115
116 sym::maximumf16 => "llvm.maximum.f16",
117 sym::maximumf32 => "llvm.maximum.f32",
118 sym::maximumf64 => "llvm.maximum.f64",
119 sym::copysignf16 => "llvm.copysign.f16",
123 sym::copysignf32 => "llvm.copysign.f32",
124 sym::copysignf64 => "llvm.copysign.f64",
125 sym::copysignf128 => "llvm.copysign.f128",
126
127 sym::floorf16 => "llvm.floor.f16",
128 sym::floorf32 => "llvm.floor.f32",
129 sym::floorf64 => "llvm.floor.f64",
130 sym::floorf128 => "llvm.floor.f128",
131
132 sym::ceilf16 => "llvm.ceil.f16",
133 sym::ceilf32 => "llvm.ceil.f32",
134 sym::ceilf64 => "llvm.ceil.f64",
135 sym::ceilf128 => "llvm.ceil.f128",
136
137 sym::truncf16 => "llvm.trunc.f16",
138 sym::truncf32 => "llvm.trunc.f32",
139 sym::truncf64 => "llvm.trunc.f64",
140 sym::truncf128 => "llvm.trunc.f128",
141
142 sym::round_ties_even_f16 => "llvm.rint.f16",
147 sym::round_ties_even_f32 => "llvm.rint.f32",
148 sym::round_ties_even_f64 => "llvm.rint.f64",
149 sym::round_ties_even_f128 => "llvm.rint.f128",
150
151 sym::roundf16 => "llvm.round.f16",
152 sym::roundf32 => "llvm.round.f32",
153 sym::roundf64 => "llvm.round.f64",
154 sym::roundf128 => "llvm.round.f128",
155
156 sym::ptr_mask => "llvm.ptrmask",
157
158 _ => return None,
159 };
160 Some(cx.get_intrinsic(llvm_name))
161}
162
163impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
164 fn codegen_intrinsic_call(
165 &mut self,
166 instance: ty::Instance<'tcx>,
167 args: &[OperandRef<'tcx, &'ll Value>],
168 result: PlaceRef<'tcx, &'ll Value>,
169 span: Span,
170 ) -> Result<(), ty::Instance<'tcx>> {
171 let tcx = self.tcx;
172
173 let name = tcx.item_name(instance.def_id());
174 let fn_args = instance.args;
175
176 let simple = get_simple_intrinsic(self, name);
177 let llval = match name {
178 _ if simple.is_some() => {
179 let (simple_ty, simple_fn) = simple.unwrap();
180 self.call(
181 simple_ty,
182 None,
183 None,
184 simple_fn,
185 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
186 None,
187 Some(instance),
188 )
189 }
190 sym::is_val_statically_known => {
191 let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
192 let kind = self.type_kind(intrinsic_type);
193 let intrinsic_name = match kind {
194 TypeKind::Pointer | TypeKind::Integer => {
195 Some(format!("llvm.is.constant.{intrinsic_type:?}"))
196 }
197 TypeKind::Half => Some(format!("llvm.is.constant.f16")),
199 TypeKind::Float => Some(format!("llvm.is.constant.f32")),
200 TypeKind::Double => Some(format!("llvm.is.constant.f64")),
201 TypeKind::FP128 => Some(format!("llvm.is.constant.f128")),
202 _ => None,
203 };
204 if let Some(intrinsic_name) = intrinsic_name {
205 self.call_intrinsic(&intrinsic_name, &[args[0].immediate()])
206 } else {
207 self.const_bool(false)
208 }
209 }
210 sym::select_unpredictable => {
211 let cond = args[0].immediate();
212 assert_eq!(args[1].layout, args[2].layout);
213 let select = |bx: &mut Self, true_val, false_val| {
214 let result = bx.select(cond, true_val, false_val);
215 bx.set_unpredictable(&result);
216 result
217 };
218 match (args[1].val, args[2].val) {
219 (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => {
220 assert!(true_val.llextra.is_none());
221 assert!(false_val.llextra.is_none());
222 assert_eq!(true_val.align, false_val.align);
223 let ptr = select(self, true_val.llval, false_val.llval);
224 let selected =
225 OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align));
226 selected.store(self, result);
227 return Ok(());
228 }
229 (OperandValue::Immediate(_), OperandValue::Immediate(_))
230 | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => {
231 let true_val = args[1].immediate_or_packed_pair(self);
232 let false_val = args[2].immediate_or_packed_pair(self);
233 select(self, true_val, false_val)
234 }
235 (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()),
236 _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"),
237 }
238 }
239 sym::catch_unwind => {
240 catch_unwind_intrinsic(
241 self,
242 args[0].immediate(),
243 args[1].immediate(),
244 args[2].immediate(),
245 result,
246 );
247 return Ok(());
248 }
249 sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]),
250 sym::va_copy => {
251 self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()])
252 }
253 sym::va_arg => {
254 match result.layout.backend_repr {
255 BackendRepr::Scalar(scalar) => {
256 match scalar.primitive() {
257 Primitive::Int(..) => {
258 if self.cx().size_of(result.layout.ty).bytes() < 4 {
259 let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
264 self.trunc(promoted_result, result.layout.llvm_type(self))
265 } else {
266 emit_va_arg(self, args[0], result.layout.ty)
267 }
268 }
269 Primitive::Float(Float::F16) => {
270 bug!("the va_arg intrinsic does not work with `f16`")
271 }
272 Primitive::Float(Float::F64) | Primitive::Pointer(_) => {
273 emit_va_arg(self, args[0], result.layout.ty)
274 }
275 Primitive::Float(Float::F32) => {
277 bug!("the va_arg intrinsic does not work with `f32`")
278 }
279 Primitive::Float(Float::F128) => {
280 bug!("the va_arg intrinsic does not work with `f128`")
281 }
282 }
283 }
284 _ => bug!("the va_arg intrinsic does not work with non-scalar types"),
285 }
286 }
287
288 sym::volatile_load | sym::unaligned_volatile_load => {
289 let ptr = args[0].immediate();
290 let load = self.volatile_load(result.layout.llvm_type(self), ptr);
291 let align = if name == sym::unaligned_volatile_load {
292 1
293 } else {
294 result.layout.align.abi.bytes() as u32
295 };
296 unsafe {
297 llvm::LLVMSetAlignment(load, align);
298 }
299 if !result.layout.is_zst() {
300 self.store_to_place(load, result.val);
301 }
302 return Ok(());
303 }
304 sym::volatile_store => {
305 let dst = args[0].deref(self.cx());
306 args[1].val.volatile_store(self, dst);
307 return Ok(());
308 }
309 sym::unaligned_volatile_store => {
310 let dst = args[0].deref(self.cx());
311 args[1].val.unaligned_volatile_store(self, dst);
312 return Ok(());
313 }
314 sym::prefetch_read_data
315 | sym::prefetch_write_data
316 | sym::prefetch_read_instruction
317 | sym::prefetch_write_instruction => {
318 let (rw, cache_type) = match name {
319 sym::prefetch_read_data => (0, 1),
320 sym::prefetch_write_data => (1, 1),
321 sym::prefetch_read_instruction => (0, 0),
322 sym::prefetch_write_instruction => (1, 0),
323 _ => bug!(),
324 };
325 self.call_intrinsic(
326 "llvm.prefetch",
327 &[
328 args[0].immediate(),
329 self.const_i32(rw),
330 args[1].immediate(),
331 self.const_i32(cache_type),
332 ],
333 )
334 }
335 sym::carrying_mul_add => {
336 let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
337
338 let wide_llty = self.type_ix(size.bits() * 2);
339 let args = args.as_array().unwrap();
340 let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed));
341
342 let wide = if signed {
343 let prod = self.unchecked_smul(a, b);
344 let acc = self.unchecked_sadd(prod, c);
345 self.unchecked_sadd(acc, d)
346 } else {
347 let prod = self.unchecked_umul(a, b);
348 let acc = self.unchecked_uadd(prod, c);
349 self.unchecked_uadd(acc, d)
350 };
351
352 let narrow_llty = self.type_ix(size.bits());
353 let low = self.trunc(wide, narrow_llty);
354 let bits_const = self.const_uint(wide_llty, size.bits());
355 let high = self.lshr(wide, bits_const);
357 let high = self.trunc(high, narrow_llty);
359
360 let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false);
361 let pair = self.const_poison(pair_llty);
362 let pair = self.insert_value(pair, low, 0);
363 let pair = self.insert_value(pair, high, 1);
364 pair
365 }
366 sym::ctlz
367 | sym::ctlz_nonzero
368 | sym::cttz
369 | sym::cttz_nonzero
370 | sym::ctpop
371 | sym::bswap
372 | sym::bitreverse
373 | sym::rotate_left
374 | sym::rotate_right
375 | sym::saturating_add
376 | sym::saturating_sub => {
377 let ty = args[0].layout.ty;
378 if !ty.is_integral() {
379 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
380 span,
381 name,
382 ty,
383 });
384 return Ok(());
385 }
386 let (size, signed) = ty.int_size_and_signed(self.tcx);
387 let width = size.bits();
388 match name {
389 sym::ctlz | sym::cttz => {
390 let y = self.const_bool(false);
391 let ret = self.call_intrinsic(
392 &format!("llvm.{name}.i{width}"),
393 &[args[0].immediate(), y],
394 );
395
396 self.intcast(ret, result.layout.llvm_type(self), false)
397 }
398 sym::ctlz_nonzero => {
399 let y = self.const_bool(true);
400 let llvm_name = &format!("llvm.ctlz.i{width}");
401 let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
402 self.intcast(ret, result.layout.llvm_type(self), false)
403 }
404 sym::cttz_nonzero => {
405 let y = self.const_bool(true);
406 let llvm_name = &format!("llvm.cttz.i{width}");
407 let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
408 self.intcast(ret, result.layout.llvm_type(self), false)
409 }
410 sym::ctpop => {
411 let ret = self.call_intrinsic(
412 &format!("llvm.ctpop.i{width}"),
413 &[args[0].immediate()],
414 );
415 self.intcast(ret, result.layout.llvm_type(self), false)
416 }
417 sym::bswap => {
418 if width == 8 {
419 args[0].immediate() } else {
421 self.call_intrinsic(
422 &format!("llvm.bswap.i{width}"),
423 &[args[0].immediate()],
424 )
425 }
426 }
427 sym::bitreverse => self.call_intrinsic(
428 &format!("llvm.bitreverse.i{width}"),
429 &[args[0].immediate()],
430 ),
431 sym::rotate_left | sym::rotate_right => {
432 let is_left = name == sym::rotate_left;
433 let val = args[0].immediate();
434 let raw_shift = args[1].immediate();
435 let llvm_name =
437 &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
438
439 let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
442
443 self.call_intrinsic(llvm_name, &[val, val, raw_shift])
444 }
445 sym::saturating_add | sym::saturating_sub => {
446 let is_add = name == sym::saturating_add;
447 let lhs = args[0].immediate();
448 let rhs = args[1].immediate();
449 let llvm_name = &format!(
450 "llvm.{}{}.sat.i{}",
451 if signed { 's' } else { 'u' },
452 if is_add { "add" } else { "sub" },
453 width
454 );
455 self.call_intrinsic(llvm_name, &[lhs, rhs])
456 }
457 _ => bug!(),
458 }
459 }
460
461 sym::raw_eq => {
462 use BackendRepr::*;
463 let tp_ty = fn_args.type_at(0);
464 let layout = self.layout_of(tp_ty).layout;
465 let use_integer_compare = match layout.backend_repr() {
466 Scalar(_) | ScalarPair(_, _) => true,
467 SimdVector { .. } => false,
468 Memory { .. } => {
469 layout.size() <= self.data_layout().pointer_size * 2
473 }
474 };
475
476 let a = args[0].immediate();
477 let b = args[1].immediate();
478 if layout.size().bytes() == 0 {
479 self.const_bool(true)
480 } else if use_integer_compare {
481 let integer_ty = self.type_ix(layout.size().bits());
482 let a_val = self.load(integer_ty, a, layout.align().abi);
483 let b_val = self.load(integer_ty, b, layout.align().abi);
484 self.icmp(IntPredicate::IntEQ, a_val, b_val)
485 } else {
486 let n = self.const_usize(layout.size().bytes());
487 let cmp = self.call_intrinsic("memcmp", &[a, b, n]);
488 match self.cx.sess().target.arch.as_ref() {
489 "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)),
490 _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)),
491 }
492 }
493 }
494
495 sym::compare_bytes => {
496 let cmp = self.call_intrinsic(
498 "memcmp",
499 &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
500 );
501 self.sext(cmp, self.type_ix(32))
503 }
504
505 sym::black_box => {
506 args[0].val.store(self, result);
507 let result_val_span = [result.val.llval];
508 let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
518 ("~{memory}", &[])
519 } else {
520 ("r,~{memory}", &result_val_span)
521 };
522 crate::asm::inline_asm_call(
523 self,
524 "",
525 constraint,
526 inputs,
527 self.type_void(),
528 &[],
529 true,
530 false,
531 llvm::AsmDialect::Att,
532 &[span],
533 false,
534 None,
535 None,
536 )
537 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
538
539 return Ok(());
541 }
542
543 _ if name.as_str().starts_with("simd_") => {
544 let mut loaded_args = Vec::new();
547 for arg in args {
548 loaded_args.push(
549 if arg.layout.ty.is_simd()
554 && let OperandValue::Ref(place) = arg.val
555 {
556 let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
557 let elem_ll_ty = match elem_ty.kind() {
558 ty::Float(f) => self.type_float_from_ty(*f),
559 ty::Int(i) => self.type_int_from_ty(*i),
560 ty::Uint(u) => self.type_uint_from_ty(*u),
561 ty::RawPtr(_, _) => self.type_ptr(),
562 _ => unreachable!(),
563 };
564 let loaded =
565 self.load_from_place(self.type_vector(elem_ll_ty, size), place);
566 OperandRef::from_immediate_or_packed_pair(self, loaded, arg.layout)
567 } else {
568 *arg
569 },
570 );
571 }
572
573 let llret_ty = if result.layout.ty.is_simd()
574 && let BackendRepr::Memory { .. } = result.layout.backend_repr
575 {
576 let (size, elem_ty) = result.layout.ty.simd_size_and_type(self.tcx());
577 let elem_ll_ty = match elem_ty.kind() {
578 ty::Float(f) => self.type_float_from_ty(*f),
579 ty::Int(i) => self.type_int_from_ty(*i),
580 ty::Uint(u) => self.type_uint_from_ty(*u),
581 ty::RawPtr(_, _) => self.type_ptr(),
582 _ => unreachable!(),
583 };
584 self.type_vector(elem_ll_ty, size)
585 } else {
586 result.layout.llvm_type(self)
587 };
588
589 match generic_simd_intrinsic(
590 self,
591 name,
592 fn_args,
593 &loaded_args,
594 result.layout.ty,
595 llret_ty,
596 span,
597 ) {
598 Ok(llval) => llval,
599 Err(()) => return Ok(()),
602 }
603 }
604
605 _ => {
606 debug!("unknown intrinsic '{}' -- falling back to default body", name);
607 return Err(ty::Instance::new_raw(instance.def_id(), instance.args));
609 }
610 };
611
612 if result.layout.ty.is_bool() {
613 let val = self.from_immediate(llval);
614 self.store_to_place(val, result.val);
615 } else if !result.layout.ty.is_unit() {
616 self.store_to_place(llval, result.val);
617 }
618 Ok(())
619 }
620
621 fn abort(&mut self) {
622 self.call_intrinsic("llvm.trap", &[]);
623 }
624
625 fn assume(&mut self, val: Self::Value) {
626 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
627 self.call_intrinsic("llvm.assume", &[val]);
628 }
629 }
630
631 fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
632 if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
633 self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)])
634 } else {
635 cond
636 }
637 }
638
639 fn type_checked_load(
640 &mut self,
641 llvtable: &'ll Value,
642 vtable_byte_offset: u64,
643 typeid: &'ll Metadata,
644 ) -> Self::Value {
645 let typeid = self.get_metadata_value(typeid);
646 let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
647 let type_checked_load =
648 self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]);
649 self.extract_value(type_checked_load, 0)
650 }
651
652 fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
653 self.call_intrinsic("llvm.va_start", &[va_list])
654 }
655
656 fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
657 self.call_intrinsic("llvm.va_end", &[va_list])
658 }
659}
660
661fn catch_unwind_intrinsic<'ll, 'tcx>(
662 bx: &mut Builder<'_, 'll, 'tcx>,
663 try_func: &'ll Value,
664 data: &'ll Value,
665 catch_func: &'ll Value,
666 dest: PlaceRef<'tcx, &'ll Value>,
667) {
668 if bx.sess().panic_strategy() == PanicStrategy::Abort {
669 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
670 bx.call(try_func_ty, None, None, try_func, &[data], None, None);
671 OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
674 } else if wants_msvc_seh(bx.sess()) {
675 codegen_msvc_try(bx, try_func, data, catch_func, dest);
676 } else if wants_wasm_eh(bx.sess()) {
677 codegen_wasm_try(bx, try_func, data, catch_func, dest);
678 } else if bx.sess().target.os == "emscripten" {
679 codegen_emcc_try(bx, try_func, data, catch_func, dest);
680 } else {
681 codegen_gnu_try(bx, try_func, data, catch_func, dest);
682 }
683}
684
685fn codegen_msvc_try<'ll, 'tcx>(
693 bx: &mut Builder<'_, 'll, 'tcx>,
694 try_func: &'ll Value,
695 data: &'ll Value,
696 catch_func: &'ll Value,
697 dest: PlaceRef<'tcx, &'ll Value>,
698) {
699 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
700 bx.set_personality_fn(bx.eh_personality());
701
702 let normal = bx.append_sibling_block("normal");
703 let catchswitch = bx.append_sibling_block("catchswitch");
704 let catchpad_rust = bx.append_sibling_block("catchpad_rust");
705 let catchpad_foreign = bx.append_sibling_block("catchpad_foreign");
706 let caught = bx.append_sibling_block("caught");
707
708 let try_func = llvm::get_param(bx.llfn(), 0);
709 let data = llvm::get_param(bx.llfn(), 1);
710 let catch_func = llvm::get_param(bx.llfn(), 2);
711
712 let ptr_size = bx.tcx().data_layout.pointer_size;
768 let ptr_align = bx.tcx().data_layout.pointer_align.abi;
769 let slot = bx.alloca(ptr_size, ptr_align);
770 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
771 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
772
773 bx.switch_to_block(normal);
774 bx.ret(bx.const_i32(0));
775
776 bx.switch_to_block(catchswitch);
777 let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]);
778
779 let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr());
794 let type_name = bx.const_bytes(b"rust_panic\0");
795 let type_info =
796 bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false);
797 let tydesc = bx.declare_global(
798 &mangle_internal_symbol(bx.tcx, "__rust_panic_type_info"),
799 bx.val_ty(type_info),
800 );
801
802 llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
803 if bx.cx.tcx.sess.target.supports_comdat() {
804 llvm::SetUniqueComdat(bx.llmod, tydesc);
805 }
806 llvm::set_initializer(tydesc, type_info);
807
808 bx.switch_to_block(catchpad_rust);
815 let flags = bx.const_i32(8);
816 let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
817 let ptr = bx.load(bx.type_ptr(), slot, ptr_align);
818 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
819 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
820 bx.catch_ret(&funclet, caught);
821
822 bx.switch_to_block(catchpad_foreign);
824 let flags = bx.const_i32(64);
825 let null = bx.const_null(bx.type_ptr());
826 let funclet = bx.catch_pad(cs, &[null, flags, null]);
827 bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None);
828 bx.catch_ret(&funclet, caught);
829
830 bx.switch_to_block(caught);
831 bx.ret(bx.const_i32(1));
832 });
833
834 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
837 OperandValue::Immediate(ret).store(bx, dest);
838}
839
840fn codegen_wasm_try<'ll, 'tcx>(
842 bx: &mut Builder<'_, 'll, 'tcx>,
843 try_func: &'ll Value,
844 data: &'ll Value,
845 catch_func: &'ll Value,
846 dest: PlaceRef<'tcx, &'ll Value>,
847) {
848 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
849 bx.set_personality_fn(bx.eh_personality());
850
851 let normal = bx.append_sibling_block("normal");
852 let catchswitch = bx.append_sibling_block("catchswitch");
853 let catchpad = bx.append_sibling_block("catchpad");
854 let caught = bx.append_sibling_block("caught");
855
856 let try_func = llvm::get_param(bx.llfn(), 0);
857 let data = llvm::get_param(bx.llfn(), 1);
858 let catch_func = llvm::get_param(bx.llfn(), 2);
859
860 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
884 bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None);
885
886 bx.switch_to_block(normal);
887 bx.ret(bx.const_i32(0));
888
889 bx.switch_to_block(catchswitch);
890 let cs = bx.catch_switch(None, None, &[catchpad]);
891
892 bx.switch_to_block(catchpad);
893 let null = bx.const_null(bx.type_ptr());
894 let funclet = bx.catch_pad(cs, &[null]);
895
896 let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]);
897 let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]);
898
899 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
900 bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
901 bx.catch_ret(&funclet, caught);
902
903 bx.switch_to_block(caught);
904 bx.ret(bx.const_i32(1));
905 });
906
907 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
910 OperandValue::Immediate(ret).store(bx, dest);
911}
912
913fn codegen_gnu_try<'ll, 'tcx>(
925 bx: &mut Builder<'_, 'll, 'tcx>,
926 try_func: &'ll Value,
927 data: &'ll Value,
928 catch_func: &'ll Value,
929 dest: PlaceRef<'tcx, &'ll Value>,
930) {
931 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
932 let then = bx.append_sibling_block("then");
945 let catch = bx.append_sibling_block("catch");
946
947 let try_func = llvm::get_param(bx.llfn(), 0);
948 let data = llvm::get_param(bx.llfn(), 1);
949 let catch_func = llvm::get_param(bx.llfn(), 2);
950 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
951 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
952
953 bx.switch_to_block(then);
954 bx.ret(bx.const_i32(0));
955
956 bx.switch_to_block(catch);
963 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
964 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1);
965 let tydesc = bx.const_null(bx.type_ptr());
966 bx.add_clause(vals, tydesc);
967 let ptr = bx.extract_value(vals, 0);
968 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
969 bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None);
970 bx.ret(bx.const_i32(1));
971 });
972
973 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
976 OperandValue::Immediate(ret).store(bx, dest);
977}
978
979fn codegen_emcc_try<'ll, 'tcx>(
983 bx: &mut Builder<'_, 'll, 'tcx>,
984 try_func: &'ll Value,
985 data: &'ll Value,
986 catch_func: &'ll Value,
987 dest: PlaceRef<'tcx, &'ll Value>,
988) {
989 let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
990 let then = bx.append_sibling_block("then");
1008 let catch = bx.append_sibling_block("catch");
1009
1010 let try_func = llvm::get_param(bx.llfn(), 0);
1011 let data = llvm::get_param(bx.llfn(), 1);
1012 let catch_func = llvm::get_param(bx.llfn(), 2);
1013 let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
1014 bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
1015
1016 bx.switch_to_block(then);
1017 bx.ret(bx.const_i32(0));
1018
1019 bx.switch_to_block(catch);
1025 let tydesc = bx.eh_catch_typeinfo();
1026 let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false);
1027 let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2);
1028 bx.add_clause(vals, tydesc);
1029 bx.add_clause(vals, bx.const_null(bx.type_ptr()));
1030 let ptr = bx.extract_value(vals, 0);
1031 let selector = bx.extract_value(vals, 1);
1032
1033 let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]);
1035 let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
1036 let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
1037
1038 let ptr_size = bx.tcx().data_layout.pointer_size;
1041 let ptr_align = bx.tcx().data_layout.pointer_align.abi;
1042 let i8_align = bx.tcx().data_layout.i8_align.abi;
1043 assert!(i8_align <= ptr_align);
1045 let catch_data = bx.alloca(2 * ptr_size, ptr_align);
1046 bx.store(ptr, catch_data, ptr_align);
1047 let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes()));
1048 bx.store(is_rust_panic, catch_data_1, i8_align);
1049
1050 let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
1051 bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None, None);
1052 bx.ret(bx.const_i32(1));
1053 });
1054
1055 let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
1058 OperandValue::Immediate(ret).store(bx, dest);
1059}
1060
1061fn gen_fn<'a, 'll, 'tcx>(
1064 cx: &'a CodegenCx<'ll, 'tcx>,
1065 name: &str,
1066 rust_fn_sig: ty::PolyFnSig<'tcx>,
1067 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1068) -> (&'ll Type, &'ll Value) {
1069 let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1070 let llty = fn_abi.llvm_type(cx);
1071 let llfn = cx.declare_fn(name, fn_abi, None);
1072 cx.set_frame_pointer_type(llfn);
1073 cx.apply_target_cpu_attr(llfn);
1074 llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage);
1076 let llbb = Builder::append_block(cx, llfn, "entry-block");
1077 let bx = Builder::build(cx, llbb);
1078 codegen(bx);
1079 (llty, llfn)
1080}
1081
1082fn get_rust_try_fn<'a, 'll, 'tcx>(
1087 cx: &'a CodegenCx<'ll, 'tcx>,
1088 codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
1089) -> (&'ll Type, &'ll Value) {
1090 if let Some(llfn) = cx.rust_try_fn.get() {
1091 return llfn;
1092 }
1093
1094 let tcx = cx.tcx;
1096 let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8);
1097 let try_fn_ty = Ty::new_fn_ptr(
1099 tcx,
1100 ty::Binder::dummy(tcx.mk_fn_sig(
1101 [i8p],
1102 tcx.types.unit,
1103 false,
1104 hir::Safety::Unsafe,
1105 ExternAbi::Rust,
1106 )),
1107 );
1108 let catch_fn_ty = Ty::new_fn_ptr(
1110 tcx,
1111 ty::Binder::dummy(tcx.mk_fn_sig(
1112 [i8p, i8p],
1113 tcx.types.unit,
1114 false,
1115 hir::Safety::Unsafe,
1116 ExternAbi::Rust,
1117 )),
1118 );
1119 let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig(
1121 [try_fn_ty, i8p, catch_fn_ty],
1122 tcx.types.i32,
1123 false,
1124 hir::Safety::Unsafe,
1125 ExternAbi::Rust,
1126 ));
1127 let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen);
1128 cx.rust_try_fn.set(Some(rust_try));
1129 rust_try
1130}
1131
1132fn generic_simd_intrinsic<'ll, 'tcx>(
1133 bx: &mut Builder<'_, 'll, 'tcx>,
1134 name: Symbol,
1135 fn_args: GenericArgsRef<'tcx>,
1136 args: &[OperandRef<'tcx, &'ll Value>],
1137 ret_ty: Ty<'tcx>,
1138 llret_ty: &'ll Type,
1139 span: Span,
1140) -> Result<&'ll Value, ()> {
1141 macro_rules! return_error {
1142 ($diag: expr) => {{
1143 bx.sess().dcx().emit_err($diag);
1144 return Err(());
1145 }};
1146 }
1147
1148 macro_rules! require {
1149 ($cond: expr, $diag: expr) => {
1150 if !$cond {
1151 return_error!($diag);
1152 }
1153 };
1154 }
1155
1156 macro_rules! require_simd {
1157 ($ty: expr, $variant:ident) => {{
1158 require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
1159 $ty.simd_size_and_type(bx.tcx())
1160 }};
1161 }
1162
1163 macro_rules! require_int_or_uint_ty {
1165 ($ty: expr, $diag: expr) => {
1166 match $ty {
1167 ty::Int(i) => i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()),
1168 ty::Uint(i) => {
1169 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
1170 }
1171 _ => {
1172 return_error!($diag);
1173 }
1174 }
1175 };
1176 }
1177
1178 fn vector_mask_to_bitmask<'a, 'll, 'tcx>(
1192 bx: &mut Builder<'a, 'll, 'tcx>,
1193 i_xn: &'ll Value,
1194 in_elem_bitwidth: u64,
1195 in_len: u64,
1196 ) -> &'ll Value {
1197 let shift_idx = bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _);
1199 let shift_indices = vec![shift_idx; in_len as _];
1200 let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice()));
1201 bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
1203 }
1204
1205 if cfg!(debug_assertions) {
1207 for arg in args {
1208 if arg.layout.ty.is_simd() {
1209 assert_matches!(arg.val, OperandValue::Immediate(_));
1210 }
1211 }
1212 }
1213
1214 if name == sym::simd_select_bitmask {
1215 let (len, _) = require_simd!(args[1].layout.ty, SimdArgument);
1216
1217 let expected_int_bits = len.max(8).next_power_of_two();
1218 let expected_bytes = len.div_ceil(8);
1219
1220 let mask_ty = args[0].layout.ty;
1221 let mask = match mask_ty.kind() {
1222 ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1223 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
1224 ty::Array(elem, len)
1225 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1226 && len
1227 .try_to_target_usize(bx.tcx)
1228 .expect("expected monomorphic const in codegen")
1229 == expected_bytes =>
1230 {
1231 let place = PlaceRef::alloca(bx, args[0].layout);
1232 args[0].val.store(bx, place);
1233 let int_ty = bx.type_ix(expected_bytes * 8);
1234 bx.load(int_ty, place.val.llval, Align::ONE)
1235 }
1236 _ => return_error!(InvalidMonomorphization::InvalidBitmask {
1237 span,
1238 name,
1239 mask_ty,
1240 expected_int_bits,
1241 expected_bytes
1242 }),
1243 };
1244
1245 let i1 = bx.type_i1();
1246 let im = bx.type_ix(len);
1247 let i1xn = bx.type_vector(i1, len);
1248 let m_im = bx.trunc(mask, im);
1249 let m_i1s = bx.bitcast(m_im, i1xn);
1250 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1251 }
1252
1253 let (in_len, in_elem) = require_simd!(args[0].layout.ty, SimdInput);
1255 let in_ty = args[0].layout.ty;
1256
1257 let comparison = match name {
1258 sym::simd_eq => Some(BinOp::Eq),
1259 sym::simd_ne => Some(BinOp::Ne),
1260 sym::simd_lt => Some(BinOp::Lt),
1261 sym::simd_le => Some(BinOp::Le),
1262 sym::simd_gt => Some(BinOp::Gt),
1263 sym::simd_ge => Some(BinOp::Ge),
1264 _ => None,
1265 };
1266
1267 if let Some(cmp_op) = comparison {
1268 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1269
1270 require!(
1271 in_len == out_len,
1272 InvalidMonomorphization::ReturnLengthInputType {
1273 span,
1274 name,
1275 in_len,
1276 in_ty,
1277 ret_ty,
1278 out_len
1279 }
1280 );
1281 require!(
1282 bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
1283 InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
1284 );
1285
1286 return Ok(compare_simd_types(
1287 bx,
1288 args[0].immediate(),
1289 args[1].immediate(),
1290 in_elem,
1291 llret_ty,
1292 cmp_op,
1293 ));
1294 }
1295
1296 if name == sym::simd_shuffle_const_generic {
1297 let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch();
1298 let n = idx.len() as u64;
1299
1300 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1301 require!(
1302 out_len == n,
1303 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1304 );
1305 require!(
1306 in_elem == out_ty,
1307 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1308 );
1309
1310 let total_len = in_len * 2;
1311
1312 let indices: Option<Vec<_>> = idx
1313 .iter()
1314 .enumerate()
1315 .map(|(arg_idx, val)| {
1316 let idx = val.unwrap_leaf().to_i32();
1317 if idx >= i32::try_from(total_len).unwrap() {
1318 bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
1319 span,
1320 name,
1321 arg_idx: arg_idx as u64,
1322 total_len: total_len.into(),
1323 });
1324 None
1325 } else {
1326 Some(bx.const_i32(idx))
1327 }
1328 })
1329 .collect();
1330 let Some(indices) = indices else {
1331 return Ok(bx.const_null(llret_ty));
1332 };
1333
1334 return Ok(bx.shuffle_vector(
1335 args[0].immediate(),
1336 args[1].immediate(),
1337 bx.const_vector(&indices),
1338 ));
1339 }
1340
1341 if name == sym::simd_shuffle {
1342 let idx_ty = args[2].layout.ty;
1344 let n: u64 = if idx_ty.is_simd()
1345 && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
1346 {
1347 idx_ty.simd_size_and_type(bx.cx.tcx).0
1348 } else {
1349 return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
1350 };
1351
1352 let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
1353 require!(
1354 out_len == n,
1355 InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
1356 );
1357 require!(
1358 in_elem == out_ty,
1359 InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
1360 );
1361
1362 let total_len = u128::from(in_len) * 2;
1363
1364 let indices = args[2].immediate();
1366 for i in 0..n {
1367 let val = bx.const_get_elt(indices, i as u64);
1368 let idx = bx
1369 .const_to_opt_u128(val, true)
1370 .unwrap_or_else(|| bug!("typeck should have already ensured that these are const"));
1371 if idx >= total_len {
1372 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1373 span,
1374 name,
1375 arg_idx: i,
1376 total_len,
1377 });
1378 }
1379 }
1380
1381 return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
1382 }
1383
1384 if name == sym::simd_insert || name == sym::simd_insert_dyn {
1385 require!(
1386 in_elem == args[2].layout.ty,
1387 InvalidMonomorphization::InsertedType {
1388 span,
1389 name,
1390 in_elem,
1391 in_ty,
1392 out_ty: args[2].layout.ty
1393 }
1394 );
1395
1396 let index_imm = if name == sym::simd_insert {
1397 let idx = bx
1398 .const_to_opt_u128(args[1].immediate(), false)
1399 .expect("typeck should have ensure that this is a const");
1400 if idx >= in_len.into() {
1401 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1402 span,
1403 name,
1404 arg_idx: 1,
1405 total_len: in_len.into(),
1406 });
1407 }
1408 bx.const_i32(idx as i32)
1409 } else {
1410 args[1].immediate()
1411 };
1412
1413 return Ok(bx.insert_element(args[0].immediate(), args[2].immediate(), index_imm));
1414 }
1415 if name == sym::simd_extract || name == sym::simd_extract_dyn {
1416 require!(
1417 ret_ty == in_elem,
1418 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
1419 );
1420 let index_imm = if name == sym::simd_extract {
1421 let idx = bx
1422 .const_to_opt_u128(args[1].immediate(), false)
1423 .expect("typeck should have ensure that this is a const");
1424 if idx >= in_len.into() {
1425 return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
1426 span,
1427 name,
1428 arg_idx: 1,
1429 total_len: in_len.into(),
1430 });
1431 }
1432 bx.const_i32(idx as i32)
1433 } else {
1434 args[1].immediate()
1435 };
1436
1437 return Ok(bx.extract_element(args[0].immediate(), index_imm));
1438 }
1439
1440 if name == sym::simd_select {
1441 let m_elem_ty = in_elem;
1442 let m_len = in_len;
1443 let (v_len, _) = require_simd!(args[1].layout.ty, SimdArgument);
1444 require!(
1445 m_len == v_len,
1446 InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
1447 );
1448 let in_elem_bitwidth = require_int_or_uint_ty!(
1449 m_elem_ty.kind(),
1450 InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1451 );
1452 let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
1453 return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
1454 }
1455
1456 if name == sym::simd_bitmask {
1457 let expected_int_bits = in_len.max(8).next_power_of_two();
1466 let expected_bytes = in_len.div_ceil(8);
1467
1468 let in_elem_bitwidth = require_int_or_uint_ty!(
1470 in_elem.kind(),
1471 InvalidMonomorphization::MaskWrongElementType { span, name, ty: in_elem }
1472 );
1473
1474 let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
1475 let i_ = bx.bitcast(i1xn, bx.type_ix(in_len));
1477
1478 match ret_ty.kind() {
1479 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
1480 return Ok(bx.zext(i_, bx.type_ix(expected_int_bits)));
1482 }
1483 ty::Array(elem, len)
1484 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
1485 && len
1486 .try_to_target_usize(bx.tcx)
1487 .expect("expected monomorphic const in codegen")
1488 == expected_bytes =>
1489 {
1490 let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8));
1492
1493 let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE);
1495 bx.store(ze, ptr, Align::ONE);
1496 let array_ty = bx.type_array(bx.type_i8(), expected_bytes);
1497 return Ok(bx.load(array_ty, ptr, Align::ONE));
1498 }
1499 _ => return_error!(InvalidMonomorphization::CannotReturn {
1500 span,
1501 name,
1502 ret_ty,
1503 expected_int_bits,
1504 expected_bytes
1505 }),
1506 }
1507 }
1508
1509 fn simd_simple_float_intrinsic<'ll, 'tcx>(
1510 name: Symbol,
1511 in_elem: Ty<'_>,
1512 in_ty: Ty<'_>,
1513 in_len: u64,
1514 bx: &mut Builder<'_, 'll, 'tcx>,
1515 span: Span,
1516 args: &[OperandRef<'tcx, &'ll Value>],
1517 ) -> Result<&'ll Value, ()> {
1518 macro_rules! return_error {
1519 ($diag: expr) => {{
1520 bx.sess().dcx().emit_err($diag);
1521 return Err(());
1522 }};
1523 }
1524
1525 let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
1526 let elem_ty = bx.cx.type_float_from_ty(*f);
1527 match f.bit_width() {
1528 16 => ("f16", elem_ty),
1529 32 => ("f32", elem_ty),
1530 64 => ("f64", elem_ty),
1531 128 => ("f128", elem_ty),
1532 _ => return_error!(InvalidMonomorphization::FloatingPointVector {
1533 span,
1534 name,
1535 f_ty: *f,
1536 in_ty,
1537 }),
1538 }
1539 } else {
1540 return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
1541 };
1542
1543 let vec_ty = bx.type_vector(elem_ty, in_len);
1544
1545 let (intr_name, fn_ty) = match name {
1546 sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
1547 sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
1548 sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
1549 sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
1550 sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
1551 sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
1552 sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
1553 sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
1554 sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
1555 sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
1556 sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
1557 sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
1558 sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
1559 sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
1560 sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
1561 _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
1562 };
1563 let llvm_name = &format!("llvm.{intr_name}.v{in_len}{elem_ty_str}");
1564 let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty);
1565 let c = bx.call(
1566 fn_ty,
1567 None,
1568 None,
1569 f,
1570 &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
1571 None,
1572 None,
1573 );
1574 Ok(c)
1575 }
1576
1577 if std::matches!(
1578 name,
1579 sym::simd_ceil
1580 | sym::simd_fabs
1581 | sym::simd_fcos
1582 | sym::simd_fexp2
1583 | sym::simd_fexp
1584 | sym::simd_flog10
1585 | sym::simd_flog2
1586 | sym::simd_flog
1587 | sym::simd_floor
1588 | sym::simd_fma
1589 | sym::simd_fsin
1590 | sym::simd_fsqrt
1591 | sym::simd_relaxed_fma
1592 | sym::simd_round
1593 | sym::simd_trunc
1594 ) {
1595 return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
1596 }
1597
1598 fn llvm_vector_str(bx: &Builder<'_, '_, '_>, elem_ty: Ty<'_>, vec_len: u64) -> String {
1602 match *elem_ty.kind() {
1603 ty::Int(v) => format!(
1604 "v{}i{}",
1605 vec_len,
1606 v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
1608 ),
1609 ty::Uint(v) => format!(
1610 "v{}i{}",
1611 vec_len,
1612 v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
1614 ),
1615 ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()),
1616 ty::RawPtr(_, _) => format!("v{}p0", vec_len),
1617 _ => unreachable!(),
1618 }
1619 }
1620
1621 fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
1622 let elem_ty = match *elem_ty.kind() {
1623 ty::Int(v) => cx.type_int_from_ty(v),
1624 ty::Uint(v) => cx.type_uint_from_ty(v),
1625 ty::Float(v) => cx.type_float_from_ty(v),
1626 ty::RawPtr(_, _) => cx.type_ptr(),
1627 _ => unreachable!(),
1628 };
1629 cx.type_vector(elem_ty, vec_len)
1630 }
1631
1632 if name == sym::simd_gather {
1633 let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1644 let (out_len, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
1645 let (out_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
1647 require_simd!(ret_ty, SimdReturn);
1648
1649 require!(
1651 in_len == out_len,
1652 InvalidMonomorphization::SecondArgumentLength {
1653 span,
1654 name,
1655 in_len,
1656 in_ty,
1657 arg_ty: args[1].layout.ty,
1658 out_len
1659 }
1660 );
1661 require!(
1662 in_len == out_len2,
1663 InvalidMonomorphization::ThirdArgumentLength {
1664 span,
1665 name,
1666 in_len,
1667 in_ty,
1668 arg_ty: args[2].layout.ty,
1669 out_len: out_len2
1670 }
1671 );
1672
1673 require!(
1675 ret_ty == in_ty,
1676 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
1677 );
1678
1679 require!(
1680 matches!(
1681 *element_ty1.kind(),
1682 ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind()
1683 ),
1684 InvalidMonomorphization::ExpectedElementType {
1685 span,
1686 name,
1687 expected_element: element_ty1,
1688 second_arg: args[1].layout.ty,
1689 in_elem,
1690 in_ty,
1691 mutability: ExpectedPointerMutability::Not,
1692 }
1693 );
1694
1695 let mask_elem_bitwidth = require_int_or_uint_ty!(
1696 element_ty2.kind(),
1697 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1698 );
1699
1700 let alignment_ty = bx.type_i32();
1702 let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1703
1704 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1706 let mask_ty = bx.type_vector(bx.type_i1(), in_len);
1707
1708 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1710 let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
1711
1712 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1714 let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
1715
1716 let llvm_intrinsic =
1717 format!("llvm.masked.gather.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
1718 let fn_ty = bx.type_func(
1719 &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty],
1720 llvm_elem_vec_ty,
1721 );
1722 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1723 let v = bx.call(
1724 fn_ty,
1725 None,
1726 None,
1727 f,
1728 &[args[1].immediate(), alignment, mask, args[0].immediate()],
1729 None,
1730 None,
1731 );
1732 return Ok(v);
1733 }
1734
1735 if name == sym::simd_masked_load {
1736 let mask_ty = in_ty;
1746 let (mask_len, mask_elem) = (in_len, in_elem);
1747
1748 let pointer_ty = args[1].layout.ty;
1750
1751 let values_ty = args[2].layout.ty;
1753 let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1754
1755 require_simd!(ret_ty, SimdReturn);
1756
1757 require!(
1759 values_len == mask_len,
1760 InvalidMonomorphization::ThirdArgumentLength {
1761 span,
1762 name,
1763 in_len: mask_len,
1764 in_ty: mask_ty,
1765 arg_ty: values_ty,
1766 out_len: values_len
1767 }
1768 );
1769
1770 require!(
1772 ret_ty == values_ty,
1773 InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
1774 );
1775
1776 require!(
1777 matches!(
1778 *pointer_ty.kind(),
1779 ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind()
1780 ),
1781 InvalidMonomorphization::ExpectedElementType {
1782 span,
1783 name,
1784 expected_element: values_elem,
1785 second_arg: pointer_ty,
1786 in_elem: values_elem,
1787 in_ty: values_ty,
1788 mutability: ExpectedPointerMutability::Not,
1789 }
1790 );
1791
1792 let m_elem_bitwidth = require_int_or_uint_ty!(
1793 mask_elem.kind(),
1794 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1795 );
1796
1797 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1798 let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
1799
1800 let alignment_ty = bx.type_i32();
1802 let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1803
1804 let llvm_pointer = bx.type_ptr();
1805
1806 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1808 let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
1809
1810 let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0");
1811 let fn_ty = bx
1812 .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty);
1813 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1814 let v = bx.call(
1815 fn_ty,
1816 None,
1817 None,
1818 f,
1819 &[args[1].immediate(), alignment, mask, args[2].immediate()],
1820 None,
1821 None,
1822 );
1823 return Ok(v);
1824 }
1825
1826 if name == sym::simd_masked_store {
1827 let mask_ty = in_ty;
1837 let (mask_len, mask_elem) = (in_len, in_elem);
1838
1839 let pointer_ty = args[1].layout.ty;
1841
1842 let values_ty = args[2].layout.ty;
1844 let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
1845
1846 require!(
1848 values_len == mask_len,
1849 InvalidMonomorphization::ThirdArgumentLength {
1850 span,
1851 name,
1852 in_len: mask_len,
1853 in_ty: mask_ty,
1854 arg_ty: values_ty,
1855 out_len: values_len
1856 }
1857 );
1858
1859 require!(
1861 matches!(
1862 *pointer_ty.kind(),
1863 ty::RawPtr(p_ty, p_mutbl)
1864 if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
1865 ),
1866 InvalidMonomorphization::ExpectedElementType {
1867 span,
1868 name,
1869 expected_element: values_elem,
1870 second_arg: pointer_ty,
1871 in_elem: values_elem,
1872 in_ty: values_ty,
1873 mutability: ExpectedPointerMutability::Mut,
1874 }
1875 );
1876
1877 let m_elem_bitwidth = require_int_or_uint_ty!(
1878 mask_elem.kind(),
1879 InvalidMonomorphization::MaskWrongElementType { span, name, ty: mask_elem }
1880 );
1881
1882 let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
1883 let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
1884
1885 let alignment_ty = bx.type_i32();
1887 let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1888
1889 let ret_t = bx.type_void();
1890
1891 let llvm_pointer = bx.type_ptr();
1892
1893 let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
1895 let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
1896
1897 let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0");
1898 let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t);
1899 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1900 let v = bx.call(
1901 fn_ty,
1902 None,
1903 None,
1904 f,
1905 &[args[2].immediate(), args[1].immediate(), alignment, mask],
1906 None,
1907 None,
1908 );
1909 return Ok(v);
1910 }
1911
1912 if name == sym::simd_scatter {
1913 let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
1923 let (element_len1, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
1924 let (element_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
1925
1926 require!(
1928 in_len == element_len1,
1929 InvalidMonomorphization::SecondArgumentLength {
1930 span,
1931 name,
1932 in_len,
1933 in_ty,
1934 arg_ty: args[1].layout.ty,
1935 out_len: element_len1
1936 }
1937 );
1938 require!(
1939 in_len == element_len2,
1940 InvalidMonomorphization::ThirdArgumentLength {
1941 span,
1942 name,
1943 in_len,
1944 in_ty,
1945 arg_ty: args[2].layout.ty,
1946 out_len: element_len2
1947 }
1948 );
1949
1950 require!(
1951 matches!(
1952 *element_ty1.kind(),
1953 ty::RawPtr(p_ty, p_mutbl)
1954 if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind()
1955 ),
1956 InvalidMonomorphization::ExpectedElementType {
1957 span,
1958 name,
1959 expected_element: element_ty1,
1960 second_arg: args[1].layout.ty,
1961 in_elem,
1962 in_ty,
1963 mutability: ExpectedPointerMutability::Mut,
1964 }
1965 );
1966
1967 let mask_elem_bitwidth = require_int_or_uint_ty!(
1969 element_ty2.kind(),
1970 InvalidMonomorphization::MaskWrongElementType { span, name, ty: element_ty2 }
1971 );
1972
1973 let alignment_ty = bx.type_i32();
1975 let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1976
1977 let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
1979 let mask_ty = bx.type_vector(bx.type_i1(), in_len);
1980
1981 let ret_t = bx.type_void();
1982
1983 let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
1985 let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
1986
1987 let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
1989 let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
1990
1991 let llvm_intrinsic =
1992 format!("llvm.masked.scatter.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
1993 let fn_ty =
1994 bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t);
1995 let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
1996 let v = bx.call(
1997 fn_ty,
1998 None,
1999 None,
2000 f,
2001 &[args[0].immediate(), args[1].immediate(), alignment, mask],
2002 None,
2003 None,
2004 );
2005 return Ok(v);
2006 }
2007
2008 macro_rules! arith_red {
2009 ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
2010 $identity:expr) => {
2011 if name == sym::$name {
2012 require!(
2013 ret_ty == in_elem,
2014 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2015 );
2016 return match in_elem.kind() {
2017 ty::Int(_) | ty::Uint(_) => {
2018 let r = bx.$integer_reduce(args[0].immediate());
2019 if $ordered {
2020 Ok(bx.$op(args[1].immediate(), r))
2023 } else {
2024 Ok(bx.$integer_reduce(args[0].immediate()))
2025 }
2026 }
2027 ty::Float(f) => {
2028 let acc = if $ordered {
2029 args[1].immediate()
2031 } else {
2032 match f.bit_width() {
2034 32 => bx.const_real(bx.type_f32(), $identity),
2035 64 => bx.const_real(bx.type_f64(), $identity),
2036 v => return_error!(
2037 InvalidMonomorphization::UnsupportedSymbolOfSize {
2038 span,
2039 name,
2040 symbol: sym::$name,
2041 in_ty,
2042 in_elem,
2043 size: v,
2044 ret_ty
2045 }
2046 ),
2047 }
2048 };
2049 Ok(bx.$float_reduce(acc, args[0].immediate()))
2050 }
2051 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2052 span,
2053 name,
2054 symbol: sym::$name,
2055 in_ty,
2056 in_elem,
2057 ret_ty
2058 }),
2059 };
2060 }
2061 };
2062 }
2063
2064 arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
2065 arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
2066 arith_red!(
2067 simd_reduce_add_unordered: vector_reduce_add,
2068 vector_reduce_fadd_reassoc,
2069 false,
2070 add,
2071 -0.0
2072 );
2073 arith_red!(
2074 simd_reduce_mul_unordered: vector_reduce_mul,
2075 vector_reduce_fmul_reassoc,
2076 false,
2077 mul,
2078 1.0
2079 );
2080
2081 macro_rules! minmax_red {
2082 ($name:ident: $int_red:ident, $float_red:ident) => {
2083 if name == sym::$name {
2084 require!(
2085 ret_ty == in_elem,
2086 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2087 );
2088 return match in_elem.kind() {
2089 ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
2090 ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
2091 ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())),
2092 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2093 span,
2094 name,
2095 symbol: sym::$name,
2096 in_ty,
2097 in_elem,
2098 ret_ty
2099 }),
2100 };
2101 }
2102 };
2103 }
2104
2105 minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin);
2106 minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax);
2107
2108 macro_rules! bitwise_red {
2109 ($name:ident : $red:ident, $boolean:expr) => {
2110 if name == sym::$name {
2111 let input = if !$boolean {
2112 require!(
2113 ret_ty == in_elem,
2114 InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
2115 );
2116 args[0].immediate()
2117 } else {
2118 let bitwidth = match in_elem.kind() {
2119 ty::Int(i) => {
2120 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
2121 }
2122 ty::Uint(i) => {
2123 i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits())
2124 }
2125 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2126 span,
2127 name,
2128 symbol: sym::$name,
2129 in_ty,
2130 in_elem,
2131 ret_ty
2132 }),
2133 };
2134
2135 vector_mask_to_bitmask(bx, args[0].immediate(), bitwidth, in_len as _)
2136 };
2137 return match in_elem.kind() {
2138 ty::Int(_) | ty::Uint(_) => {
2139 let r = bx.$red(input);
2140 Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
2141 }
2142 _ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
2143 span,
2144 name,
2145 symbol: sym::$name,
2146 in_ty,
2147 in_elem,
2148 ret_ty
2149 }),
2150 };
2151 }
2152 };
2153 }
2154
2155 bitwise_red!(simd_reduce_and: vector_reduce_and, false);
2156 bitwise_red!(simd_reduce_or: vector_reduce_or, false);
2157 bitwise_red!(simd_reduce_xor: vector_reduce_xor, false);
2158 bitwise_red!(simd_reduce_all: vector_reduce_and, true);
2159 bitwise_red!(simd_reduce_any: vector_reduce_or, true);
2160
2161 if name == sym::simd_cast_ptr {
2162 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2163 require!(
2164 in_len == out_len,
2165 InvalidMonomorphization::ReturnLengthInputType {
2166 span,
2167 name,
2168 in_len,
2169 in_ty,
2170 ret_ty,
2171 out_len
2172 }
2173 );
2174
2175 match in_elem.kind() {
2176 ty::RawPtr(p_ty, _) => {
2177 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2178 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2179 });
2180 require!(
2181 metadata.is_unit(),
2182 InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
2183 );
2184 }
2185 _ => {
2186 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2187 }
2188 }
2189 match out_elem.kind() {
2190 ty::RawPtr(p_ty, _) => {
2191 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
2192 bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
2193 });
2194 require!(
2195 metadata.is_unit(),
2196 InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
2197 );
2198 }
2199 _ => {
2200 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2201 }
2202 }
2203
2204 return Ok(args[0].immediate());
2205 }
2206
2207 if name == sym::simd_expose_provenance {
2208 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2209 require!(
2210 in_len == out_len,
2211 InvalidMonomorphization::ReturnLengthInputType {
2212 span,
2213 name,
2214 in_len,
2215 in_ty,
2216 ret_ty,
2217 out_len
2218 }
2219 );
2220
2221 match in_elem.kind() {
2222 ty::RawPtr(_, _) => {}
2223 _ => {
2224 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
2225 }
2226 }
2227 match out_elem.kind() {
2228 ty::Uint(ty::UintTy::Usize) => {}
2229 _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
2230 }
2231
2232 return Ok(bx.ptrtoint(args[0].immediate(), llret_ty));
2233 }
2234
2235 if name == sym::simd_with_exposed_provenance {
2236 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2237 require!(
2238 in_len == out_len,
2239 InvalidMonomorphization::ReturnLengthInputType {
2240 span,
2241 name,
2242 in_len,
2243 in_ty,
2244 ret_ty,
2245 out_len
2246 }
2247 );
2248
2249 match in_elem.kind() {
2250 ty::Uint(ty::UintTy::Usize) => {}
2251 _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
2252 }
2253 match out_elem.kind() {
2254 ty::RawPtr(_, _) => {}
2255 _ => {
2256 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
2257 }
2258 }
2259
2260 return Ok(bx.inttoptr(args[0].immediate(), llret_ty));
2261 }
2262
2263 if name == sym::simd_cast || name == sym::simd_as {
2264 let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
2265 require!(
2266 in_len == out_len,
2267 InvalidMonomorphization::ReturnLengthInputType {
2268 span,
2269 name,
2270 in_len,
2271 in_ty,
2272 ret_ty,
2273 out_len
2274 }
2275 );
2276 if in_elem == out_elem {
2278 return Ok(args[0].immediate());
2279 }
2280
2281 #[derive(Copy, Clone)]
2282 enum Sign {
2283 Unsigned,
2284 Signed,
2285 }
2286 use Sign::*;
2287
2288 enum Style {
2289 Float,
2290 Int(Sign),
2291 Unsupported,
2292 }
2293
2294 let (in_style, in_width) = match in_elem.kind() {
2295 ty::Int(i) => (
2298 Style::Int(Signed),
2299 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2300 ),
2301 ty::Uint(u) => (
2302 Style::Int(Unsigned),
2303 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2304 ),
2305 ty::Float(f) => (Style::Float, f.bit_width()),
2306 _ => (Style::Unsupported, 0),
2307 };
2308 let (out_style, out_width) = match out_elem.kind() {
2309 ty::Int(i) => (
2310 Style::Int(Signed),
2311 i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2312 ),
2313 ty::Uint(u) => (
2314 Style::Int(Unsigned),
2315 u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(),
2316 ),
2317 ty::Float(f) => (Style::Float, f.bit_width()),
2318 _ => (Style::Unsupported, 0),
2319 };
2320
2321 match (in_style, out_style) {
2322 (Style::Int(sign), Style::Int(_)) => {
2323 return Ok(match in_width.cmp(&out_width) {
2324 Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty),
2325 Ordering::Equal => args[0].immediate(),
2326 Ordering::Less => match sign {
2327 Sign::Signed => bx.sext(args[0].immediate(), llret_ty),
2328 Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty),
2329 },
2330 });
2331 }
2332 (Style::Int(Sign::Signed), Style::Float) => {
2333 return Ok(bx.sitofp(args[0].immediate(), llret_ty));
2334 }
2335 (Style::Int(Sign::Unsigned), Style::Float) => {
2336 return Ok(bx.uitofp(args[0].immediate(), llret_ty));
2337 }
2338 (Style::Float, Style::Int(sign)) => {
2339 return Ok(match (sign, name == sym::simd_as) {
2340 (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty),
2341 (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty),
2342 (_, true) => bx.cast_float_to_int(
2343 matches!(sign, Sign::Signed),
2344 args[0].immediate(),
2345 llret_ty,
2346 ),
2347 });
2348 }
2349 (Style::Float, Style::Float) => {
2350 return Ok(match in_width.cmp(&out_width) {
2351 Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty),
2352 Ordering::Equal => args[0].immediate(),
2353 Ordering::Less => bx.fpext(args[0].immediate(), llret_ty),
2354 });
2355 }
2356 _ => { }
2357 }
2358 return_error!(InvalidMonomorphization::UnsupportedCast {
2359 span,
2360 name,
2361 in_ty,
2362 in_elem,
2363 ret_ty,
2364 out_elem
2365 });
2366 }
2367 macro_rules! arith_binary {
2368 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2369 $(if name == sym::$name {
2370 match in_elem.kind() {
2371 $($(ty::$p(_))|* => {
2372 return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
2373 })*
2374 _ => {},
2375 }
2376 return_error!(
2377 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2378 );
2379 })*
2380 }
2381 }
2382 arith_binary! {
2383 simd_add: Uint, Int => add, Float => fadd;
2384 simd_sub: Uint, Int => sub, Float => fsub;
2385 simd_mul: Uint, Int => mul, Float => fmul;
2386 simd_div: Uint => udiv, Int => sdiv, Float => fdiv;
2387 simd_rem: Uint => urem, Int => srem, Float => frem;
2388 simd_shl: Uint, Int => shl;
2389 simd_shr: Uint => lshr, Int => ashr;
2390 simd_and: Uint, Int => and;
2391 simd_or: Uint, Int => or;
2392 simd_xor: Uint, Int => xor;
2393 simd_fmax: Float => maxnum;
2394 simd_fmin: Float => minnum;
2395
2396 }
2397 macro_rules! arith_unary {
2398 ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
2399 $(if name == sym::$name {
2400 match in_elem.kind() {
2401 $($(ty::$p(_))|* => {
2402 return Ok(bx.$call(args[0].immediate()))
2403 })*
2404 _ => {},
2405 }
2406 return_error!(
2407 InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
2408 );
2409 })*
2410 }
2411 }
2412 arith_unary! {
2413 simd_neg: Int => neg, Float => fneg;
2414 }
2415
2416 if matches!(
2418 name,
2419 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz
2420 ) {
2421 let vec_ty = bx.cx.type_vector(
2422 match *in_elem.kind() {
2423 ty::Int(i) => bx.cx.type_int_from_ty(i),
2424 ty::Uint(i) => bx.cx.type_uint_from_ty(i),
2425 _ => return_error!(InvalidMonomorphization::UnsupportedOperation {
2426 span,
2427 name,
2428 in_ty,
2429 in_elem
2430 }),
2431 },
2432 in_len as u64,
2433 );
2434 let intrinsic_name = match name {
2435 sym::simd_bswap => "bswap",
2436 sym::simd_bitreverse => "bitreverse",
2437 sym::simd_ctlz => "ctlz",
2438 sym::simd_ctpop => "ctpop",
2439 sym::simd_cttz => "cttz",
2440 _ => unreachable!(),
2441 };
2442 let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
2443 let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,);
2444
2445 return match name {
2446 sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
2448 sym::simd_ctlz | sym::simd_cttz => {
2449 let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
2451 let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
2452 let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2453 Ok(bx.call(
2454 fn_ty,
2455 None,
2456 None,
2457 f,
2458 &[args[0].immediate(), dont_poison_on_zero],
2459 None,
2460 None,
2461 ))
2462 }
2463 sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
2464 let fn_ty = bx.type_func(&[vec_ty], vec_ty);
2466 let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2467 Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
2468 }
2469 _ => unreachable!(),
2470 };
2471 }
2472
2473 if name == sym::simd_arith_offset {
2474 let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| {
2476 span_bug!(span, "must be called with a vector of pointer types as first argument")
2477 });
2478 let layout = bx.layout_of(pointee);
2479 let ptrs = args[0].immediate();
2480 let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
2483 if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
2484 span_bug!(
2485 span,
2486 "must be called with a vector of pointer-sized integers as second argument"
2487 );
2488 }
2489 let offsets = args[1].immediate();
2490
2491 return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets]));
2492 }
2493
2494 if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
2495 let lhs = args[0].immediate();
2496 let rhs = args[1].immediate();
2497 let is_add = name == sym::simd_saturating_add;
2498 let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
2499 let (signed, elem_width, elem_ty) = match *in_elem.kind() {
2500 ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
2501 ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
2502 _ => {
2503 return_error!(InvalidMonomorphization::ExpectedVectorElementType {
2504 span,
2505 name,
2506 expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
2507 vector_type: args[0].layout.ty
2508 });
2509 }
2510 };
2511 let llvm_intrinsic = &format!(
2512 "llvm.{}{}.sat.v{}i{}",
2513 if signed { 's' } else { 'u' },
2514 if is_add { "add" } else { "sub" },
2515 in_len,
2516 elem_width
2517 );
2518 let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
2519
2520 let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty);
2521 let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2522 let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None, None);
2523 return Ok(v);
2524 }
2525
2526 span_bug!(span, "unknown SIMD intrinsic");
2527}