rustc_codegen_ssa/mir/
operand.rs

1use std::fmt;
2
3use arrayvec::ArrayVec;
4use either::Either;
5use rustc_abi as abi;
6use rustc_abi::{Align, BackendRepr, FIRST_VARIANT, Primitive, Size, TagEncoding, Variants};
7use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range};
8use rustc_middle::mir::{self, ConstValue};
9use rustc_middle::ty::Ty;
10use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
11use rustc_middle::{bug, span_bug};
12use rustc_session::config::OptLevel;
13use tracing::{debug, instrument};
14
15use super::place::{PlaceRef, PlaceValue};
16use super::{FunctionCx, LocalRef};
17use crate::common::IntPredicate;
18use crate::traits::*;
19use crate::{MemFlags, size_of_val};
20
21/// The representation of a Rust value. The enum variant is in fact
22/// uniquely determined by the value's type, but is kept as a
23/// safety check.
24#[derive(Copy, Clone, Debug)]
25pub enum OperandValue<V> {
26    /// A reference to the actual operand. The data is guaranteed
27    /// to be valid for the operand's lifetime.
28    /// The second value, if any, is the extra data (vtable or length)
29    /// which indicates that it refers to an unsized rvalue.
30    ///
31    /// An `OperandValue` *must* be this variant for any type for which
32    /// [`LayoutTypeCodegenMethods::is_backend_ref`] returns `true`.
33    /// (That basically amounts to "isn't one of the other variants".)
34    ///
35    /// This holds a [`PlaceValue`] (like a [`PlaceRef`] does) with a pointer
36    /// to the location holding the value. The type behind that pointer is the
37    /// one returned by [`LayoutTypeCodegenMethods::backend_type`].
38    Ref(PlaceValue<V>),
39    /// A single LLVM immediate value.
40    ///
41    /// An `OperandValue` *must* be this variant for any type for which
42    /// [`LayoutTypeCodegenMethods::is_backend_immediate`] returns `true`.
43    /// The backend value in this variant must be the *immediate* backend type,
44    /// as returned by [`LayoutTypeCodegenMethods::immediate_backend_type`].
45    Immediate(V),
46    /// A pair of immediate LLVM values. Used by wide pointers too.
47    ///
48    /// # Invariants
49    /// - For `Pair(a, b)`, `a` is always at offset 0, but may have `FieldIdx(1..)`
50    /// - `b` is not at offset 0, because `V` is not a 1ZST type.
51    /// - `a` and `b` will have a different FieldIdx, but otherwise `b`'s may be lower
52    ///   or they may not be adjacent, due to arbitrary numbers of 1ZST fields that
53    ///   will not affect the shape of the data which determines if `Pair` will be used.
54    /// - An `OperandValue` *must* be this variant for any type for which
55    /// [`LayoutTypeCodegenMethods::is_backend_scalar_pair`] returns `true`.
56    /// - The backend values in this variant must be the *immediate* backend types,
57    /// as returned by [`LayoutTypeCodegenMethods::scalar_pair_element_backend_type`]
58    /// with `immediate: true`.
59    Pair(V, V),
60    /// A value taking no bytes, and which therefore needs no LLVM value at all.
61    ///
62    /// If you ever need a `V` to pass to something, get a fresh poison value
63    /// from [`ConstCodegenMethods::const_poison`].
64    ///
65    /// An `OperandValue` *must* be this variant for any type for which
66    /// `is_zst` on its `Layout` returns `true`. Note however that
67    /// these values can still require alignment.
68    ZeroSized,
69}
70
71impl<V: CodegenObject> OperandValue<V> {
72    /// If this is ZeroSized/Immediate/Pair, return an array of the 0/1/2 values.
73    /// If this is Ref, return the place.
74    #[inline]
75    pub(crate) fn immediates_or_place(self) -> Either<ArrayVec<V, 2>, PlaceValue<V>> {
76        match self {
77            OperandValue::ZeroSized => Either::Left(ArrayVec::new()),
78            OperandValue::Immediate(a) => Either::Left(ArrayVec::from_iter([a])),
79            OperandValue::Pair(a, b) => Either::Left([a, b].into()),
80            OperandValue::Ref(p) => Either::Right(p),
81        }
82    }
83
84    /// Given an array of 0/1/2 immediate values, return ZeroSized/Immediate/Pair.
85    #[inline]
86    pub(crate) fn from_immediates(immediates: ArrayVec<V, 2>) -> Self {
87        let mut it = immediates.into_iter();
88        let Some(a) = it.next() else {
89            return OperandValue::ZeroSized;
90        };
91        let Some(b) = it.next() else {
92            return OperandValue::Immediate(a);
93        };
94        OperandValue::Pair(a, b)
95    }
96
97    /// Treat this value as a pointer and return the data pointer and
98    /// optional metadata as backend values.
99    ///
100    /// If you're making a place, use [`Self::deref`] instead.
101    pub(crate) fn pointer_parts(self) -> (V, Option<V>) {
102        match self {
103            OperandValue::Immediate(llptr) => (llptr, None),
104            OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)),
105            _ => bug!("OperandValue cannot be a pointer: {self:?}"),
106        }
107    }
108
109    /// Treat this value as a pointer and return the place to which it points.
110    ///
111    /// The pointer immediate doesn't inherently know its alignment,
112    /// so you need to pass it in. If you want to get it from a type's ABI
113    /// alignment, then maybe you want [`OperandRef::deref`] instead.
114    ///
115    /// This is the inverse of [`PlaceValue::address`].
116    pub(crate) fn deref(self, align: Align) -> PlaceValue<V> {
117        let (llval, llextra) = self.pointer_parts();
118        PlaceValue { llval, llextra, align }
119    }
120
121    pub(crate) fn is_expected_variant_for_type<'tcx, Cx: LayoutTypeCodegenMethods<'tcx>>(
122        &self,
123        cx: &Cx,
124        ty: TyAndLayout<'tcx>,
125    ) -> bool {
126        match self {
127            OperandValue::ZeroSized => ty.is_zst(),
128            OperandValue::Immediate(_) => cx.is_backend_immediate(ty),
129            OperandValue::Pair(_, _) => cx.is_backend_scalar_pair(ty),
130            OperandValue::Ref(_) => cx.is_backend_ref(ty),
131        }
132    }
133}
134
135/// An `OperandRef` is an "SSA" reference to a Rust value, along with
136/// its type.
137///
138/// NOTE: unless you know a value's type exactly, you should not
139/// generate LLVM opcodes acting on it and instead act via methods,
140/// to avoid nasty edge cases. In particular, using `Builder::store`
141/// directly is sure to cause problems -- use `OperandRef::store`
142/// instead.
143#[derive(Copy, Clone)]
144pub struct OperandRef<'tcx, V> {
145    /// The value.
146    pub val: OperandValue<V>,
147
148    /// The layout of value, based on its Rust type.
149    pub layout: TyAndLayout<'tcx>,
150}
151
152impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
153    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
154        write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
155    }
156}
157
158impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
159    pub fn zero_sized(layout: TyAndLayout<'tcx>) -> OperandRef<'tcx, V> {
160        assert!(layout.is_zst());
161        OperandRef { val: OperandValue::ZeroSized, layout }
162    }
163
164    pub(crate) fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
165        bx: &mut Bx,
166        val: mir::ConstValue<'tcx>,
167        ty: Ty<'tcx>,
168    ) -> Self {
169        let layout = bx.layout_of(ty);
170
171        let val = match val {
172            ConstValue::Scalar(x) => {
173                let BackendRepr::Scalar(scalar) = layout.backend_repr else {
174                    bug!("from_const: invalid ByVal layout: {:#?}", layout);
175                };
176                let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
177                OperandValue::Immediate(llval)
178            }
179            ConstValue::ZeroSized => return OperandRef::zero_sized(layout),
180            ConstValue::Slice { data, meta } => {
181                let BackendRepr::ScalarPair(a_scalar, _) = layout.backend_repr else {
182                    bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
183                };
184                let a = Scalar::from_pointer(
185                    Pointer::new(bx.tcx().reserve_and_set_memory_alloc(data).into(), Size::ZERO),
186                    &bx.tcx(),
187                );
188                let a_llval = bx.scalar_to_backend(
189                    a,
190                    a_scalar,
191                    bx.scalar_pair_element_backend_type(layout, 0, true),
192                );
193                let b_llval = bx.const_usize(meta);
194                OperandValue::Pair(a_llval, b_llval)
195            }
196            ConstValue::Indirect { alloc_id, offset } => {
197                let alloc = bx.tcx().global_alloc(alloc_id).unwrap_memory();
198                return Self::from_const_alloc(bx, layout, alloc, offset);
199            }
200        };
201
202        OperandRef { val, layout }
203    }
204
205    fn from_const_alloc<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
206        bx: &mut Bx,
207        layout: TyAndLayout<'tcx>,
208        alloc: rustc_middle::mir::interpret::ConstAllocation<'tcx>,
209        offset: Size,
210    ) -> Self {
211        let alloc_align = alloc.inner().align;
212        assert!(alloc_align >= layout.align.abi);
213
214        let read_scalar = |start, size, s: abi::Scalar, ty| {
215            match alloc.0.read_scalar(
216                bx,
217                alloc_range(start, size),
218                /*read_provenance*/ matches!(s.primitive(), abi::Primitive::Pointer(_)),
219            ) {
220                Ok(val) => bx.scalar_to_backend(val, s, ty),
221                Err(_) => bx.const_poison(ty),
222            }
223        };
224
225        // It may seem like all types with `Scalar` or `ScalarPair` ABI are fair game at this point.
226        // However, `MaybeUninit<u64>` is considered a `Scalar` as far as its layout is concerned --
227        // and yet cannot be represented by an interpreter `Scalar`, since we have to handle the
228        // case where some of the bytes are initialized and others are not. So, we need an extra
229        // check that walks over the type of `mplace` to make sure it is truly correct to treat this
230        // like a `Scalar` (or `ScalarPair`).
231        match layout.backend_repr {
232            BackendRepr::Scalar(s @ abi::Scalar::Initialized { .. }) => {
233                let size = s.size(bx);
234                assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
235                let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
236                OperandRef { val: OperandValue::Immediate(val), layout }
237            }
238            BackendRepr::ScalarPair(
239                a @ abi::Scalar::Initialized { .. },
240                b @ abi::Scalar::Initialized { .. },
241            ) => {
242                let (a_size, b_size) = (a.size(bx), b.size(bx));
243                let b_offset = (offset + a_size).align_to(b.align(bx).abi);
244                assert!(b_offset.bytes() > 0);
245                let a_val = read_scalar(
246                    offset,
247                    a_size,
248                    a,
249                    bx.scalar_pair_element_backend_type(layout, 0, true),
250                );
251                let b_val = read_scalar(
252                    b_offset,
253                    b_size,
254                    b,
255                    bx.scalar_pair_element_backend_type(layout, 1, true),
256                );
257                OperandRef { val: OperandValue::Pair(a_val, b_val), layout }
258            }
259            _ if layout.is_zst() => OperandRef::zero_sized(layout),
260            _ => {
261                // Neither a scalar nor scalar pair. Load from a place
262                // FIXME: should we cache `const_data_from_alloc` to avoid repeating this for the
263                // same `ConstAllocation`?
264                let init = bx.const_data_from_alloc(alloc);
265                let base_addr = bx.static_addr_of(init, alloc_align, None);
266
267                let llval = bx.const_ptr_byte_offset(base_addr, offset);
268                bx.load_operand(PlaceRef::new_sized(llval, layout))
269            }
270        }
271    }
272
273    /// Asserts that this operand refers to a scalar and returns
274    /// a reference to its value.
275    pub fn immediate(self) -> V {
276        match self.val {
277            OperandValue::Immediate(s) => s,
278            _ => bug!("not immediate: {:?}", self),
279        }
280    }
281
282    /// Asserts that this operand is a pointer (or reference) and returns
283    /// the place to which it points.  (This requires no code to be emitted
284    /// as we represent places using the pointer to the place.)
285    ///
286    /// This uses [`Ty::builtin_deref`] to include the type of the place and
287    /// assumes the place is aligned to the pointee's usual ABI alignment.
288    ///
289    /// If you don't need the type, see [`OperandValue::pointer_parts`]
290    /// or [`OperandValue::deref`].
291    pub fn deref<Cx: CodegenMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> {
292        if self.layout.ty.is_box() {
293            // Derefer should have removed all Box derefs
294            bug!("dereferencing {:?} in codegen", self.layout.ty);
295        }
296
297        let projected_ty = self
298            .layout
299            .ty
300            .builtin_deref(true)
301            .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self));
302
303        let layout = cx.layout_of(projected_ty);
304        self.val.deref(layout.align.abi).with_type(layout)
305    }
306
307    /// If this operand is a `Pair`, we return an aggregate with the two values.
308    /// For other cases, see `immediate`.
309    pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
310        self,
311        bx: &mut Bx,
312    ) -> V {
313        if let OperandValue::Pair(a, b) = self.val {
314            let llty = bx.cx().immediate_backend_type(self.layout);
315            debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
316            // Reconstruct the immediate aggregate.
317            let mut llpair = bx.cx().const_poison(llty);
318            llpair = bx.insert_value(llpair, a, 0);
319            llpair = bx.insert_value(llpair, b, 1);
320            llpair
321        } else {
322            self.immediate()
323        }
324    }
325
326    /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
327    pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
328        bx: &mut Bx,
329        llval: V,
330        layout: TyAndLayout<'tcx>,
331    ) -> Self {
332        let val = if let BackendRepr::ScalarPair(..) = layout.backend_repr {
333            debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout);
334
335            // Deconstruct the immediate aggregate.
336            let a_llval = bx.extract_value(llval, 0);
337            let b_llval = bx.extract_value(llval, 1);
338            OperandValue::Pair(a_llval, b_llval)
339        } else {
340            OperandValue::Immediate(llval)
341        };
342        OperandRef { val, layout }
343    }
344
345    pub(crate) fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
346        &self,
347        fx: &mut FunctionCx<'a, 'tcx, Bx>,
348        bx: &mut Bx,
349        i: usize,
350    ) -> Self {
351        let field = self.layout.field(bx.cx(), i);
352        let offset = self.layout.fields.offset(i);
353
354        if !bx.is_backend_ref(self.layout) && bx.is_backend_ref(field) {
355            if let BackendRepr::SimdVector { count, .. } = self.layout.backend_repr
356                && let BackendRepr::Memory { sized: true } = field.backend_repr
357                && count.is_power_of_two()
358            {
359                assert_eq!(field.size, self.layout.size);
360                // This is being deprecated, but for now stdarch still needs it for
361                // Newtype vector of array, e.g. #[repr(simd)] struct S([i32; 4]);
362                let place = PlaceRef::alloca(bx, field);
363                self.val.store(bx, place.val.with_type(self.layout));
364                return bx.load_operand(place);
365            } else {
366                // Part of https://github.com/rust-lang/compiler-team/issues/838
367                bug!("Non-ref type {self:?} cannot project to ref field type {field:?}");
368            }
369        }
370
371        let val = if field.is_zst() {
372            OperandValue::ZeroSized
373        } else if field.size == self.layout.size {
374            assert_eq!(offset.bytes(), 0);
375            fx.codegen_transmute_operand(bx, *self, field).unwrap_or_else(|| {
376                bug!(
377                    "Expected `codegen_transmute_operand` to handle equal-size \
378                      field {i:?} projection from {self:?} to {field:?}"
379                )
380            })
381        } else {
382            let (in_scalar, imm) = match (self.val, self.layout.backend_repr) {
383                // Extract a scalar component from a pair.
384                (OperandValue::Pair(a_llval, b_llval), BackendRepr::ScalarPair(a, b)) => {
385                    if offset.bytes() == 0 {
386                        assert_eq!(field.size, a.size(bx.cx()));
387                        (Some(a), a_llval)
388                    } else {
389                        assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi));
390                        assert_eq!(field.size, b.size(bx.cx()));
391                        (Some(b), b_llval)
392                    }
393                }
394
395                _ => {
396                    span_bug!(fx.mir.span, "OperandRef::extract_field({:?}): not applicable", self)
397                }
398            };
399            OperandValue::Immediate(match field.backend_repr {
400                BackendRepr::SimdVector { .. } => imm,
401                BackendRepr::Scalar(out_scalar) => {
402                    let Some(in_scalar) = in_scalar else {
403                        span_bug!(
404                            fx.mir.span,
405                            "OperandRef::extract_field({:?}): missing input scalar for output scalar",
406                            self
407                        )
408                    };
409                    if in_scalar != out_scalar {
410                        // If the backend and backend_immediate types might differ,
411                        // flip back to the backend type then to the new immediate.
412                        // This avoids nop truncations, but still handles things like
413                        // Bools in union fields needs to be truncated.
414                        let backend = bx.from_immediate(imm);
415                        bx.to_immediate_scalar(backend, out_scalar)
416                    } else {
417                        imm
418                    }
419                }
420                BackendRepr::ScalarPair(_, _) | BackendRepr::Memory { .. } => bug!(),
421            })
422        };
423
424        OperandRef { val, layout: field }
425    }
426
427    /// Obtain the actual discriminant of a value.
428    #[instrument(level = "trace", skip(fx, bx))]
429    pub fn codegen_get_discr<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
430        self,
431        fx: &mut FunctionCx<'a, 'tcx, Bx>,
432        bx: &mut Bx,
433        cast_to: Ty<'tcx>,
434    ) -> V {
435        let dl = &bx.tcx().data_layout;
436        let cast_to_layout = bx.cx().layout_of(cast_to);
437        let cast_to = bx.cx().immediate_backend_type(cast_to_layout);
438
439        // We check uninhabitedness separately because a type like
440        // `enum Foo { Bar(i32, !) }` is still reported as `Variants::Single`,
441        // *not* as `Variants::Empty`.
442        if self.layout.is_uninhabited() {
443            return bx.cx().const_poison(cast_to);
444        }
445
446        let (tag_scalar, tag_encoding, tag_field) = match self.layout.variants {
447            Variants::Empty => unreachable!("we already handled uninhabited types"),
448            Variants::Single { index } => {
449                let discr_val =
450                    if let Some(discr) = self.layout.ty.discriminant_for_variant(bx.tcx(), index) {
451                        discr.val
452                    } else {
453                        // This arm is for types which are neither enums nor coroutines,
454                        // and thus for which the only possible "variant" should be the first one.
455                        assert_eq!(index, FIRST_VARIANT);
456                        // There's thus no actual discriminant to return, so we return
457                        // what it would have been if this was a single-variant enum.
458                        0
459                    };
460                return bx.cx().const_uint_big(cast_to, discr_val);
461            }
462            Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => {
463                (tag, tag_encoding, tag_field)
464            }
465        };
466
467        // Read the tag/niche-encoded discriminant from memory.
468        let tag_op = match self.val {
469            OperandValue::ZeroSized => bug!(),
470            OperandValue::Immediate(_) | OperandValue::Pair(_, _) => {
471                self.extract_field(fx, bx, tag_field.as_usize())
472            }
473            OperandValue::Ref(place) => {
474                let tag = place.with_type(self.layout).project_field(bx, tag_field.as_usize());
475                bx.load_operand(tag)
476            }
477        };
478        let tag_imm = tag_op.immediate();
479
480        // Decode the discriminant (specifically if it's niche-encoded).
481        match *tag_encoding {
482            TagEncoding::Direct => {
483                let signed = match tag_scalar.primitive() {
484                    // We use `i1` for bytes that are always `0` or `1`,
485                    // e.g., `#[repr(i8)] enum E { A, B }`, but we can't
486                    // let LLVM interpret the `i1` as signed, because
487                    // then `i1 1` (i.e., `E::B`) is effectively `i8 -1`.
488                    Primitive::Int(_, signed) => !tag_scalar.is_bool() && signed,
489                    _ => false,
490                };
491                bx.intcast(tag_imm, cast_to, signed)
492            }
493            TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
494                // Cast to an integer so we don't have to treat a pointer as a
495                // special case.
496                let (tag, tag_llty) = match tag_scalar.primitive() {
497                    // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
498                    Primitive::Pointer(_) => {
499                        let t = bx.type_from_integer(dl.ptr_sized_integer());
500                        let tag = bx.ptrtoint(tag_imm, t);
501                        (tag, t)
502                    }
503                    _ => (tag_imm, bx.cx().immediate_backend_type(tag_op.layout)),
504                };
505
506                // Layout ensures that we only get here for cases where the discriminant
507                // value and the variant index match, since that's all `Niche` can encode.
508                // But for emphasis and debugging, let's double-check one anyway.
509                debug_assert_eq!(
510                    self.layout
511                        .ty
512                        .discriminant_for_variant(bx.tcx(), untagged_variant)
513                        .unwrap()
514                        .val,
515                    u128::from(untagged_variant.as_u32()),
516                );
517
518                let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
519
520                // We have a subrange `niche_start..=niche_end` inside `range`.
521                // If the value of the tag is inside this subrange, it's a
522                // "niche value", an increment of the discriminant. Otherwise it
523                // indicates the untagged variant.
524                // A general algorithm to extract the discriminant from the tag
525                // is:
526                // relative_tag = tag - niche_start
527                // is_niche = relative_tag <= (ule) relative_max
528                // discr = if is_niche {
529                //     cast(relative_tag) + niche_variants.start()
530                // } else {
531                //     untagged_variant
532                // }
533                // However, we will likely be able to emit simpler code.
534                let (is_niche, tagged_discr, delta) = if relative_max == 0 {
535                    // Best case scenario: only one tagged variant. This will
536                    // likely become just a comparison and a jump.
537                    // The algorithm is:
538                    // is_niche = tag == niche_start
539                    // discr = if is_niche {
540                    //     niche_start
541                    // } else {
542                    //     untagged_variant
543                    // }
544                    let niche_start = bx.cx().const_uint_big(tag_llty, niche_start);
545                    let is_niche = bx.icmp(IntPredicate::IntEQ, tag, niche_start);
546                    let tagged_discr =
547                        bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64);
548                    (is_niche, tagged_discr, 0)
549                } else {
550                    // The special cases don't apply, so we'll have to go with
551                    // the general algorithm.
552                    let relative_discr = bx.sub(tag, bx.cx().const_uint_big(tag_llty, niche_start));
553                    let cast_tag = bx.intcast(relative_discr, cast_to, false);
554                    let is_niche = bx.icmp(
555                        IntPredicate::IntULE,
556                        relative_discr,
557                        bx.cx().const_uint(tag_llty, relative_max as u64),
558                    );
559
560                    // Thanks to parameter attributes and load metadata, LLVM already knows
561                    // the general valid range of the tag. It's possible, though, for there
562                    // to be an impossible value *in the middle*, which those ranges don't
563                    // communicate, so it's worth an `assume` to let the optimizer know.
564                    if niche_variants.contains(&untagged_variant)
565                        && bx.cx().sess().opts.optimize != OptLevel::No
566                    {
567                        let impossible =
568                            u64::from(untagged_variant.as_u32() - niche_variants.start().as_u32());
569                        let impossible = bx.cx().const_uint(tag_llty, impossible);
570                        let ne = bx.icmp(IntPredicate::IntNE, relative_discr, impossible);
571                        bx.assume(ne);
572                    }
573
574                    (is_niche, cast_tag, niche_variants.start().as_u32() as u128)
575                };
576
577                let tagged_discr = if delta == 0 {
578                    tagged_discr
579                } else {
580                    bx.add(tagged_discr, bx.cx().const_uint_big(cast_to, delta))
581                };
582
583                let discr = bx.select(
584                    is_niche,
585                    tagged_discr,
586                    bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64),
587                );
588
589                // In principle we could insert assumes on the possible range of `discr`, but
590                // currently in LLVM this isn't worth it because the original `tag` will
591                // have either a `range` parameter attribute or `!range` metadata,
592                // or come from a `transmute` that already `assume`d it.
593
594                discr
595            }
596        }
597    }
598}
599
600impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
601    /// Returns an `OperandValue` that's generally UB to use in any way.
602    ///
603    /// Depending on the `layout`, returns `ZeroSized` for ZSTs, an `Immediate` or
604    /// `Pair` containing poison value(s), or a `Ref` containing a poison pointer.
605    ///
606    /// Supports sized types only.
607    pub fn poison<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
608        bx: &mut Bx,
609        layout: TyAndLayout<'tcx>,
610    ) -> OperandValue<V> {
611        assert!(layout.is_sized());
612        if layout.is_zst() {
613            OperandValue::ZeroSized
614        } else if bx.cx().is_backend_immediate(layout) {
615            let ibty = bx.cx().immediate_backend_type(layout);
616            OperandValue::Immediate(bx.const_poison(ibty))
617        } else if bx.cx().is_backend_scalar_pair(layout) {
618            let ibty0 = bx.cx().scalar_pair_element_backend_type(layout, 0, true);
619            let ibty1 = bx.cx().scalar_pair_element_backend_type(layout, 1, true);
620            OperandValue::Pair(bx.const_poison(ibty0), bx.const_poison(ibty1))
621        } else {
622            let ptr = bx.cx().type_ptr();
623            OperandValue::Ref(PlaceValue::new_sized(bx.const_poison(ptr), layout.align.abi))
624        }
625    }
626
627    pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
628        self,
629        bx: &mut Bx,
630        dest: PlaceRef<'tcx, V>,
631    ) {
632        self.store_with_flags(bx, dest, MemFlags::empty());
633    }
634
635    pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
636        self,
637        bx: &mut Bx,
638        dest: PlaceRef<'tcx, V>,
639    ) {
640        self.store_with_flags(bx, dest, MemFlags::VOLATILE);
641    }
642
643    pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
644        self,
645        bx: &mut Bx,
646        dest: PlaceRef<'tcx, V>,
647    ) {
648        self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
649    }
650
651    pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
652        self,
653        bx: &mut Bx,
654        dest: PlaceRef<'tcx, V>,
655    ) {
656        self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
657    }
658
659    pub(crate) fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
660        self,
661        bx: &mut Bx,
662        dest: PlaceRef<'tcx, V>,
663        flags: MemFlags,
664    ) {
665        debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
666        match self {
667            OperandValue::ZeroSized => {
668                // Avoid generating stores of zero-sized values, because the only way to have a
669                // zero-sized value is through `undef`/`poison`, and the store itself is useless.
670            }
671            OperandValue::Ref(val) => {
672                assert!(dest.layout.is_sized(), "cannot directly store unsized values");
673                if val.llextra.is_some() {
674                    bug!("cannot directly store unsized values");
675                }
676                bx.typed_place_copy_with_flags(dest.val, val, dest.layout, flags);
677            }
678            OperandValue::Immediate(s) => {
679                let val = bx.from_immediate(s);
680                bx.store_with_flags(val, dest.val.llval, dest.val.align, flags);
681            }
682            OperandValue::Pair(a, b) => {
683                let BackendRepr::ScalarPair(a_scalar, b_scalar) = dest.layout.backend_repr else {
684                    bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout);
685                };
686                let b_offset = a_scalar.size(bx).align_to(b_scalar.align(bx).abi);
687
688                let val = bx.from_immediate(a);
689                let align = dest.val.align;
690                bx.store_with_flags(val, dest.val.llval, align, flags);
691
692                let llptr = bx.inbounds_ptradd(dest.val.llval, bx.const_usize(b_offset.bytes()));
693                let val = bx.from_immediate(b);
694                let align = dest.val.align.restrict_for_offset(b_offset);
695                bx.store_with_flags(val, llptr, align, flags);
696            }
697        }
698    }
699
700    pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
701        self,
702        bx: &mut Bx,
703        indirect_dest: PlaceRef<'tcx, V>,
704    ) {
705        debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest);
706        // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
707        let unsized_ty = indirect_dest
708            .layout
709            .ty
710            .builtin_deref(true)
711            .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest));
712
713        let OperandValue::Ref(PlaceValue { llval: llptr, llextra: Some(llextra), .. }) = self
714        else {
715            bug!("store_unsized called with a sized value (or with an extern type)")
716        };
717
718        // Allocate an appropriate region on the stack, and copy the value into it. Since alloca
719        // doesn't support dynamic alignment, we allocate an extra align - 1 bytes, and align the
720        // pointer manually.
721        let (size, align) = size_of_val::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
722        let one = bx.const_usize(1);
723        let align_minus_1 = bx.sub(align, one);
724        let size_extra = bx.add(size, align_minus_1);
725        let min_align = Align::ONE;
726        let alloca = bx.dynamic_alloca(size_extra, min_align);
727        let address = bx.ptrtoint(alloca, bx.type_isize());
728        let neg_address = bx.neg(address);
729        let offset = bx.and(neg_address, align_minus_1);
730        let dst = bx.inbounds_ptradd(alloca, offset);
731        bx.memcpy(dst, min_align, llptr, min_align, size, MemFlags::empty());
732
733        // Store the allocated region and the extra to the indirect place.
734        let indirect_operand = OperandValue::Pair(dst, llextra);
735        indirect_operand.store(bx, indirect_dest);
736    }
737}
738
739impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
740    fn maybe_codegen_consume_direct(
741        &mut self,
742        bx: &mut Bx,
743        place_ref: mir::PlaceRef<'tcx>,
744    ) -> Option<OperandRef<'tcx, Bx::Value>> {
745        debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref);
746
747        match self.locals[place_ref.local] {
748            LocalRef::Operand(mut o) => {
749                // Moves out of scalar and scalar pair fields are trivial.
750                for elem in place_ref.projection.iter() {
751                    match elem {
752                        mir::ProjectionElem::Field(f, _) => {
753                            assert!(
754                                !o.layout.ty.is_any_ptr(),
755                                "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \
756                                 but tried to access field {f:?} of pointer {o:?}",
757                            );
758                            o = o.extract_field(self, bx, f.index());
759                        }
760                        mir::ProjectionElem::Index(_)
761                        | mir::ProjectionElem::ConstantIndex { .. } => {
762                            // ZSTs don't require any actual memory access.
763                            // FIXME(eddyb) deduplicate this with the identical
764                            // checks in `codegen_consume` and `extract_field`.
765                            let elem = o.layout.field(bx.cx(), 0);
766                            if elem.is_zst() {
767                                o = OperandRef::zero_sized(elem);
768                            } else {
769                                return None;
770                            }
771                        }
772                        _ => return None,
773                    }
774                }
775
776                Some(o)
777            }
778            LocalRef::PendingOperand => {
779                bug!("use of {:?} before def", place_ref);
780            }
781            LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
782                // watch out for locals that do not have an
783                // alloca; they are handled somewhat differently
784                None
785            }
786        }
787    }
788
789    pub fn codegen_consume(
790        &mut self,
791        bx: &mut Bx,
792        place_ref: mir::PlaceRef<'tcx>,
793    ) -> OperandRef<'tcx, Bx::Value> {
794        debug!("codegen_consume(place_ref={:?})", place_ref);
795
796        let ty = self.monomorphized_place_ty(place_ref);
797        let layout = bx.cx().layout_of(ty);
798
799        // ZSTs don't require any actual memory access.
800        if layout.is_zst() {
801            return OperandRef::zero_sized(layout);
802        }
803
804        if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) {
805            return o;
806        }
807
808        // for most places, to consume them we just load them
809        // out from their home
810        let place = self.codegen_place(bx, place_ref);
811        bx.load_operand(place)
812    }
813
814    pub fn codegen_operand(
815        &mut self,
816        bx: &mut Bx,
817        operand: &mir::Operand<'tcx>,
818    ) -> OperandRef<'tcx, Bx::Value> {
819        debug!("codegen_operand(operand={:?})", operand);
820
821        match *operand {
822            mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
823                self.codegen_consume(bx, place.as_ref())
824            }
825
826            mir::Operand::Constant(ref constant) => {
827                let constant_ty = self.monomorphize(constant.ty());
828                // Most SIMD vector constants should be passed as immediates.
829                // (In particular, some intrinsics really rely on this.)
830                if constant_ty.is_simd() {
831                    // However, some SIMD types do not actually use the vector ABI
832                    // (in particular, packed SIMD types do not). Ensure we exclude those.
833                    let layout = bx.layout_of(constant_ty);
834                    if let BackendRepr::SimdVector { .. } = layout.backend_repr {
835                        let (llval, ty) = self.immediate_const_vector(bx, constant);
836                        return OperandRef {
837                            val: OperandValue::Immediate(llval),
838                            layout: bx.layout_of(ty),
839                        };
840                    }
841                }
842                self.eval_mir_constant_to_operand(bx, constant)
843            }
844        }
845    }
846}