1use std::fmt;
2
3use arrayvec::ArrayVec;
4use either::Either;
5use rustc_abi as abi;
6use rustc_abi::{Align, BackendRepr, FIRST_VARIANT, Primitive, Size, TagEncoding, Variants};
7use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range};
8use rustc_middle::mir::{self, ConstValue};
9use rustc_middle::ty::Ty;
10use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
11use rustc_middle::{bug, span_bug};
12use rustc_session::config::OptLevel;
13use tracing::{debug, instrument};
14
15use super::place::{PlaceRef, PlaceValue};
16use super::{FunctionCx, LocalRef};
17use crate::common::IntPredicate;
18use crate::traits::*;
19use crate::{MemFlags, size_of_val};
20
21#[derive(Copy, Clone, Debug)]
25pub enum OperandValue<V> {
26 Ref(PlaceValue<V>),
39 Immediate(V),
46 Pair(V, V),
60 ZeroSized,
69}
70
71impl<V: CodegenObject> OperandValue<V> {
72 #[inline]
75 pub(crate) fn immediates_or_place(self) -> Either<ArrayVec<V, 2>, PlaceValue<V>> {
76 match self {
77 OperandValue::ZeroSized => Either::Left(ArrayVec::new()),
78 OperandValue::Immediate(a) => Either::Left(ArrayVec::from_iter([a])),
79 OperandValue::Pair(a, b) => Either::Left([a, b].into()),
80 OperandValue::Ref(p) => Either::Right(p),
81 }
82 }
83
84 #[inline]
86 pub(crate) fn from_immediates(immediates: ArrayVec<V, 2>) -> Self {
87 let mut it = immediates.into_iter();
88 let Some(a) = it.next() else {
89 return OperandValue::ZeroSized;
90 };
91 let Some(b) = it.next() else {
92 return OperandValue::Immediate(a);
93 };
94 OperandValue::Pair(a, b)
95 }
96
97 pub(crate) fn pointer_parts(self) -> (V, Option<V>) {
102 match self {
103 OperandValue::Immediate(llptr) => (llptr, None),
104 OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)),
105 _ => bug!("OperandValue cannot be a pointer: {self:?}"),
106 }
107 }
108
109 pub(crate) fn deref(self, align: Align) -> PlaceValue<V> {
117 let (llval, llextra) = self.pointer_parts();
118 PlaceValue { llval, llextra, align }
119 }
120
121 pub(crate) fn is_expected_variant_for_type<'tcx, Cx: LayoutTypeCodegenMethods<'tcx>>(
122 &self,
123 cx: &Cx,
124 ty: TyAndLayout<'tcx>,
125 ) -> bool {
126 match self {
127 OperandValue::ZeroSized => ty.is_zst(),
128 OperandValue::Immediate(_) => cx.is_backend_immediate(ty),
129 OperandValue::Pair(_, _) => cx.is_backend_scalar_pair(ty),
130 OperandValue::Ref(_) => cx.is_backend_ref(ty),
131 }
132 }
133}
134
135#[derive(Copy, Clone)]
144pub struct OperandRef<'tcx, V> {
145 pub val: OperandValue<V>,
147
148 pub layout: TyAndLayout<'tcx>,
150}
151
152impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
153 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
154 write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
155 }
156}
157
158impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
159 pub fn zero_sized(layout: TyAndLayout<'tcx>) -> OperandRef<'tcx, V> {
160 assert!(layout.is_zst());
161 OperandRef { val: OperandValue::ZeroSized, layout }
162 }
163
164 pub(crate) fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
165 bx: &mut Bx,
166 val: mir::ConstValue<'tcx>,
167 ty: Ty<'tcx>,
168 ) -> Self {
169 let layout = bx.layout_of(ty);
170
171 let val = match val {
172 ConstValue::Scalar(x) => {
173 let BackendRepr::Scalar(scalar) = layout.backend_repr else {
174 bug!("from_const: invalid ByVal layout: {:#?}", layout);
175 };
176 let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
177 OperandValue::Immediate(llval)
178 }
179 ConstValue::ZeroSized => return OperandRef::zero_sized(layout),
180 ConstValue::Slice { data, meta } => {
181 let BackendRepr::ScalarPair(a_scalar, _) = layout.backend_repr else {
182 bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
183 };
184 let a = Scalar::from_pointer(
185 Pointer::new(bx.tcx().reserve_and_set_memory_alloc(data).into(), Size::ZERO),
186 &bx.tcx(),
187 );
188 let a_llval = bx.scalar_to_backend(
189 a,
190 a_scalar,
191 bx.scalar_pair_element_backend_type(layout, 0, true),
192 );
193 let b_llval = bx.const_usize(meta);
194 OperandValue::Pair(a_llval, b_llval)
195 }
196 ConstValue::Indirect { alloc_id, offset } => {
197 let alloc = bx.tcx().global_alloc(alloc_id).unwrap_memory();
198 return Self::from_const_alloc(bx, layout, alloc, offset);
199 }
200 };
201
202 OperandRef { val, layout }
203 }
204
205 fn from_const_alloc<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
206 bx: &mut Bx,
207 layout: TyAndLayout<'tcx>,
208 alloc: rustc_middle::mir::interpret::ConstAllocation<'tcx>,
209 offset: Size,
210 ) -> Self {
211 let alloc_align = alloc.inner().align;
212 assert!(alloc_align >= layout.align.abi);
213
214 let read_scalar = |start, size, s: abi::Scalar, ty| {
215 match alloc.0.read_scalar(
216 bx,
217 alloc_range(start, size),
218 matches!(s.primitive(), abi::Primitive::Pointer(_)),
219 ) {
220 Ok(val) => bx.scalar_to_backend(val, s, ty),
221 Err(_) => bx.const_poison(ty),
222 }
223 };
224
225 match layout.backend_repr {
232 BackendRepr::Scalar(s @ abi::Scalar::Initialized { .. }) => {
233 let size = s.size(bx);
234 assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
235 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
236 OperandRef { val: OperandValue::Immediate(val), layout }
237 }
238 BackendRepr::ScalarPair(
239 a @ abi::Scalar::Initialized { .. },
240 b @ abi::Scalar::Initialized { .. },
241 ) => {
242 let (a_size, b_size) = (a.size(bx), b.size(bx));
243 let b_offset = (offset + a_size).align_to(b.align(bx).abi);
244 assert!(b_offset.bytes() > 0);
245 let a_val = read_scalar(
246 offset,
247 a_size,
248 a,
249 bx.scalar_pair_element_backend_type(layout, 0, true),
250 );
251 let b_val = read_scalar(
252 b_offset,
253 b_size,
254 b,
255 bx.scalar_pair_element_backend_type(layout, 1, true),
256 );
257 OperandRef { val: OperandValue::Pair(a_val, b_val), layout }
258 }
259 _ if layout.is_zst() => OperandRef::zero_sized(layout),
260 _ => {
261 let init = bx.const_data_from_alloc(alloc);
265 let base_addr = bx.static_addr_of(init, alloc_align, None);
266
267 let llval = bx.const_ptr_byte_offset(base_addr, offset);
268 bx.load_operand(PlaceRef::new_sized(llval, layout))
269 }
270 }
271 }
272
273 pub fn immediate(self) -> V {
276 match self.val {
277 OperandValue::Immediate(s) => s,
278 _ => bug!("not immediate: {:?}", self),
279 }
280 }
281
282 pub fn deref<Cx: CodegenMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> {
292 if self.layout.ty.is_box() {
293 bug!("dereferencing {:?} in codegen", self.layout.ty);
295 }
296
297 let projected_ty = self
298 .layout
299 .ty
300 .builtin_deref(true)
301 .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self));
302
303 let layout = cx.layout_of(projected_ty);
304 self.val.deref(layout.align.abi).with_type(layout)
305 }
306
307 pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
310 self,
311 bx: &mut Bx,
312 ) -> V {
313 if let OperandValue::Pair(a, b) = self.val {
314 let llty = bx.cx().immediate_backend_type(self.layout);
315 debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
316 let mut llpair = bx.cx().const_poison(llty);
318 llpair = bx.insert_value(llpair, a, 0);
319 llpair = bx.insert_value(llpair, b, 1);
320 llpair
321 } else {
322 self.immediate()
323 }
324 }
325
326 pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
328 bx: &mut Bx,
329 llval: V,
330 layout: TyAndLayout<'tcx>,
331 ) -> Self {
332 let val = if let BackendRepr::ScalarPair(..) = layout.backend_repr {
333 debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout);
334
335 let a_llval = bx.extract_value(llval, 0);
337 let b_llval = bx.extract_value(llval, 1);
338 OperandValue::Pair(a_llval, b_llval)
339 } else {
340 OperandValue::Immediate(llval)
341 };
342 OperandRef { val, layout }
343 }
344
345 pub(crate) fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
346 &self,
347 fx: &mut FunctionCx<'a, 'tcx, Bx>,
348 bx: &mut Bx,
349 i: usize,
350 ) -> Self {
351 let field = self.layout.field(bx.cx(), i);
352 let offset = self.layout.fields.offset(i);
353
354 if !bx.is_backend_ref(self.layout) && bx.is_backend_ref(field) {
355 if let BackendRepr::SimdVector { count, .. } = self.layout.backend_repr
356 && let BackendRepr::Memory { sized: true } = field.backend_repr
357 && count.is_power_of_two()
358 {
359 assert_eq!(field.size, self.layout.size);
360 let place = PlaceRef::alloca(bx, field);
363 self.val.store(bx, place.val.with_type(self.layout));
364 return bx.load_operand(place);
365 } else {
366 bug!("Non-ref type {self:?} cannot project to ref field type {field:?}");
368 }
369 }
370
371 let val = if field.is_zst() {
372 OperandValue::ZeroSized
373 } else if field.size == self.layout.size {
374 assert_eq!(offset.bytes(), 0);
375 fx.codegen_transmute_operand(bx, *self, field).unwrap_or_else(|| {
376 bug!(
377 "Expected `codegen_transmute_operand` to handle equal-size \
378 field {i:?} projection from {self:?} to {field:?}"
379 )
380 })
381 } else {
382 let (in_scalar, imm) = match (self.val, self.layout.backend_repr) {
383 (OperandValue::Pair(a_llval, b_llval), BackendRepr::ScalarPair(a, b)) => {
385 if offset.bytes() == 0 {
386 assert_eq!(field.size, a.size(bx.cx()));
387 (Some(a), a_llval)
388 } else {
389 assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi));
390 assert_eq!(field.size, b.size(bx.cx()));
391 (Some(b), b_llval)
392 }
393 }
394
395 _ => {
396 span_bug!(fx.mir.span, "OperandRef::extract_field({:?}): not applicable", self)
397 }
398 };
399 OperandValue::Immediate(match field.backend_repr {
400 BackendRepr::SimdVector { .. } => imm,
401 BackendRepr::Scalar(out_scalar) => {
402 let Some(in_scalar) = in_scalar else {
403 span_bug!(
404 fx.mir.span,
405 "OperandRef::extract_field({:?}): missing input scalar for output scalar",
406 self
407 )
408 };
409 if in_scalar != out_scalar {
410 let backend = bx.from_immediate(imm);
415 bx.to_immediate_scalar(backend, out_scalar)
416 } else {
417 imm
418 }
419 }
420 BackendRepr::ScalarPair(_, _) | BackendRepr::Memory { .. } => bug!(),
421 })
422 };
423
424 OperandRef { val, layout: field }
425 }
426
427 #[instrument(level = "trace", skip(fx, bx))]
429 pub fn codegen_get_discr<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
430 self,
431 fx: &mut FunctionCx<'a, 'tcx, Bx>,
432 bx: &mut Bx,
433 cast_to: Ty<'tcx>,
434 ) -> V {
435 let dl = &bx.tcx().data_layout;
436 let cast_to_layout = bx.cx().layout_of(cast_to);
437 let cast_to = bx.cx().immediate_backend_type(cast_to_layout);
438
439 if self.layout.is_uninhabited() {
443 return bx.cx().const_poison(cast_to);
444 }
445
446 let (tag_scalar, tag_encoding, tag_field) = match self.layout.variants {
447 Variants::Empty => unreachable!("we already handled uninhabited types"),
448 Variants::Single { index } => {
449 let discr_val =
450 if let Some(discr) = self.layout.ty.discriminant_for_variant(bx.tcx(), index) {
451 discr.val
452 } else {
453 assert_eq!(index, FIRST_VARIANT);
456 0
459 };
460 return bx.cx().const_uint_big(cast_to, discr_val);
461 }
462 Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => {
463 (tag, tag_encoding, tag_field)
464 }
465 };
466
467 let tag_op = match self.val {
469 OperandValue::ZeroSized => bug!(),
470 OperandValue::Immediate(_) | OperandValue::Pair(_, _) => {
471 self.extract_field(fx, bx, tag_field.as_usize())
472 }
473 OperandValue::Ref(place) => {
474 let tag = place.with_type(self.layout).project_field(bx, tag_field.as_usize());
475 bx.load_operand(tag)
476 }
477 };
478 let tag_imm = tag_op.immediate();
479
480 match *tag_encoding {
482 TagEncoding::Direct => {
483 let signed = match tag_scalar.primitive() {
484 Primitive::Int(_, signed) => !tag_scalar.is_bool() && signed,
489 _ => false,
490 };
491 bx.intcast(tag_imm, cast_to, signed)
492 }
493 TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
494 let (tag, tag_llty) = match tag_scalar.primitive() {
497 Primitive::Pointer(_) => {
499 let t = bx.type_from_integer(dl.ptr_sized_integer());
500 let tag = bx.ptrtoint(tag_imm, t);
501 (tag, t)
502 }
503 _ => (tag_imm, bx.cx().immediate_backend_type(tag_op.layout)),
504 };
505
506 debug_assert_eq!(
510 self.layout
511 .ty
512 .discriminant_for_variant(bx.tcx(), untagged_variant)
513 .unwrap()
514 .val,
515 u128::from(untagged_variant.as_u32()),
516 );
517
518 let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
519
520 let (is_niche, tagged_discr, delta) = if relative_max == 0 {
535 let niche_start = bx.cx().const_uint_big(tag_llty, niche_start);
545 let is_niche = bx.icmp(IntPredicate::IntEQ, tag, niche_start);
546 let tagged_discr =
547 bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64);
548 (is_niche, tagged_discr, 0)
549 } else {
550 let relative_discr = bx.sub(tag, bx.cx().const_uint_big(tag_llty, niche_start));
553 let cast_tag = bx.intcast(relative_discr, cast_to, false);
554 let is_niche = bx.icmp(
555 IntPredicate::IntULE,
556 relative_discr,
557 bx.cx().const_uint(tag_llty, relative_max as u64),
558 );
559
560 if niche_variants.contains(&untagged_variant)
565 && bx.cx().sess().opts.optimize != OptLevel::No
566 {
567 let impossible =
568 u64::from(untagged_variant.as_u32() - niche_variants.start().as_u32());
569 let impossible = bx.cx().const_uint(tag_llty, impossible);
570 let ne = bx.icmp(IntPredicate::IntNE, relative_discr, impossible);
571 bx.assume(ne);
572 }
573
574 (is_niche, cast_tag, niche_variants.start().as_u32() as u128)
575 };
576
577 let tagged_discr = if delta == 0 {
578 tagged_discr
579 } else {
580 bx.add(tagged_discr, bx.cx().const_uint_big(cast_to, delta))
581 };
582
583 let discr = bx.select(
584 is_niche,
585 tagged_discr,
586 bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64),
587 );
588
589 discr
595 }
596 }
597 }
598}
599
600impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
601 pub fn poison<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
608 bx: &mut Bx,
609 layout: TyAndLayout<'tcx>,
610 ) -> OperandValue<V> {
611 assert!(layout.is_sized());
612 if layout.is_zst() {
613 OperandValue::ZeroSized
614 } else if bx.cx().is_backend_immediate(layout) {
615 let ibty = bx.cx().immediate_backend_type(layout);
616 OperandValue::Immediate(bx.const_poison(ibty))
617 } else if bx.cx().is_backend_scalar_pair(layout) {
618 let ibty0 = bx.cx().scalar_pair_element_backend_type(layout, 0, true);
619 let ibty1 = bx.cx().scalar_pair_element_backend_type(layout, 1, true);
620 OperandValue::Pair(bx.const_poison(ibty0), bx.const_poison(ibty1))
621 } else {
622 let ptr = bx.cx().type_ptr();
623 OperandValue::Ref(PlaceValue::new_sized(bx.const_poison(ptr), layout.align.abi))
624 }
625 }
626
627 pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
628 self,
629 bx: &mut Bx,
630 dest: PlaceRef<'tcx, V>,
631 ) {
632 self.store_with_flags(bx, dest, MemFlags::empty());
633 }
634
635 pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
636 self,
637 bx: &mut Bx,
638 dest: PlaceRef<'tcx, V>,
639 ) {
640 self.store_with_flags(bx, dest, MemFlags::VOLATILE);
641 }
642
643 pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
644 self,
645 bx: &mut Bx,
646 dest: PlaceRef<'tcx, V>,
647 ) {
648 self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
649 }
650
651 pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
652 self,
653 bx: &mut Bx,
654 dest: PlaceRef<'tcx, V>,
655 ) {
656 self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
657 }
658
659 pub(crate) fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
660 self,
661 bx: &mut Bx,
662 dest: PlaceRef<'tcx, V>,
663 flags: MemFlags,
664 ) {
665 debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
666 match self {
667 OperandValue::ZeroSized => {
668 }
671 OperandValue::Ref(val) => {
672 assert!(dest.layout.is_sized(), "cannot directly store unsized values");
673 if val.llextra.is_some() {
674 bug!("cannot directly store unsized values");
675 }
676 bx.typed_place_copy_with_flags(dest.val, val, dest.layout, flags);
677 }
678 OperandValue::Immediate(s) => {
679 let val = bx.from_immediate(s);
680 bx.store_with_flags(val, dest.val.llval, dest.val.align, flags);
681 }
682 OperandValue::Pair(a, b) => {
683 let BackendRepr::ScalarPair(a_scalar, b_scalar) = dest.layout.backend_repr else {
684 bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout);
685 };
686 let b_offset = a_scalar.size(bx).align_to(b_scalar.align(bx).abi);
687
688 let val = bx.from_immediate(a);
689 let align = dest.val.align;
690 bx.store_with_flags(val, dest.val.llval, align, flags);
691
692 let llptr = bx.inbounds_ptradd(dest.val.llval, bx.const_usize(b_offset.bytes()));
693 let val = bx.from_immediate(b);
694 let align = dest.val.align.restrict_for_offset(b_offset);
695 bx.store_with_flags(val, llptr, align, flags);
696 }
697 }
698 }
699
700 pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
701 self,
702 bx: &mut Bx,
703 indirect_dest: PlaceRef<'tcx, V>,
704 ) {
705 debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest);
706 let unsized_ty = indirect_dest
708 .layout
709 .ty
710 .builtin_deref(true)
711 .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest));
712
713 let OperandValue::Ref(PlaceValue { llval: llptr, llextra: Some(llextra), .. }) = self
714 else {
715 bug!("store_unsized called with a sized value (or with an extern type)")
716 };
717
718 let (size, align) = size_of_val::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
722 let one = bx.const_usize(1);
723 let align_minus_1 = bx.sub(align, one);
724 let size_extra = bx.add(size, align_minus_1);
725 let min_align = Align::ONE;
726 let alloca = bx.dynamic_alloca(size_extra, min_align);
727 let address = bx.ptrtoint(alloca, bx.type_isize());
728 let neg_address = bx.neg(address);
729 let offset = bx.and(neg_address, align_minus_1);
730 let dst = bx.inbounds_ptradd(alloca, offset);
731 bx.memcpy(dst, min_align, llptr, min_align, size, MemFlags::empty());
732
733 let indirect_operand = OperandValue::Pair(dst, llextra);
735 indirect_operand.store(bx, indirect_dest);
736 }
737}
738
739impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
740 fn maybe_codegen_consume_direct(
741 &mut self,
742 bx: &mut Bx,
743 place_ref: mir::PlaceRef<'tcx>,
744 ) -> Option<OperandRef<'tcx, Bx::Value>> {
745 debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref);
746
747 match self.locals[place_ref.local] {
748 LocalRef::Operand(mut o) => {
749 for elem in place_ref.projection.iter() {
751 match elem {
752 mir::ProjectionElem::Field(f, _) => {
753 assert!(
754 !o.layout.ty.is_any_ptr(),
755 "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \
756 but tried to access field {f:?} of pointer {o:?}",
757 );
758 o = o.extract_field(self, bx, f.index());
759 }
760 mir::ProjectionElem::Index(_)
761 | mir::ProjectionElem::ConstantIndex { .. } => {
762 let elem = o.layout.field(bx.cx(), 0);
766 if elem.is_zst() {
767 o = OperandRef::zero_sized(elem);
768 } else {
769 return None;
770 }
771 }
772 _ => return None,
773 }
774 }
775
776 Some(o)
777 }
778 LocalRef::PendingOperand => {
779 bug!("use of {:?} before def", place_ref);
780 }
781 LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
782 None
785 }
786 }
787 }
788
789 pub fn codegen_consume(
790 &mut self,
791 bx: &mut Bx,
792 place_ref: mir::PlaceRef<'tcx>,
793 ) -> OperandRef<'tcx, Bx::Value> {
794 debug!("codegen_consume(place_ref={:?})", place_ref);
795
796 let ty = self.monomorphized_place_ty(place_ref);
797 let layout = bx.cx().layout_of(ty);
798
799 if layout.is_zst() {
801 return OperandRef::zero_sized(layout);
802 }
803
804 if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) {
805 return o;
806 }
807
808 let place = self.codegen_place(bx, place_ref);
811 bx.load_operand(place)
812 }
813
814 pub fn codegen_operand(
815 &mut self,
816 bx: &mut Bx,
817 operand: &mir::Operand<'tcx>,
818 ) -> OperandRef<'tcx, Bx::Value> {
819 debug!("codegen_operand(operand={:?})", operand);
820
821 match *operand {
822 mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
823 self.codegen_consume(bx, place.as_ref())
824 }
825
826 mir::Operand::Constant(ref constant) => {
827 let constant_ty = self.monomorphize(constant.ty());
828 if constant_ty.is_simd() {
831 let layout = bx.layout_of(constant_ty);
834 if let BackendRepr::SimdVector { .. } = layout.backend_repr {
835 let (llval, ty) = self.immediate_const_vector(bx, constant);
836 return OperandRef {
837 val: OperandValue::Immediate(llval),
838 layout: bx.layout_of(ty),
839 };
840 }
841 }
842 self.eval_mir_constant_to_operand(bx, constant)
843 }
844 }
845 }
846}