Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Always generate GEP i8 / ptradd for struct offsets #121665

Merged
merged 5 commits into from
Mar 4, 2024
Merged

Always generate GEP i8 / ptradd for struct offsets #121665

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
123015e
always use gep inbounds i8 (ptradd) for field offsets
erikdesjardins Feb 24, 2024
beed25b
remove struct_gep, use manual layout calculations for va_arg
erikdesjardins Feb 24, 2024
4724cd4
introduce and use ptradd/inbounds_ptradd instead of gep
erikdesjardins Feb 24, 2024
c1017d4
use non-inbounds GEP for ZSTs, add fixmes
erikdesjardins Feb 28, 2024
4016510
test merging of multiple match branches that access fields of the sam…
erikdesjardins Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
introduce and use ptradd/inbounds_ptradd instead of gep
  • Loading branch information
@erikdesjardins
erikdesjardins committed Feb 27, 2024
commit 4724cd4dc4a4776ddedf3612180acfe172c16997
6 changes: 1 addition & 5 deletions compiler/rustc_codegen_gcc/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -839,11 +839,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
let llptr = if i == 0 {
place.llval
} else {
self.inbounds_gep(
self.type_i8(),
place.llval,
&[self.const_usize(b_offset.bytes())],
)
self.inbounds_ptradd(place.llval, self.const_usize(b_offset.bytes()))
};
let llty = place.layout.scalar_pair_element_gcc_type(self, i);
let load = self.load(llty, llptr, align);
Expand Down
6 changes: 1 addition & 5 deletions compiler/rustc_codegen_llvm/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -603,11 +603,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
let llptr = if i == 0 {
place.llval
} else {
self.inbounds_gep(
self.type_i8(),
place.llval,
&[self.const_usize(b_offset.bytes())],
)
self.inbounds_ptradd(place.llval, self.const_usize(b_offset.bytes()))
};
let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
let load = self.load(llty, llptr, align);
Expand Down
38 changes: 15 additions & 23 deletions compiler/rustc_codegen_llvm/src/va_arg.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(

let aligned_size = size.align_to(slot_size).bytes() as i32;
let full_direct_size = bx.cx().const_i32(aligned_size);
let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]);
let next = bx.inbounds_ptradd(addr, full_direct_size);
bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi);

if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big {
let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32);
let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]);
let adjusted = bx.inbounds_ptradd(addr, adjusted_size);
(adjusted, addr_align)
} else {
(addr, addr_align)
Expand Down Expand Up @@ -109,14 +109,10 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
// Table 3, Mapping of C & C++ built-in data types
let ptr_offset = 8;
let i32_offset = 4;
let gr_top = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(ptr_offset)]);
let vr_top = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(2 * ptr_offset)]);
let gr_offs = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(3 * ptr_offset)]);
let vr_offs = bx.inbounds_gep(
bx.type_i8(),
va_list_addr,
&[bx.cx.const_usize(3 * ptr_offset + i32_offset)],
);
let gr_top = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(ptr_offset));
let vr_top = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * ptr_offset));
let gr_offs = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(3 * ptr_offset));
let vr_offs = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(3 * ptr_offset + i32_offset));

let layout = bx.cx.layout_of(target_ty);

Expand Down Expand Up @@ -164,11 +160,11 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
let top = bx.load(top_type, reg_top, dl.pointer_align.abi);

// reg_value = *(@top + reg_off_v);
let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]);
let mut reg_addr = bx.ptradd(top, reg_off_v);
if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size {
// On big-endian systems the value is right-aligned in its slot.
let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32);
reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]);
reg_addr = bx.ptradd(reg_addr, offset);
}
let reg_type = layout.llvm_type(bx);
let reg_value = bx.load(reg_type, reg_addr, layout.align.abi);
Expand Down Expand Up @@ -210,14 +206,10 @@ fn emit_s390x_va_arg<'ll, 'tcx>(
let i64_offset = 8;
let ptr_offset = 8;
let gpr = va_list_addr;
let fpr = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(i64_offset)]);
let overflow_arg_area =
bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(2 * i64_offset)]);
let reg_save_area = bx.inbounds_gep(
bx.type_i8(),
va_list_addr,
&[bx.cx.const_usize(2 * i64_offset + ptr_offset)],
);
let fpr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(i64_offset));
let overflow_arg_area = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * i64_offset));
let reg_save_area =
bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * i64_offset + ptr_offset));

let layout = bx.cx.layout_of(target_ty);

Expand Down Expand Up @@ -248,7 +240,7 @@ fn emit_s390x_va_arg<'ll, 'tcx>(
let reg_ptr_v = bx.load(bx.type_ptr(), reg_save_area, dl.pointer_align.abi);
let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8));
let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding));
let reg_addr = bx.gep(bx.type_i8(), reg_ptr_v, &[reg_off]);
let reg_addr = bx.ptradd(reg_ptr_v, reg_off);

// Update the register count.
let new_reg_count_v = bx.add(reg_count_v, bx.const_u64(1));
Expand All @@ -262,11 +254,11 @@ fn emit_s390x_va_arg<'ll, 'tcx>(
let arg_ptr_v =
bx.load(bx.type_ptr(), overflow_arg_area, bx.tcx().data_layout.pointer_align.abi);
let arg_off = bx.const_u64(padding);
let mem_addr = bx.gep(bx.type_i8(), arg_ptr_v, &[arg_off]);
let mem_addr = bx.ptradd(arg_ptr_v, arg_off);

// Update the argument overflow area pointer.
let arg_size = bx.cx().const_u64(padded_size);
let new_arg_ptr_v = bx.inbounds_gep(bx.type_i8(), arg_ptr_v, &[arg_size]);
let new_arg_ptr_v = bx.inbounds_ptradd(arg_ptr_v, arg_size);
bx.store(new_arg_ptr_v, overflow_arg_area, dl.pointer_align.abi);
bx.br(end);

Expand Down
5 changes: 2 additions & 3 deletions compiler/rustc_codegen_ssa/src/mir/operand.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,8 +437,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
let align = dest.align;
bx.store_with_flags(val, dest.llval, align, flags);

let llptr =
bx.inbounds_gep(bx.type_i8(), dest.llval, &[bx.const_usize(b_offset.bytes())]);
let llptr = bx.inbounds_ptradd(dest.llval, bx.const_usize(b_offset.bytes()));
let val = bx.from_immediate(b);
let align = dest.align.restrict_for_offset(b_offset);
bx.store_with_flags(val, llptr, align, flags);
Expand Down Expand Up @@ -476,7 +475,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
let address = bx.ptrtoint(alloca, bx.type_isize());
let neg_address = bx.neg(address);
let offset = bx.and(neg_address, align_minus_1);
let dst = bx.inbounds_gep(bx.type_i8(), alloca, &[offset]);
let dst = bx.inbounds_ptradd(alloca, offset);
bx.memcpy(dst, min_align, llptr, min_align, size, MemFlags::empty());

// Store the allocated region and the extra to the indirect place.
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_ssa/src/mir/place.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
let llval = if offset.bytes() == 0 {
self.llval
} else {
bx.inbounds_gep(bx.type_i8(), self.llval, &[bx.const_usize(offset.bytes())])
bx.inbounds_ptradd(self.llval, bx.const_usize(offset.bytes()))
};
PlaceRef {
llval,
Expand Down Expand Up @@ -164,7 +164,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
debug!("struct_field_ptr: DST field offset: {:?}", offset);

// Adjust pointer.
let ptr = bx.gep(bx.cx().type_i8(), self.llval, &[offset]);
let ptr = bx.ptradd(self.llval, offset);
erikdesjardins marked this conversation as resolved.
Show resolved Hide resolved

PlaceRef { llval: ptr, llextra: self.llextra, layout: field, align: effective_field_align }
}
Expand Down
6 changes: 6 additions & 0 deletions compiler/rustc_codegen_ssa/src/traits/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,12 @@ pub trait BuilderMethods<'a, 'tcx>:
ptr: Self::Value,
indices: &[Self::Value],
) -> Self::Value;
fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
self.gep(self.cx().type_i8(), ptr, &[offset])
}
fn inbounds_ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
self.inbounds_gep(self.cx().type_i8(), ptr, &[offset])
}

fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
Expand Down
Loading