Skip to content

Commit

Permalink
[CHERI] Do not expand memory intrinsics in pre-isel-intrinsic-lowering
Browse files Browse the repository at this point in the history 
Since 3c84819 building with -fno-builtin will expand memory
intrinsics in IR instead of going to the default SelectionDAG IR.
This lowering does not consider capabilities so will result in IR that
does not preserve tag bits. Disable the pass if the datalayout contains
capability types to work around this problem.

Filed #753 as a tracking
bug to enable it with fixes in the future.
  • Loading branch information
@arichardson
arichardson committed Dec 11, 2024
1 parent 6c46ba2 commit 192a0e0
Show file tree
Hide file tree
Showing 2 changed files with 194 additions and 0 deletions.
8 changes: 8 additions & 0 deletions llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,10 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
auto *Memcpy = cast<MemCpyInst>(Inst);
Function *ParentFunc = Memcpy->getFunction();
const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
// TODO: we can't expand memcpy for CHERI targets yet.
// See https://github.com/CTSRD-CHERI/llvm-project/issues/753
if (Memcpy->getModule()->getDataLayout().hasCheriCapabilities())
break;
if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
if (UseMemIntrinsicLibFunc &&
LookupLibInfo(*ParentFunc).has(LibFunc_memcpy))
Expand All @@ -217,6 +221,10 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
auto *Memmove = cast<MemMoveInst>(Inst);
Function *ParentFunc = Memmove->getFunction();
const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
// TODO: we can't expand memmove for CHERI targets yet.
// See https://github.com/CTSRD-CHERI/llvm-project/issues/753
if (Memmove->getModule()->getDataLayout().hasCheriCapabilities())
break;
if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
if (UseMemIntrinsicLibFunc &&
LookupLibInfo(*ParentFunc).has(LibFunc_memmove))
Expand Down
186 changes: 186 additions & 0 deletions llvm/test/CodeGen/CHERI-Generic/lower-mem-intrinsics.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
;; Check that we don't expand for CHERI targets (yet): https://github.com/CTSRD-CHERI/llvm-project/issues/753
; RUN: opt -S -mtriple=riscv64 -pre-isel-intrinsic-lowering -mem-intrinsic-expand-size=0 --print-before-all < %s | FileCheck %s --check-prefixes=CHECK,RISCV
; RUN: opt -S -mtriple=riscv64 -data-layout="pf200:128:128:128:64" -mem-intrinsic-expand-size=0 -pre-isel-intrinsic-lowering < %s | FileCheck %s --check-prefixes=CHECK,HYBRID

declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1)
declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1)
declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1)

define void @call_memset(ptr align 16 %dst) #0 {
; CHECK-LABEL: @call_memset(
; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SPLIT:%.*]], label [[LOADSTORELOOP:%.*]]
; CHECK: loadstoreloop:
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[LOADSTORELOOP]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]]
; CHECK-NEXT: store i8 0, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2]] = add i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 40
; CHECK-NEXT: br i1 [[TMP3]], label [[LOADSTORELOOP]], label [[SPLIT]]
; CHECK: split:
; CHECK-NEXT: ret void
;
entry:
call void @llvm.memset.p0.i64(ptr align 16 %dst, i8 0, i64 40, i1 false)
ret void
}

define void @call_memset_variable(ptr align 16 %dst, i64 %len) #0 {
; CHECK-LABEL: @call_memset_variable(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 0, [[LEN:%.*]]
; CHECK-NEXT: br i1 [[TMP0]], label [[SPLIT:%.*]], label [[LOADSTORELOOP:%.*]]
; CHECK: loadstoreloop:
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP3:%.*]], [[LOADSTORELOOP]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP1]]
; CHECK-NEXT: store i8 0, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP3]] = add i64 [[TMP1]], 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], [[LEN]]
; CHECK-NEXT: br i1 [[TMP4]], label [[LOADSTORELOOP]], label [[SPLIT]]
; CHECK: split:
; CHECK-NEXT: ret void
;
entry:
call void @llvm.memset.p0.i64(ptr align 16 %dst, i8 0, i64 %len, i1 false)
ret void
}

define void @call_memcpy(ptr align 16 %dst, ptr align 16 %src) #0 {
; RISCV-LABEL: @call_memcpy(
; RISCV-NEXT: entry:
; RISCV-NEXT: br label [[LOAD_STORE_LOOP:%.*]]
; RISCV: load-store-loop:
; RISCV-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP3:%.*]], [[LOAD_STORE_LOOP]] ]
; RISCV-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[LOOP_INDEX]]
; RISCV-NEXT: [[TMP1:%.*]] = load i8, ptr [[TMP0]], align 1
; RISCV-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[LOOP_INDEX]]
; RISCV-NEXT: store i8 [[TMP1]], ptr [[TMP2]], align 1
; RISCV-NEXT: [[TMP3]] = add i64 [[LOOP_INDEX]], 1
; RISCV-NEXT: [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 40
; RISCV-NEXT: br i1 [[TMP4]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
; RISCV: memcpy-split:
; RISCV-NEXT: ret void
;
; HYBRID-LABEL: @call_memcpy(
; HYBRID-NEXT: entry:
; HYBRID-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[DST:%.*]], ptr align 16 [[SRC:%.*]], i64 40, i1 false)
; HYBRID-NEXT: ret void
;
entry:
call void @llvm.memcpy.p0.p0.i64(ptr align 16 %dst, ptr align 16 %src, i64 40, i1 false)
ret void
}

define void @call_memcpy_variable(ptr align 16 %dst, ptr align 16 %src, i64 %len) #0 {
; RISCV-LABEL: @call_memcpy_variable(
; RISCV-NEXT: entry:
; RISCV-NEXT: [[TMP0:%.*]] = icmp ne i64 [[LEN:%.*]], 0
; RISCV-NEXT: br i1 [[TMP0]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
; RISCV: loop-memcpy-expansion:
; RISCV-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP4:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
; RISCV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[LOOP_INDEX]]
; RISCV-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP1]], align 1
; RISCV-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[LOOP_INDEX]]
; RISCV-NEXT: store i8 [[TMP2]], ptr [[TMP3]], align 1
; RISCV-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 1
; RISCV-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], [[LEN]]
; RISCV-NEXT: br i1 [[TMP5]], label [[LOOP_MEMCPY_EXPANSION]], label [[POST_LOOP_MEMCPY_EXPANSION]]
; RISCV: post-loop-memcpy-expansion:
; RISCV-NEXT: ret void
;
; HYBRID-LABEL: @call_memcpy_variable(
; HYBRID-NEXT: entry:
; HYBRID-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[DST:%.*]], ptr align 16 [[SRC:%.*]], i64 [[LEN:%.*]], i1 false)
; HYBRID-NEXT: ret void
;
entry:
call void @llvm.memcpy.p0.p0.i64(ptr align 16 %dst, ptr align 16 %src, i64 %len, i1 false)
ret void
}

define void @call_memmove(ptr align 16 %dst, ptr align 16 %src) #0 {
; RISCV-LABEL: @call_memmove(
; RISCV-NEXT: entry:
; RISCV-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[DST:%.*]]
; RISCV-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 40, 0
; RISCV-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
; RISCV: copy_backwards:
; RISCV-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
; RISCV: copy_backwards_loop:
; RISCV-NEXT: [[TMP0:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 40, [[COPY_BACKWARDS]] ]
; RISCV-NEXT: [[INDEX_PTR]] = sub i64 [[TMP0]], 1
; RISCV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR]]
; RISCV-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP1]], align 1
; RISCV-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR]]
; RISCV-NEXT: store i8 [[ELEMENT]], ptr [[TMP2]], align 1
; RISCV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
; RISCV-NEXT: br i1 [[TMP3]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
; RISCV: copy_forward:
; RISCV-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
; RISCV: copy_forward_loop:
; RISCV-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
; RISCV-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR1]]
; RISCV-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP4]], align 1
; RISCV-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR1]]
; RISCV-NEXT: store i8 [[ELEMENT2]], ptr [[TMP5]], align 1
; RISCV-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
; RISCV-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 40
; RISCV-NEXT: br i1 [[TMP6]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
; RISCV: memmove_done:
; RISCV-NEXT: ret void
;
; HYBRID-LABEL: @call_memmove(
; HYBRID-NEXT: entry:
; HYBRID-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 16 [[DST:%.*]], ptr align 16 [[SRC:%.*]], i64 40, i1 false)
; HYBRID-NEXT: ret void
;
entry:
call void @llvm.memmove.p0.p0.i64(ptr align 16 %dst, ptr align 16 %src, i64 40, i1 false)
ret void
}

define void @call_memmove_variable(ptr align 16 %dst, ptr align 16 %src, i64 %len) #0 {
; RISCV-LABEL: @call_memmove_variable(
; RISCV-NEXT: entry:
; RISCV-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[DST:%.*]]
; RISCV-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 [[LEN:%.*]], 0
; RISCV-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
; RISCV: copy_backwards:
; RISCV-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
; RISCV: copy_backwards_loop:
; RISCV-NEXT: [[TMP0:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ [[LEN]], [[COPY_BACKWARDS]] ]
; RISCV-NEXT: [[INDEX_PTR]] = sub i64 [[TMP0]], 1
; RISCV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR]]
; RISCV-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP1]], align 1
; RISCV-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR]]
; RISCV-NEXT: store i8 [[ELEMENT]], ptr [[TMP2]], align 1
; RISCV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
; RISCV-NEXT: br i1 [[TMP3]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
; RISCV: copy_forward:
; RISCV-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
; RISCV: copy_forward_loop:
; RISCV-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
; RISCV-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR1]]
; RISCV-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP4]], align 1
; RISCV-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR1]]
; RISCV-NEXT: store i8 [[ELEMENT2]], ptr [[TMP5]], align 1
; RISCV-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
; RISCV-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], [[LEN]]
; RISCV-NEXT: br i1 [[TMP6]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
; RISCV: memmove_done:
; RISCV-NEXT: ret void
;
; HYBRID-LABEL: @call_memmove_variable(
; HYBRID-NEXT: entry:
; HYBRID-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 16 [[DST:%.*]], ptr align 16 [[SRC:%.*]], i64 [[LEN:%.*]], i1 false)
; HYBRID-NEXT: ret void
;
entry:
call void @llvm.memmove.p0.p0.i64(ptr align 16 %dst, ptr align 16 %src, i64 %len, i1 false)
ret void
}

;; We need the "no-builtins" attribute here otherwise the IR lowering pass will
;; be skipped since it assumes the libfunc can be called.
attributes #0 = { nounwind "no-builtins" }

0 comments on commit 192a0e0

Please sign in to comment.