1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
Source: http://pkgs.fedoraproject.org/cgit/rpms/llvm.git/tree/llvm-d18644-systemz-atomic-fence.patch?h=f24
===================================================================
--- a/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ a/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -260,6 +260,11 @@
.addImm(15).addReg(SystemZ::R0D);
break;
+ // Emit nothing here but a comment if we can.
+ case SystemZ::MemBarrier:
+ OutStreamer->emitRawComment("MEMBARRIER");
+ return;
+
default:
Lower.lower(MI, LoweredMI);
break;
===================================================================
--- a/lib/Target/SystemZ/SystemZISelLowering.h
+++ a/lib/Target/SystemZ/SystemZISelLowering.h
@@ -146,6 +146,9 @@
// Perform a serialization operation. (BCR 15,0 or BCR 14,0.)
SERIALIZE,
+ // Compiler barrier only; generate a no-op.
+ MEMBARRIER,
+
// Transaction begin. The first operand is the chain, the second
// the TDB pointer, and the third the immediate control field.
// Returns chain and glue.
@@ -479,6 +482,7 @@
SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
===================================================================
--- a/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ a/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -216,6 +216,8 @@
setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom);
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom);
+ setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
+
// z10 has instructions for signed but not unsigned FP conversion.
// Handle unsigned 32-bit types as signed 64-bit types.
if (!Subtarget.hasFPExtension()) {
@@ -3118,6 +3120,25 @@
return Op;
}
+SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
+ cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
+ SynchronizationScope FenceScope = static_cast<SynchronizationScope>(
+ cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
+
+ // The only fence that needs an instruction is a sequentially-consistent
+ // cross-thread fence.
+ if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) {
+ return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
+ Op.getOperand(0)), 0);
+ }
+
+ // MEMBARRIER is a compiler barrier; it codegens to a no-op.
+ return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
+}
+
// Op is an atomic load. Lower it into a normal volatile load.
SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
SelectionDAG &DAG) const {
@@ -4444,6 +4465,8 @@
case ISD::CTTZ_ZERO_UNDEF:
return DAG.getNode(ISD::CTTZ, SDLoc(Op),
Op.getValueType(), Op.getOperand(0));
+ case ISD::ATOMIC_FENCE:
+ return lowerATOMIC_FENCE(Op, DAG);
case ISD::ATOMIC_SWAP:
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
case ISD::ATOMIC_STORE:
@@ -4547,6 +4570,7 @@
OPCODE(SEARCH_STRING);
OPCODE(IPM);
OPCODE(SERIALIZE);
+ OPCODE(MEMBARRIER);
OPCODE(TBEGIN);
OPCODE(TBEGIN_NOFLOAT);
OPCODE(TEND);
@@ -5307,6 +5331,7 @@
MachineBasicBlock *
SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI,
MachineBasicBlock *MBB) const {
+
MachineFunction &MF = *MBB->getParent();
const SystemZInstrInfo *TII =
static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
===================================================================
--- a/lib/Target/SystemZ/SystemZInstrInfo.td
+++ a/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -1231,6 +1231,10 @@
let hasSideEffects = 1 in
def Serialize : Alias<2, (outs), (ins), [(z_serialize)]>;
+// A pseudo instruction that serves as a compiler barrier.
+let hasSideEffects = 1 in
+def MemBarrier : Pseudo<(outs), (ins), [(z_membarrier)]>;
+
let Predicates = [FeatureInterlockedAccess1], Defs = [CC] in {
def LAA : LoadAndOpRSY<"laa", 0xEBF8, atomic_load_add_32, GR32>;
def LAAG : LoadAndOpRSY<"laag", 0xEBE8, atomic_load_add_64, GR64>;
===================================================================
--- a/lib/Target/SystemZ/SystemZOperators.td
+++ a/lib/Target/SystemZ/SystemZOperators.td
@@ -188,6 +188,8 @@
def z_serialize : SDNode<"SystemZISD::SERIALIZE", SDTNone,
[SDNPHasChain, SDNPMayStore]>;
+def z_membarrier : SDNode<"SystemZISD::MEMBARRIER", SDTNone,
+ [SDNPHasChain, SDNPSideEffect]>;
// Defined because the index is an i32 rather than a pointer.
def z_vector_insert : SDNode<"ISD::INSERT_VECTOR_ELT",
===================================================================
--- a/test/CodeGen/SystemZ/atomic-fence-01.ll
+++ a/test/CodeGen/SystemZ/atomic-fence-01.ll
@@ -0,0 +1,16 @@
+; Test (fast) serialization.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s --check-prefix=Z10
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z196 | FileCheck %s --check-prefix=Z196
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=zEC12 | FileCheck %s --check-prefix=ZEC12
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s --check-prefix=Z13
+
+define void @test() {
+; Z10: bcr 15, %r0
+; Z196: bcr 14, %r0
+; ZEC12: bcr 14, %r0
+; Z13: bcr 14, %r0
+ fence seq_cst
+ ret void
+}
+
===================================================================
--- a/test/CodeGen/SystemZ/atomic-fence-02.ll
+++ a/test/CodeGen/SystemZ/atomic-fence-02.ll
@@ -0,0 +1,13 @@
+; Serialization is emitted only for fence seq_cst.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @test() {
+; CHECK: #MEMBARRIER
+ fence acquire
+; CHECK: #MEMBARRIER
+ fence release
+; CHECK: #MEMBARRIER
+ fence acq_rel
+ ret void
+}
|
