Skip to main content

rapx/vm/mir_rules/call/effect/
simd.rs

1use super::*;
2
3impl<'z3, 'tcx> VmExecutor<'z3, 'tcx> {
4    pub(super) fn apply_simd_effect(
5        &mut self,
6        effect: &CallEffect,
7        args: &[VmValue<'z3, 'tcx>],
8        caller_arg_locals: &[Option<Local>],
9        dest: Local,
10    ) {
11        match effect {
12            CallEffect::SimdCompare {
13                lhs_arg,
14                rhs_arg,
15                op,
16            } => {
17                let dest_ty = self.body().local_decls[dest].ty;
18                let Some((elem_ty, len)) = self.simd_lane_type(dest_ty) else {
19                    self.set_opaque_intrinsic_result(dest);
20                    return;
21                };
22                let Some((lhs_local, rhs_local)) = caller_arg_locals
23                    .get(*lhs_arg)
24                    .copied()
25                    .flatten()
26                    .zip(caller_arg_locals.get(*rhs_arg).copied().flatten())
27                else {
28                    self.set_opaque_intrinsic_result(dest);
29                    return;
30                };
31                let Some((lhs, rhs)) = args.get(*lhs_arg).zip(args.get(*rhs_arg)) else {
32                    self.set_opaque_intrinsic_result(dest);
33                    return;
34                };
35                let Some((input_ty, input_len)) = self.simd_lane_type(lhs.ty) else {
36                    self.set_opaque_intrinsic_result(dest);
37                    return;
38                };
39                if input_len != len
40                    || rhs.ty != lhs.ty
41                    || !matches!(input_ty.kind(), TyKind::Int(_) | TyKind::Uint(_))
42                {
43                    self.set_opaque_intrinsic_result(dest);
44                    return;
45                }
46                let lanes: Option<Vec<_>> = (0..len)
47                    .map(|index| {
48                        let left = self.field_value(lhs_local, &[0, index])?.z3_term.clone();
49                        let right = self.field_value(rhs_local, &[0, index])?.z3_term.clone();
50                        let condition = match op {
51                            SimdCmpOp::Eq => left._eq(&right),
52                            SimdCmpOp::Ne => left._eq(&right).not(),
53                            SimdCmpOp::Lt => left.lt(&right),
54                            SimdCmpOp::Le => left.le(&right),
55                            SimdCmpOp::Gt => left.gt(&right),
56                            SimdCmpOp::Ge => left.ge(&right),
57                        };
58                        Some(condition)
59                    })
60                    .collect();
61                let Some(lanes) = lanes else {
62                    self.set_opaque_intrinsic_result(dest);
63                    return;
64                };
65                let true_value = if matches!(elem_ty.kind(), TyKind::Int(_)) {
66                    Int::from_i64(self.z3_ctx, -1)
67                } else if let Some((_, max, _, _)) = self.integer_limits(elem_ty) {
68                    max
69                } else {
70                    self.set_opaque_intrinsic_result(dest);
71                    return;
72                };
73                let zero = Int::from_u64(self.z3_ctx, 0);
74                for (index, condition) in lanes.into_iter().enumerate() {
75                    let mut value = VmValue::new(condition.ite(&true_value, &zero), elem_ty);
76                    value.facts.init = true;
77                    self.set_field_value(dest, vec![0, index], value);
78                }
79            }
80            CallEffect::SimdArithmetic {
81                lhs_arg,
82                rhs_arg,
83                op,
84            } => {
85                let dest_ty = self.body().local_decls[dest].ty;
86                let Some((elem_ty, len)) = self.simd_lane_type(dest_ty) else {
87                    self.set_opaque_intrinsic_result(dest);
88                    return;
89                };
90                let Some((min, max, modulus, signed)) = self.integer_limits(elem_ty) else {
91                    self.set_opaque_simd_result(dest);
92                    return;
93                };
94                let Some((lhs_local, rhs_local)) = caller_arg_locals
95                    .get(*lhs_arg)
96                    .copied()
97                    .flatten()
98                    .zip(caller_arg_locals.get(*rhs_arg).copied().flatten())
99                else {
100                    self.set_opaque_simd_result(dest);
101                    return;
102                };
103                if args.get(*lhs_arg).is_none_or(|value| value.ty != dest_ty)
104                    || args.get(*rhs_arg).is_none_or(|value| value.ty != dest_ty)
105                {
106                    self.set_opaque_simd_result(dest);
107                    return;
108                }
109                let lanes: Option<Vec<_>> = (0..len)
110                    .map(|index| {
111                        let left = self.field_value(lhs_local, &[0, index])?.z3_term.clone();
112                        let right = self.field_value(rhs_local, &[0, index])?.z3_term.clone();
113                        let raw = match op {
114                            SimdArithOp::Add | SimdArithOp::SaturatingAdd => {
115                                Int::add(self.z3_ctx, &[&left, &right])
116                            }
117                            SimdArithOp::Sub | SimdArithOp::SaturatingSub => {
118                                Int::sub(self.z3_ctx, &[&left, &right])
119                            }
120                            SimdArithOp::Mul => Int::mul(self.z3_ctx, &[&left, &right]),
121                        };
122                        let term = match op {
123                            SimdArithOp::SaturatingAdd | SimdArithOp::SaturatingSub => {
124                                raw.lt(&min).ite(&min, &raw.gt(&max).ite(&max, &raw))
125                            }
126                            _ => {
127                                let wrapped = raw.modulo(&modulus);
128                                if signed {
129                                    wrapped.gt(&max).ite(
130                                        &Int::sub(self.z3_ctx, &[&wrapped, &modulus]),
131                                        &wrapped,
132                                    )
133                                } else {
134                                    wrapped
135                                }
136                            }
137                        };
138                        Some(term)
139                    })
140                    .collect();
141                let Some(lanes) = lanes else {
142                    self.set_opaque_simd_result(dest);
143                    return;
144                };
145                for (index, term) in lanes.into_iter().enumerate() {
146                    let mut value = VmValue::new(term, elem_ty);
147                    value.facts.init = true;
148                    self.set_field_value(dest, vec![0, index], value);
149                }
150            }
151            CallEffect::SimdOpaque => self.set_opaque_simd_result(dest),
152            CallEffect::SimdSplat { arg } => {
153                let dest_ty = self.body().local_decls[dest].ty;
154                let Some((elem_ty, len)) = self.simd_lane_type(dest_ty) else {
155                    self.set_opaque_intrinsic_result(dest);
156                    return;
157                };
158                let Some(input) = args.get(*arg) else {
159                    self.set_opaque_simd_result(dest);
160                    return;
161                };
162                for index in 0..len {
163                    let mut value = input.clone();
164                    value.ty = elem_ty;
165                    self.set_field_value(dest, vec![0, index], value);
166                }
167            }
168            CallEffect::SimdExtract {
169                vector_arg,
170                index_arg,
171            } => {
172                let Some((local, index)) =
173                    caller_arg_locals.get(*vector_arg).copied().flatten().zip(
174                        args.get(*index_arg)
175                            .and_then(|value| value.z3_term.simplify().as_u64()),
176                    )
177                else {
178                    self.set_opaque_intrinsic_result(dest);
179                    return;
180                };
181                let Some((_, len)) = args
182                    .get(*vector_arg)
183                    .and_then(|value| self.simd_lane_type(value.ty))
184                else {
185                    self.set_opaque_intrinsic_result(dest);
186                    return;
187                };
188                if index >= len as u64 {
189                    self.set_opaque_intrinsic_result(dest);
190                    return;
191                }
192                if let Some(mut value) = self.field_value(local, &[0, index as usize]).cloned() {
193                    value.ty = self.body().local_decls[dest].ty;
194                    self.set_local(dest, value);
195                } else {
196                    self.set_opaque_intrinsic_result(dest);
197                }
198            }
199            CallEffect::SimdInsert {
200                vector_arg,
201                index_arg,
202                value_arg,
203            } => {
204                let dest_ty = self.body().local_decls[dest].ty;
205                let Some((elem_ty, len)) = self.simd_lane_type(dest_ty) else {
206                    self.set_opaque_intrinsic_result(dest);
207                    return;
208                };
209                let Some((local, index, replacement)) = caller_arg_locals
210                    .get(*vector_arg)
211                    .copied()
212                    .flatten()
213                    .zip(
214                        args.get(*index_arg)
215                            .and_then(|value| value.z3_term.simplify().as_u64()),
216                    )
217                    .zip(args.get(*value_arg))
218                    .map(|((local, index), value)| (local, index, value.clone()))
219                else {
220                    self.set_opaque_simd_result(dest);
221                    return;
222                };
223                if index >= len as u64 {
224                    self.set_opaque_simd_result(dest);
225                    return;
226                }
227                let lanes: Option<Vec<_>> = (0..len)
228                    .map(|lane| {
229                        if lane == index as usize {
230                            let mut value = replacement.clone();
231                            value.ty = elem_ty;
232                            Some(value)
233                        } else {
234                            self.field_value(local, &[0, lane]).cloned()
235                        }
236                    })
237                    .collect();
238                let Some(lanes) = lanes else {
239                    self.set_opaque_simd_result(dest);
240                    return;
241                };
242                for (lane, value) in lanes.into_iter().enumerate() {
243                    self.set_field_value(dest, vec![0, lane], value);
244                }
245            }
246            CallEffect::SimdHavocMemory { mask_arg, .. } => {
247                let mask_is_zero = caller_arg_locals
248                    .get(*mask_arg)
249                    .copied()
250                    .flatten()
251                    .zip(
252                        args.get(*mask_arg)
253                            .and_then(|value| self.simd_lane_type(value.ty)),
254                    )
255                    .is_some_and(|(local, (_, len))| {
256                        (0..len).all(|lane| {
257                            self.field_value(local, &[0, lane])
258                                .and_then(|value| value.z3_term.simplify().as_u64())
259                                == Some(0)
260                        })
261                    });
262                if !mask_is_zero {
263                    for index in 0..self.path_state.mem_units.len() {
264                        self.forget_memory_values(AllocId(index));
265                    }
266                }
267            }
268            CallEffect::SimdMaskedFallback {
269                mask_arg,
270                fallback_arg,
271            } => {
272                let mask_is_zero = caller_arg_locals
273                    .get(*mask_arg)
274                    .copied()
275                    .flatten()
276                    .zip(
277                        args.get(*mask_arg)
278                            .and_then(|value| self.simd_lane_type(value.ty)),
279                    )
280                    .is_some_and(|(local, (_, len))| {
281                        (0..len).all(|lane| {
282                            self.field_value(local, &[0, lane])
283                                .and_then(|value| value.z3_term.simplify().as_u64())
284                                == Some(0)
285                        })
286                    });
287                let copied = mask_is_zero
288                    && caller_arg_locals
289                        .get(*fallback_arg)
290                        .copied()
291                        .flatten()
292                        .is_some_and(|local| self.copy_simd_result_from_local(dest, local));
293                if !copied {
294                    self.set_opaque_simd_result(dest);
295                }
296            }
297            CallEffect::SimdBitmask { arg } => {
298                let Some(local) = caller_arg_locals.get(*arg).copied().flatten() else {
299                    self.set_opaque_intrinsic_result(dest);
300                    return;
301                };
302                let Some((_, len)) = args
303                    .get(*arg)
304                    .and_then(|value| self.simd_lane_type(value.ty))
305                else {
306                    self.set_opaque_intrinsic_result(dest);
307                    return;
308                };
309                if len == 0
310                    || len > 64
311                    || !matches!(self.body().local_decls[dest].ty.kind(), TyKind::Uint(_))
312                {
313                    self.set_opaque_intrinsic_result(dest);
314                    return;
315                }
316                let lanes: Option<Vec<_>> = (0..len)
317                    .map(|index| {
318                        self.field_value(local, &[0, index])
319                            .map(|value| value.z3_term.clone())
320                    })
321                    .collect();
322                let Some(lanes) = lanes else {
323                    self.set_opaque_intrinsic_result(dest);
324                    return;
325                };
326                let zero = Int::from_u64(self.z3_ctx, 0);
327                let mut mask = zero.clone();
328                for (index, lane) in lanes.into_iter().enumerate() {
329                    let bit = if self.tcx.data_layout.endian == rustc_abi::Endian::Little {
330                        index
331                    } else {
332                        len - 1 - index
333                    };
334                    let weight = Int::from_u64(self.z3_ctx, 1_u64 << bit);
335                    mask = Int::add(
336                        self.z3_ctx,
337                        &[&mask, &lane._eq(&zero).not().ite(&weight, &zero)],
338                    );
339                }
340                let dest_ty = self.body().local_decls[dest].ty;
341                self.set_local(dest, VmValue::new(mask, dest_ty));
342            }
343            _ => unreachable!("non-SIMD call effect"),
344        }
345    }
346}