OpenShot Audio Library | OpenShotAudio  0.6.0
juce_SIMDNativeOps_neon.h
1 /*
2  ==============================================================================
3 
4  This file is part of the JUCE library.
5  Copyright (c) 2022 - Raw Material Software Limited
6 
7  JUCE is an open source library subject to commercial or open-source
8  licensing.
9 
10  By using JUCE, you agree to the terms of both the JUCE 7 End-User License
11  Agreement and JUCE Privacy Policy.
12 
13  End User License Agreement: www.juce.com/juce-7-licence
14  Privacy Policy: www.juce.com/juce-privacy-policy
15 
16  Or: You may also use this code under the terms of the GPL v3 (see
17  www.gnu.org/licenses).
18 
19  JUCE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER
20  EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE
21  DISCLAIMED.
22 
23  ==============================================================================
24 */
25 
26 namespace juce::dsp
27 {
28 
29 #ifndef DOXYGEN
30 
31 JUCE_BEGIN_IGNORE_WARNINGS_GCC_LIKE ("-Wignored-attributes")
32 
33 #ifdef _MSC_VER
34  #define DECLARE_NEON_SIMD_CONST(type, name) \
35  static __declspec (align (16)) const type name [16 / sizeof (type)]
36 
37  #define DEFINE_NEON_SIMD_CONST(type, class_type, name) \
38  __declspec (align (16)) const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)]
39 
40 #else
41  #define DECLARE_NEON_SIMD_CONST(type, name) \
42  static const type name [16 / sizeof (type)] __attribute__ ((aligned (16)))
43 
44  #define DEFINE_NEON_SIMD_CONST(type, class_type, name) \
45  const type SIMDNativeOps<class_type>:: name [16 / sizeof (type)] __attribute__ ((aligned (16)))
46 
47 #endif
48 
49 template <typename type>
50 struct SIMDNativeOps;
51 
52 //==============================================================================
57 template <>
58 struct SIMDNativeOps<uint32_t>
59 {
60  //==============================================================================
61  using vSIMDType = uint32x4_t;
62  using fb = SIMDFallbackOps<uint32_t, vSIMDType>;
63 
64  //==============================================================================
65  DECLARE_NEON_SIMD_CONST (uint32_t, kAllBitsSet);
66 
67  //==============================================================================
68  static forcedinline vSIMDType expand (uint32_t s) noexcept { return vdupq_n_u32 (s); }
69  static forcedinline vSIMDType load (const uint32_t* a) noexcept { return vld1q_u32 (a); }
70  static forcedinline void store (vSIMDType value, uint32_t* a) noexcept { vst1q_u32 (a, value); }
71  static forcedinline uint32_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
72  static forcedinline vSIMDType set (vSIMDType v, size_t i, uint32_t s) noexcept { return fb::set (v, i, s); }
73  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_u32 (a, b); }
74  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_u32 (a, b); }
75  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_u32 (a, b); }
76  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_u32 (a, b); }
77  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_u32 (a, b); }
78  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_u32 (a, b); }
79  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u32 (b, a); }
80  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_u32 ((uint32_t*) kAllBitsSet)); }
81  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_u32 (a, b); }
82  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_u32 (a, b); }
83  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_u32 (a, b); }
84  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (sum (notEqual (a, b)) == 0); }
85  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
86  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_u32 (a, b); }
87  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_u32 (a, b); }
88  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_u32 (a, b, c); }
89  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
90 
91  static forcedinline uint32_t sum (vSIMDType a) noexcept
92  {
93  auto rr = vadd_u32 (vget_high_u32 (a), vget_low_u32 (a));
94  return vget_lane_u32 (vpadd_u32 (rr, rr), 0);
95  }
96 };
97 
98 //==============================================================================
103 template <>
104 struct SIMDNativeOps<int32_t>
105 {
106  //==============================================================================
107  using vSIMDType = int32x4_t;
108  using fb = SIMDFallbackOps<int32_t, vSIMDType>;
109 
110  //==============================================================================
111  DECLARE_NEON_SIMD_CONST (int32_t, kAllBitsSet);
112 
113  //==============================================================================
114  static forcedinline vSIMDType expand (int32_t s) noexcept { return vdupq_n_s32 (s); }
115  static forcedinline vSIMDType load (const int32_t* a) noexcept { return vld1q_s32 (a); }
116  static forcedinline void store (vSIMDType value, int32_t* a) noexcept { vst1q_s32 (a, value); }
117  static forcedinline int32_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
118  static forcedinline vSIMDType set (vSIMDType v, size_t i, int32_t s) noexcept { return fb::set (v, i, s); }
119  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_s32 (a, b); }
120  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_s32 (a, b); }
121  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_s32 (a, b); }
122  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_s32 (a, b); }
123  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_s32 (a, b); }
124  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_s32 (a, b); }
125  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s32 (b, a); }
126  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_s32 ((int32_t*) kAllBitsSet)); }
127  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_s32 (a, b); }
128  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_s32 (a, b); }
129  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_s32 (a, b); }
130  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (sum (notEqual (a, b)) == 0); }
131  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
132  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_s32 (a, b); }
133  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_s32 (a, b); }
134  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_s32 (a, b, c); }
135  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
136 
137  static forcedinline int32_t sum (vSIMDType a) noexcept
138  {
139  auto rr = vadd_s32 (vget_high_s32 (a), vget_low_s32 (a));
140  rr = vpadd_s32 (rr, rr);
141  return vget_lane_s32 (rr, 0);
142  }
143 };
144 
145 //==============================================================================
150 template <>
151 struct SIMDNativeOps<int8_t>
152 {
153  //==============================================================================
154  using vSIMDType = int8x16_t;
155  using fb = SIMDFallbackOps<int8_t, vSIMDType>;
156 
157  //==============================================================================
158  DECLARE_NEON_SIMD_CONST (int8_t, kAllBitsSet);
159 
160  //==============================================================================
161  static forcedinline vSIMDType expand (int8_t s) noexcept { return vdupq_n_s8 (s); }
162  static forcedinline vSIMDType load (const int8_t* a) noexcept { return vld1q_s8 (a); }
163  static forcedinline void store (vSIMDType value, int8_t* a) noexcept { vst1q_s8 (a, value); }
164  static forcedinline int8_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
165  static forcedinline vSIMDType set (vSIMDType v, size_t i, int8_t s) noexcept { return fb::set (v, i, s); }
166  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_s8 (a, b); }
167  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_s8 (a, b); }
168  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_s8 (a, b); }
169  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_s8 (a, b); }
170  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_s8 (a, b); }
171  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_s8 (a, b); }
172  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s8 (b, a); }
173  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_s8 ((int8_t*) kAllBitsSet)); }
174  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_s8 (a, b); }
175  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_s8 (a, b); }
176  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_s8 (a, b); }
177  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
178  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_s8 (a, b); }
179  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_s8 (a, b); }
180  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<int32_t>::sum ((SIMDNativeOps<int32_t>::vSIMDType) notEqual (a, b)) == 0); }
181  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_s8 (a, b, c); }
182  static forcedinline int8_t sum (vSIMDType a) noexcept { return fb::sum (a); }
183  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
184 };
185 
186 //==============================================================================
191 template <>
192 struct SIMDNativeOps<uint8_t>
193 {
194  //==============================================================================
195  using vSIMDType = uint8x16_t;
196  using fb = SIMDFallbackOps<uint8_t, vSIMDType>;
197 
198  //==============================================================================
199  DECLARE_NEON_SIMD_CONST (uint8_t, kAllBitsSet);
200 
201  //==============================================================================
202  static forcedinline vSIMDType expand (uint8_t s) noexcept { return vdupq_n_u8 (s); }
203  static forcedinline vSIMDType load (const uint8_t* a) noexcept { return vld1q_u8 (a); }
204  static forcedinline void store (vSIMDType value, uint8_t* a) noexcept { vst1q_u8 (a, value); }
205  static forcedinline uint8_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
206  static forcedinline vSIMDType set (vSIMDType v, size_t i, uint8_t s) noexcept { return fb::set (v, i, s); }
207  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_u8 (a, b); }
208  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_u8 (a, b); }
209  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_u8 (a, b); }
210  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_u8 (a, b); }
211  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_u8 (a, b); }
212  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_u8 (a, b); }
213  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u8 (b, a); }
214  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_u8 ((uint8_t*) kAllBitsSet)); }
215  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_u8 (a, b); }
216  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_u8 (a, b); }
217  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_u8 (a, b); }
218  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
219  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_u8 (a, b); }
220  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_u8 (a, b); }
221  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }
222  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_u8 (a, b, c); }
223  static forcedinline uint8_t sum (vSIMDType a) noexcept { return fb::sum (a); }
224  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
225 };
226 
227 //==============================================================================
232 template <>
233 struct SIMDNativeOps<int16_t>
234 {
235  //==============================================================================
236  using vSIMDType = int16x8_t;
237  using fb = SIMDFallbackOps<int16_t, vSIMDType>;
238 
239  //==============================================================================
240  DECLARE_NEON_SIMD_CONST (int16_t, kAllBitsSet);
241 
242  //==============================================================================
243  static forcedinline vSIMDType expand (int16_t s) noexcept { return vdupq_n_s16 (s); }
244  static forcedinline vSIMDType load (const int16_t* a) noexcept { return vld1q_s16 (a); }
245  static forcedinline void store (vSIMDType value, int16_t* a) noexcept { vst1q_s16 (a, value); }
246  static forcedinline int16_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
247  static forcedinline vSIMDType set (vSIMDType v, size_t i, int16_t s) noexcept { return fb::set (v, i, s); }
248  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_s16 (a, b); }
249  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_s16 (a, b); }
250  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_s16 (a, b); }
251  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_s16 (a, b); }
252  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_s16 (a, b); }
253  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_s16 (a, b); }
254  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s16 (b, a); }
255  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_s16 ((int16_t*) kAllBitsSet)); }
256  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_s16 (a, b); }
257  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_s16 (a, b); }
258  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_s16 (a, b); }
259  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
260  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_s16 (a, b); }
261  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_s16 (a, b); }
262  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<int32_t>::sum ((SIMDNativeOps<int32_t>::vSIMDType) notEqual (a, b)) == 0); }
263  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_s16 (a, b, c); }
264  static forcedinline int16_t sum (vSIMDType a) noexcept { return fb::sum (a); }
265  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
266 };
267 
268 
269 //==============================================================================
274 template <>
275 struct SIMDNativeOps<uint16_t>
276 {
277  //==============================================================================
278  using vSIMDType = uint16x8_t;
279  using fb = SIMDFallbackOps<uint16_t, vSIMDType>;
280 
281  //==============================================================================
282  DECLARE_NEON_SIMD_CONST (uint16_t, kAllBitsSet);
283 
284  //==============================================================================
285  static forcedinline vSIMDType expand (uint16_t s) noexcept { return vdupq_n_u16 (s); }
286  static forcedinline vSIMDType load (const uint16_t* a) noexcept { return vld1q_u16 (a); }
287  static forcedinline void store (vSIMDType value, uint16_t* a) noexcept { vst1q_u16 (a, value); }
288  static forcedinline uint16_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
289  static forcedinline vSIMDType set (vSIMDType v, size_t i, uint16_t s) noexcept { return fb::set (v, i, s); }
290  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_u16 (a, b); }
291  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_u16 (a, b); }
292  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_u16 (a, b); }
293  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_u16 (a, b); }
294  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_u16 (a, b); }
295  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_u16 (a, b); }
296  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u16 (b, a); }
297  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_u16 ((uint16_t*) kAllBitsSet)); }
298  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_u16 (a, b); }
299  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_u16 (a, b); }
300  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_u16 (a, b); }
301  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
302  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_u16 (a, b); }
303  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_u16 (a, b); }
304  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }
305  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_u16 (a, b, c); }
306  static forcedinline uint16_t sum (vSIMDType a) noexcept { return fb::sum (a); }
307  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
308 };
309 
310 //==============================================================================
315 template <>
316 struct SIMDNativeOps<int64_t>
317 {
318  //==============================================================================
319  using vSIMDType = int64x2_t;
320  using fb = SIMDFallbackOps<int64_t, vSIMDType>;
321 
322  //==============================================================================
323  DECLARE_NEON_SIMD_CONST (int64_t, kAllBitsSet);
324 
325  //==============================================================================
326  static forcedinline vSIMDType expand (int64_t s) noexcept { return vdupq_n_s64 (s); }
327  static forcedinline vSIMDType load (const int64_t* a) noexcept { return vld1q_s64 (a); }
328  static forcedinline void store (vSIMDType value, int64_t* a) noexcept { vst1q_s64 (a, value); }
329  static forcedinline int64_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
330  static forcedinline vSIMDType set (vSIMDType v, size_t i, int64_t s) noexcept { return fb::set (v, i, s); }
331  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_s64 (a, b); }
332  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_s64 (a, b); }
333  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return fb::mul (a, b); }
334  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_s64 (a, b); }
335  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_s64 (a, b); }
336  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_s64 (a, b); }
337  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_s64 (b, a); }
338  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_s64 ((int64_t*) kAllBitsSet)); }
339  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return fb::min (a, b); }
340  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return fb::max (a, b); }
341  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return fb::equal (a, b); }
342  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return fb::notEqual (a, b); }
343  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return fb::greaterThan (a, b); }
344  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return fb::greaterThanOrEqual (a, b); }
345  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<int32_t>::sum ((SIMDNativeOps<int32_t>::vSIMDType) notEqual (a, b)) == 0); }
346  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return fb::multiplyAdd (a, b, c); }
347  static forcedinline int64_t sum (vSIMDType a) noexcept { return fb::sum (a); }
348  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
349 };
350 
351 
352 //==============================================================================
357 template <>
358 struct SIMDNativeOps<uint64_t>
359 {
360  //==============================================================================
361  using vSIMDType = uint64x2_t;
362  using fb = SIMDFallbackOps<uint64_t, vSIMDType>;
363 
364  //==============================================================================
365  DECLARE_NEON_SIMD_CONST (uint64_t, kAllBitsSet);
366 
367  //==============================================================================
368  static forcedinline vSIMDType expand (uint64_t s) noexcept { return vdupq_n_u64 (s); }
369  static forcedinline vSIMDType load (const uint64_t* a) noexcept { return vld1q_u64 (a); }
370  static forcedinline void store (vSIMDType value, uint64_t* a) noexcept { vst1q_u64 (a, value); }
371  static forcedinline uint64_t get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
372  static forcedinline vSIMDType set (vSIMDType v, size_t i, uint64_t s) noexcept { return fb::set (v, i, s); }
373  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_u64 (a, b); }
374  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_u64 (a, b); }
375  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return fb::mul (a, b); }
376  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return vandq_u64 (a, b); }
377  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return vorrq_u64 (a, b); }
378  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return veorq_u64 (a, b); }
379  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return vbicq_u64 (b, a); }
380  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_u64 ((uint64_t*) kAllBitsSet)); }
381  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return fb::min (a, b); }
382  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return fb::max (a, b); }
383  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return fb::equal (a, b); }
384  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return fb::notEqual (a, b); }
385  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return fb::greaterThan (a, b); }
386  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return fb::greaterThanOrEqual (a, b); }
387  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }
388  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return fb::multiplyAdd (a, b, c); }
389  static forcedinline uint64_t sum (vSIMDType a) noexcept { return fb::sum (a); }
390  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return a; }
391 };
392 
393  //==============================================================================
398 template <>
399 struct SIMDNativeOps<float>
400 {
401  //==============================================================================
402  using vSIMDType = float32x4_t;
403  using vMaskType = uint32x4_t;
404  using fb = SIMDFallbackOps<float, vSIMDType>;
405 
406  //==============================================================================
407  DECLARE_NEON_SIMD_CONST (int32_t, kAllBitsSet);
408  DECLARE_NEON_SIMD_CONST (int32_t, kEvenHighBit);
409  DECLARE_NEON_SIMD_CONST (float, kOne);
410 
411  //==============================================================================
412  static forcedinline vSIMDType expand (float s) noexcept { return vdupq_n_f32 (s); }
413  static forcedinline vSIMDType load (const float* a) noexcept { return vld1q_f32 (a); }
414  static forcedinline float get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
415  static forcedinline vSIMDType set (vSIMDType v, size_t i, float s) noexcept { return fb::set (v, i, s); }
416  static forcedinline void store (vSIMDType value, float* a) noexcept { vst1q_f32 (a, value); }
417  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_f32 (a, b); }
418  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_f32 (a, b); }
419  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_f32 (a, b); }
420  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vandq_u32 ((vMaskType) a, (vMaskType) b); }
421  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vorrq_u32 ((vMaskType) a, (vMaskType) b); }
422  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) veorq_u32 ((vMaskType) a, (vMaskType) b); }
423  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vbicq_u32 ((vMaskType) b, (vMaskType) a); }
424  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_f32 ((float*) kAllBitsSet)); }
425  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_f32 (a, b); }
426  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_f32 (a, b); }
427  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_f32 (a, b); }
428  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
429  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_f32 (a, b); }
430  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_f32 (a, b); }
431  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }
432  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_f32 (a, b, c); }
433  static forcedinline vSIMDType dupeven (vSIMDType a) noexcept { return fb::shuffle<(0 << 0) | (0 << 2) | (2 << 4) | (2 << 6)> (a); }
434  static forcedinline vSIMDType dupodd (vSIMDType a) noexcept { return fb::shuffle<(1 << 0) | (1 << 2) | (3 << 4) | (3 << 6)> (a); }
435  static forcedinline vSIMDType swapevenodd (vSIMDType a) noexcept { return fb::shuffle<(1 << 0) | (0 << 2) | (3 << 4) | (2 << 6)> (a); }
436  static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept { return add (fb::shuffle<(2 << 0) | (3 << 2) | (0 << 4) | (1 << 6)> (a), a); }
437  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return vcvtq_f32_s32 (vcvtq_s32_f32 (a)); }
438 
439  //==============================================================================
440  static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept
441  {
442  vSIMDType rr_ir = mul (a, dupeven (b));
443  vSIMDType ii_ri = mul (swapevenodd (a), dupodd (b));
444  return add (rr_ir, bit_xor (ii_ri, vld1q_f32 ((float*) kEvenHighBit)));
445  }
446 
447  static forcedinline float sum (vSIMDType a) noexcept
448  {
449  auto rr = vadd_f32 (vget_high_f32 (a), vget_low_f32 (a));
450  return vget_lane_f32 (vpadd_f32 (rr, rr), 0);
451  }
452 };
453 
454 //==============================================================================
460 #if JUCE_64BIT
461 template <>
462 struct SIMDNativeOps<double>
463 {
464  //==============================================================================
465  using vSIMDType = float64x2_t;
466  using vMaskType = uint64x2_t;
467  using fb = SIMDFallbackOps<double, vSIMDType>;
468 
469  //==============================================================================
470  DECLARE_NEON_SIMD_CONST (int64_t, kAllBitsSet);
471  DECLARE_NEON_SIMD_CONST (double, kOne);
472 
473  //==============================================================================
474  static forcedinline vSIMDType expand (double s) noexcept { return vdupq_n_f64 (s); }
475  static forcedinline vSIMDType load (const double* a) noexcept { return vld1q_f64 (a); }
476  static forcedinline double get (vSIMDType v, size_t i) noexcept { return fb::get (v, i); }
477  static forcedinline vSIMDType set (vSIMDType v, size_t i, double s) noexcept { return fb::set (v, i, s); }
478  static forcedinline void store (vSIMDType value, double* a) noexcept { vst1q_f64 (a, value); }
479  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return vaddq_f64 (a, b); }
480  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return vsubq_f64 (a, b); }
481  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return vmulq_f64 (a, b); }
482  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vandq_u64 ((vMaskType) a, (vMaskType) b); }
483  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vorrq_u64 ((vMaskType) a, (vMaskType) b); }
484  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) veorq_u64 ((vMaskType) a, (vMaskType) b); }
485  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vbicq_u64 ((vMaskType) b, (vMaskType) a); }
486  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return bit_notand (a, vld1q_f64 ((double*) kAllBitsSet)); }
487  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return vminq_f64 (a, b); }
488  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return vmaxq_f64 (a, b); }
489  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vceqq_f64 (a, b); }
490  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return bit_not (equal (a, b)); }
491  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgtq_f64 (a, b); }
492  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return (vSIMDType) vcgeq_f64 (a, b); }
493  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return (SIMDNativeOps<uint32_t>::sum ((SIMDNativeOps<uint32_t>::vSIMDType) notEqual (a, b)) == 0); }
494  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return vmlaq_f64 (a, b, c); }
495  static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept { return fb::cmplxmul (a, b); }
496  static forcedinline double sum (vSIMDType a) noexcept { return fb::sum (a); }
497  static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept { return a; }
498  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return vcvtq_f64_s64 (vcvtq_s64_f64 (a)); }
499 };
500 #else
501 template <>
502 struct SIMDNativeOps<double>
503 {
504  //==============================================================================
505  using vSIMDType = struct { double v[2]; };
506  using fb = SIMDFallbackOps<double, vSIMDType>;
507 
508  static forcedinline vSIMDType expand (double s) noexcept { return {{s, s}}; }
509  static forcedinline vSIMDType load (const double* a) noexcept { return {{a[0], a[1]}}; }
510  static forcedinline void store (vSIMDType v, double* a) noexcept { a[0] = v.v[0]; a[1] = v.v[1]; }
511  static forcedinline double get (vSIMDType v, size_t i) noexcept { return v.v[i]; }
512  static forcedinline vSIMDType set (vSIMDType v, size_t i, double s) noexcept { v.v[i] = s; return v; }
513  static forcedinline vSIMDType add (vSIMDType a, vSIMDType b) noexcept { return {{a.v[0] + b.v[0], a.v[1] + b.v[1]}}; }
514  static forcedinline vSIMDType sub (vSIMDType a, vSIMDType b) noexcept { return {{a.v[0] - b.v[0], a.v[1] - b.v[1]}}; }
515  static forcedinline vSIMDType mul (vSIMDType a, vSIMDType b) noexcept { return {{a.v[0] * b.v[0], a.v[1] * b.v[1]}}; }
516  static forcedinline vSIMDType bit_and (vSIMDType a, vSIMDType b) noexcept { return fb::bit_and (a, b); }
517  static forcedinline vSIMDType bit_or (vSIMDType a, vSIMDType b) noexcept { return fb::bit_or (a, b); }
518  static forcedinline vSIMDType bit_xor (vSIMDType a, vSIMDType b) noexcept { return fb::bit_xor (a, b); }
519  static forcedinline vSIMDType bit_notand (vSIMDType a, vSIMDType b) noexcept { return fb::bit_notand (a, b); }
520  static forcedinline vSIMDType bit_not (vSIMDType a) noexcept { return fb::bit_not (a); }
521  static forcedinline vSIMDType min (vSIMDType a, vSIMDType b) noexcept { return fb::min (a, b); }
522  static forcedinline vSIMDType max (vSIMDType a, vSIMDType b) noexcept { return fb::max (a, b); }
523  static forcedinline vSIMDType equal (vSIMDType a, vSIMDType b) noexcept { return fb::equal (a, b); }
524  static forcedinline vSIMDType notEqual (vSIMDType a, vSIMDType b) noexcept { return fb::notEqual (a, b); }
525  static forcedinline vSIMDType greaterThan (vSIMDType a, vSIMDType b) noexcept { return fb::greaterThan (a, b); }
526  static forcedinline vSIMDType greaterThanOrEqual (vSIMDType a, vSIMDType b) noexcept { return fb::greaterThanOrEqual (a, b); }
527  static forcedinline bool allEqual (vSIMDType a, vSIMDType b) noexcept { return fb::allEqual (a, b); }
528  static forcedinline vSIMDType multiplyAdd (vSIMDType a, vSIMDType b, vSIMDType c) noexcept { return fb::multiplyAdd (a, b, c); }
529  static forcedinline vSIMDType cmplxmul (vSIMDType a, vSIMDType b) noexcept { return fb::cmplxmul (a, b); }
530  static forcedinline double sum (vSIMDType a) noexcept { return fb::sum (a); }
531  static forcedinline vSIMDType oddevensum (vSIMDType a) noexcept { return a; }
532  static forcedinline vSIMDType truncate (vSIMDType a) noexcept { return fb::truncate (a); }
533 };
534 #endif // JUCE_64BIT
535 #endif // #ifndef DOXYGEN
536 
537 JUCE_END_IGNORE_WARNINGS_GCC_LIKE
538 
539 } // namespace juce::dsp