rustfft/
common.rs

1use num_traits::{FromPrimitive, Signed};
2use std::fmt::Debug;
3
4/// Generic floating point number, implemented for f32 and f64
5pub trait FftNum: Copy + FromPrimitive + Signed + Sync + Send + Debug + 'static {}
6
7impl<T> FftNum for T where T: Copy + FromPrimitive + Signed + Sync + Send + Debug + 'static {}
8
9// Prints an error raised by an in-place FFT algorithm's `process_inplace` method
10// Marked cold and inline never to keep all formatting code out of the many monomorphized process_inplace methods
11#[cold]
12#[inline(never)]
13pub fn fft_error_inplace(
14    expected_len: usize,
15    actual_len: usize,
16    expected_scratch: usize,
17    actual_scratch: usize,
18) {
19    assert!(
20        actual_len >= expected_len,
21        "Provided FFT buffer was too small. Expected len = {}, got len = {}",
22        expected_len,
23        actual_len
24    );
25    assert_eq!(
26        actual_len % expected_len,
27        0,
28        "Input FFT buffer must be a multiple of FFT length. Expected multiple of {}, got len = {}",
29        expected_len,
30        actual_len
31    );
32    assert!(
33        actual_scratch >= expected_scratch,
34        "Not enough scratch space was provided. Expected scratch len >= {}, got scratch len = {}",
35        expected_scratch,
36        actual_scratch
37    );
38}
39
40// Prints an error raised by an in-place FFT algorithm's `process_inplace` method
41// Marked cold and inline never to keep all formatting code out of the many monomorphized process_inplace methods
42#[cold]
43#[inline(never)]
44pub fn fft_error_outofplace(
45    expected_len: usize,
46    actual_input: usize,
47    actual_output: usize,
48    expected_scratch: usize,
49    actual_scratch: usize,
50) {
51    assert_eq!(actual_input, actual_output, "Provided FFT input buffer and output buffer must have the same length. Got input.len() = {}, output.len() = {}", actual_input, actual_output);
52    assert!(
53        actual_input >= expected_len,
54        "Provided FFT buffer was too small. Expected len = {}, got len = {}",
55        expected_len,
56        actual_input
57    );
58    assert_eq!(
59        actual_input % expected_len,
60        0,
61        "Input FFT buffer must be a multiple of FFT length. Expected multiple of {}, got len = {}",
62        expected_len,
63        actual_input
64    );
65    assert!(
66        actual_scratch >= expected_scratch,
67        "Not enough scratch space was provided. Expected scratch len >= {}, got scratch len = {}",
68        expected_scratch,
69        actual_scratch
70    );
71}
72
73macro_rules! boilerplate_fft_oop {
74    ($struct_name:ident, $len_fn:expr) => {
75        impl<T: FftNum> Fft<T> for $struct_name<T> {
76            fn process_outofplace_with_scratch(
77                &self,
78                input: &mut [Complex<T>],
79                output: &mut [Complex<T>],
80                scratch: &mut [Complex<T>],
81            ) {
82                if self.len() == 0 {
83                    return;
84                }
85
86                let required_scratch = self.get_outofplace_scratch_len();
87                if input.len() < self.len()
88                    || output.len() != input.len()
89                    || scratch.len() < required_scratch
90                {
91                    // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
92                    fft_error_outofplace(
93                        self.len(),
94                        input.len(),
95                        output.len(),
96                        required_scratch,
97                        scratch.len(),
98                    );
99                    return; // Unreachable, because fft_error_outofplace asserts, but it helps codegen to put it here
100                }
101
102                let result = array_utils::iter_chunks_zipped(
103                    input,
104                    output,
105                    self.len(),
106                    |in_chunk, out_chunk| {
107                        self.perform_fft_out_of_place(in_chunk, out_chunk, scratch)
108                    },
109                );
110
111                if result.is_err() {
112                    // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
113                    // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
114                    fft_error_outofplace(self.len(), input.len(), output.len(), 0, 0);
115                }
116            }
117            fn process_with_scratch(&self, buffer: &mut [Complex<T>], scratch: &mut [Complex<T>]) {
118                if self.len() == 0 {
119                    return;
120                }
121
122                let required_scratch = self.get_inplace_scratch_len();
123                if scratch.len() < required_scratch || buffer.len() < self.len() {
124                    // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
125                    fft_error_inplace(
126                        self.len(),
127                        buffer.len(),
128                        self.get_inplace_scratch_len(),
129                        scratch.len(),
130                    );
131                    return; // Unreachable, because fft_error_inplace asserts, but it helps codegen to put it here
132                }
133
134                let (scratch, extra_scratch) = scratch.split_at_mut(self.len());
135                let result = array_utils::iter_chunks(buffer, self.len(), |chunk| {
136                    self.perform_fft_out_of_place(chunk, scratch, extra_scratch);
137                    chunk.copy_from_slice(scratch);
138                });
139
140                if result.is_err() {
141                    // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
142                    // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
143                    fft_error_inplace(
144                        self.len(),
145                        buffer.len(),
146                        self.get_inplace_scratch_len(),
147                        scratch.len(),
148                    );
149                }
150            }
151            #[inline(always)]
152            fn get_inplace_scratch_len(&self) -> usize {
153                self.inplace_scratch_len()
154            }
155            #[inline(always)]
156            fn get_outofplace_scratch_len(&self) -> usize {
157                self.outofplace_scratch_len()
158            }
159        }
160        impl<T> Length for $struct_name<T> {
161            #[inline(always)]
162            fn len(&self) -> usize {
163                $len_fn(self)
164            }
165        }
166        impl<T> Direction for $struct_name<T> {
167            #[inline(always)]
168            fn fft_direction(&self) -> FftDirection {
169                self.direction
170            }
171        }
172    };
173}
174
175macro_rules! boilerplate_fft {
176    ($struct_name:ident, $len_fn:expr, $inplace_scratch_len_fn:expr, $out_of_place_scratch_len_fn:expr) => {
177        impl<T: FftNum> Fft<T> for $struct_name<T> {
178            fn process_outofplace_with_scratch(
179                &self,
180                input: &mut [Complex<T>],
181                output: &mut [Complex<T>],
182                scratch: &mut [Complex<T>],
183            ) {
184                if self.len() == 0 {
185                    return;
186                }
187
188                let required_scratch = self.get_outofplace_scratch_len();
189                if scratch.len() < required_scratch
190                    || input.len() < self.len()
191                    || output.len() != input.len()
192                {
193                    // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
194                    fft_error_outofplace(
195                        self.len(),
196                        input.len(),
197                        output.len(),
198                        self.get_outofplace_scratch_len(),
199                        scratch.len(),
200                    );
201                    return; // Unreachable, because fft_error_outofplace asserts, but it helps codegen to put it here
202                }
203
204                let scratch = &mut scratch[..required_scratch];
205                let result = array_utils::iter_chunks_zipped(
206                    input,
207                    output,
208                    self.len(),
209                    |in_chunk, out_chunk| {
210                        self.perform_fft_out_of_place(in_chunk, out_chunk, scratch)
211                    },
212                );
213
214                if result.is_err() {
215                    // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
216                    // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
217                    fft_error_outofplace(
218                        self.len(),
219                        input.len(),
220                        output.len(),
221                        self.get_outofplace_scratch_len(),
222                        scratch.len(),
223                    );
224                }
225            }
226            fn process_with_scratch(&self, buffer: &mut [Complex<T>], scratch: &mut [Complex<T>]) {
227                if self.len() == 0 {
228                    return;
229                }
230
231                let required_scratch = self.get_inplace_scratch_len();
232                if scratch.len() < required_scratch || buffer.len() < self.len() {
233                    // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
234                    fft_error_inplace(
235                        self.len(),
236                        buffer.len(),
237                        self.get_inplace_scratch_len(),
238                        scratch.len(),
239                    );
240                    return; // Unreachable, because fft_error_inplace asserts, but it helps codegen to put it here
241                }
242
243                let scratch = &mut scratch[..required_scratch];
244                let result = array_utils::iter_chunks(buffer, self.len(), |chunk| {
245                    self.perform_fft_inplace(chunk, scratch)
246                });
247
248                if result.is_err() {
249                    // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
250                    // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
251                    fft_error_inplace(
252                        self.len(),
253                        buffer.len(),
254                        self.get_inplace_scratch_len(),
255                        scratch.len(),
256                    );
257                }
258            }
259            #[inline(always)]
260            fn get_inplace_scratch_len(&self) -> usize {
261                $inplace_scratch_len_fn(self)
262            }
263            #[inline(always)]
264            fn get_outofplace_scratch_len(&self) -> usize {
265                $out_of_place_scratch_len_fn(self)
266            }
267        }
268        impl<T: FftNum> Length for $struct_name<T> {
269            #[inline(always)]
270            fn len(&self) -> usize {
271                $len_fn(self)
272            }
273        }
274        impl<T: FftNum> Direction for $struct_name<T> {
275            #[inline(always)]
276            fn fft_direction(&self) -> FftDirection {
277                self.direction
278            }
279        }
280    };
281}
282
283#[non_exhaustive]
284#[repr(u8)]
285#[derive(Copy, Clone, Debug, PartialEq)]
286pub(crate) enum RadixFactor {
287    Factor2,
288    Factor3,
289    Factor4,
290    Factor5,
291    Factor6,
292    Factor7,
293}
294impl RadixFactor {
295    pub const fn radix(&self) -> usize {
296        // note: if we had rustc 1.66, we could just turn these values explicit discriminators on the enum
297        match self {
298            RadixFactor::Factor2 => 2,
299            RadixFactor::Factor3 => 3,
300            RadixFactor::Factor4 => 4,
301            RadixFactor::Factor5 => 5,
302            RadixFactor::Factor6 => 6,
303            RadixFactor::Factor7 => 7,
304        }
305    }
306}
rustfft/common.rs

rustfft/
common.rs
