polars_core/series/
from.rs

1use arrow::datatypes::Metadata;
2#[cfg(feature = "dtype-categorical")]
3use arrow::legacy::kernels::concatenate::concatenate_owned_unchecked;
4#[cfg(any(
5    feature = "dtype-date",
6    feature = "dtype-datetime",
7    feature = "dtype-time",
8    feature = "dtype-duration"
9))]
10use arrow::temporal_conversions::*;
11use polars_compute::cast::cast_unchecked as cast;
12use polars_error::feature_gated;
13use polars_utils::itertools::Itertools;
14
15use crate::chunked_array::cast::{cast_chunks, CastOptions};
16#[cfg(feature = "object")]
17use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
18#[cfg(feature = "timezones")]
19use crate::chunked_array::temporal::parse_fixed_offset;
20#[cfg(feature = "timezones")]
21use crate::chunked_array::temporal::validate_time_zone;
22use crate::prelude::*;
23
24impl Series {
25    /// Takes chunks and a polars datatype and constructs the Series
26    /// This is faster than creating from chunks and an arrow datatype because there is no
27    /// casting involved
28    ///
29    /// # Safety
30    ///
31    /// The caller must ensure that the given `dtype`'s physical type matches all the `ArrayRef` dtypes.
32    pub unsafe fn from_chunks_and_dtype_unchecked(
33        name: PlSmallStr,
34        chunks: Vec<ArrayRef>,
35        dtype: &DataType,
36    ) -> Self {
37        use DataType::*;
38        match dtype {
39            #[cfg(feature = "dtype-i8")]
40            Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
41            #[cfg(feature = "dtype-i16")]
42            Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
43            Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
44            Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
45            #[cfg(feature = "dtype-u8")]
46            UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
47            #[cfg(feature = "dtype-u16")]
48            UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
49            UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
50            UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
51            #[cfg(feature = "dtype-i128")]
52            Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
53            #[cfg(feature = "dtype-date")]
54            Date => Int32Chunked::from_chunks(name, chunks)
55                .into_date()
56                .into_series(),
57            #[cfg(feature = "dtype-time")]
58            Time => Int64Chunked::from_chunks(name, chunks)
59                .into_time()
60                .into_series(),
61            #[cfg(feature = "dtype-duration")]
62            Duration(tu) => Int64Chunked::from_chunks(name, chunks)
63                .into_duration(*tu)
64                .into_series(),
65            #[cfg(feature = "dtype-datetime")]
66            Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
67                .into_datetime(*tu, tz.clone())
68                .into_series(),
69            #[cfg(feature = "dtype-decimal")]
70            Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
71                .into_decimal_unchecked(
72                    *precision,
73                    scale.unwrap_or_else(|| unreachable!("scale should be set")),
74                )
75                .into_series(),
76            #[cfg(feature = "dtype-array")]
77            Array(_, _) => {
78                ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
79                    .into_series()
80            },
81            List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
82                .into_series(),
83            String => StringChunked::from_chunks(name, chunks).into_series(),
84            Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
85            #[cfg(feature = "dtype-categorical")]
86            dt @ (Categorical(rev_map, ordering) | Enum(rev_map, ordering)) => {
87                let cats = UInt32Chunked::from_chunks(name, chunks);
88                let rev_map = rev_map.clone().unwrap_or_else(|| {
89                    assert!(cats.is_empty());
90                    Arc::new(RevMapping::default())
91                });
92                let mut ca = CategoricalChunked::from_cats_and_rev_map_unchecked(
93                    cats,
94                    rev_map,
95                    matches!(dt, Enum(_, _)),
96                    *ordering,
97                );
98                ca.set_fast_unique(false);
99                ca.into_series()
100            },
101            Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
102            Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
103            Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
104            BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
105            #[cfg(feature = "dtype-struct")]
106            Struct(_) => {
107                let mut ca =
108                    StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
109                ca.propagate_nulls();
110                ca.into_series()
111            },
112            #[cfg(feature = "object")]
113            Object(_, _) => {
114                assert_eq!(chunks.len(), 1);
115                let arr = chunks[0]
116                    .as_any()
117                    .downcast_ref::<FixedSizeBinaryArray>()
118                    .unwrap();
119                // SAFETY:
120                // this is highly unsafe. it will dereference a raw ptr on the heap
121                // make sure the ptr is allocated and from this pid
122                // (the pid is checked before dereference)
123                {
124                    let pe = PolarsExtension::new(arr.clone());
125                    let s = pe.get_series(&name);
126                    pe.take_and_forget();
127                    s
128                }
129            },
130            Null => new_null(name, &chunks),
131            Unknown(_) => {
132                panic!("dtype is unknown; consider supplying data-types for all operations")
133            },
134            #[allow(unreachable_patterns)]
135            _ => unreachable!(),
136        }
137    }
138
139    /// # Safety
140    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
141    pub unsafe fn _try_from_arrow_unchecked(
142        name: PlSmallStr,
143        chunks: Vec<ArrayRef>,
144        dtype: &ArrowDataType,
145    ) -> PolarsResult<Self> {
146        Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
147    }
148
149    /// Create a new Series without checking if the inner dtype of the chunks is correct
150    ///
151    /// # Safety
152    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
153    pub unsafe fn _try_from_arrow_unchecked_with_md(
154        name: PlSmallStr,
155        chunks: Vec<ArrayRef>,
156        dtype: &ArrowDataType,
157        md: Option<&Metadata>,
158    ) -> PolarsResult<Self> {
159        match dtype {
160            ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
161            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
162                let chunks =
163                    cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
164                Ok(StringChunked::from_chunks(name, chunks).into_series())
165            },
166            ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
167            ArrowDataType::LargeBinary => {
168                if let Some(md) = md {
169                    if md.maintain_type() {
170                        return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
171                    }
172                }
173                let chunks =
174                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
175                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
176            },
177            ArrowDataType::Binary => {
178                let chunks =
179                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
180                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
181            },
182            ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
183                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
184                unsafe {
185                    Ok(
186                        ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
187                            .into_series(),
188                    )
189                }
190            },
191            #[cfg(feature = "dtype-array")]
192            ArrowDataType::FixedSizeList(_, _) => {
193                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
194                unsafe {
195                    Ok(
196                        ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
197                            .into_series(),
198                    )
199                }
200            },
201            ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
202            #[cfg(feature = "dtype-u8")]
203            ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
204            #[cfg(feature = "dtype-u16")]
205            ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
206            ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
207            ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
208            #[cfg(feature = "dtype-i8")]
209            ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
210            #[cfg(feature = "dtype-i16")]
211            ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
212            ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
213            ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
214            ArrowDataType::Int128 => feature_gated!(
215                "dtype-i128",
216                Ok(Int128Chunked::from_chunks(name, chunks).into_series())
217            ),
218            ArrowDataType::Float16 => {
219                let chunks =
220                    cast_chunks(&chunks, &DataType::Float32, CastOptions::NonStrict).unwrap();
221                Ok(Float32Chunked::from_chunks(name, chunks).into_series())
222            },
223            ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
224            ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
225            #[cfg(feature = "dtype-date")]
226            ArrowDataType::Date32 => {
227                let chunks =
228                    cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
229                Ok(Int32Chunked::from_chunks(name, chunks)
230                    .into_date()
231                    .into_series())
232            },
233            #[cfg(feature = "dtype-datetime")]
234            ArrowDataType::Date64 => {
235                let chunks =
236                    cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
237                let ca = Int64Chunked::from_chunks(name, chunks);
238                Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
239            },
240            #[cfg(feature = "dtype-datetime")]
241            ArrowDataType::Timestamp(tu, tz) => {
242                let canonical_tz = DataType::canonical_timezone(tz);
243                let tz = match canonical_tz.as_deref() {
244                    #[cfg(feature = "timezones")]
245                    Some(tz_str) => match validate_time_zone(tz_str) {
246                        Ok(_) => canonical_tz,
247                        Err(_) => Some(parse_fixed_offset(tz_str)?),
248                    },
249                    _ => canonical_tz,
250                };
251                let chunks =
252                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
253                let s = Int64Chunked::from_chunks(name, chunks)
254                    .into_datetime(tu.into(), tz)
255                    .into_series();
256                Ok(match tu {
257                    ArrowTimeUnit::Second => &s * MILLISECONDS,
258                    ArrowTimeUnit::Millisecond => s,
259                    ArrowTimeUnit::Microsecond => s,
260                    ArrowTimeUnit::Nanosecond => s,
261                })
262            },
263            #[cfg(feature = "dtype-duration")]
264            ArrowDataType::Duration(tu) => {
265                let chunks =
266                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
267                let s = Int64Chunked::from_chunks(name, chunks)
268                    .into_duration(tu.into())
269                    .into_series();
270                Ok(match tu {
271                    ArrowTimeUnit::Second => &s * MILLISECONDS,
272                    ArrowTimeUnit::Millisecond => s,
273                    ArrowTimeUnit::Microsecond => s,
274                    ArrowTimeUnit::Nanosecond => s,
275                })
276            },
277            #[cfg(feature = "dtype-time")]
278            ArrowDataType::Time64(tu) | ArrowDataType::Time32(tu) => {
279                let mut chunks = chunks;
280                if matches!(dtype, ArrowDataType::Time32(_)) {
281                    chunks =
282                        cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
283                }
284                let chunks =
285                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
286                let s = Int64Chunked::from_chunks(name, chunks)
287                    .into_time()
288                    .into_series();
289                Ok(match tu {
290                    ArrowTimeUnit::Second => &s * NANOSECONDS,
291                    ArrowTimeUnit::Millisecond => &s * 1_000_000,
292                    ArrowTimeUnit::Microsecond => &s * 1_000,
293                    ArrowTimeUnit::Nanosecond => s,
294                })
295            },
296            ArrowDataType::Decimal(precision, scale)
297            | ArrowDataType::Decimal256(precision, scale) => {
298                feature_gated!("dtype-decimal", {
299                    polars_ensure!(*scale <= *precision, InvalidOperation: "invalid decimal precision and scale (prec={precision}, scale={scale})");
300                    polars_ensure!(*precision <= 38, InvalidOperation: "polars does not support decimals about 38 precision");
301
302                    let mut chunks = chunks;
303                    // @NOTE: We cannot cast here as that will lower the scale.
304                    for chunk in chunks.iter_mut() {
305                        *chunk = std::mem::take(
306                            chunk
307                                .as_any_mut()
308                                .downcast_mut::<PrimitiveArray<i128>>()
309                                .unwrap(),
310                        )
311                        .to(ArrowDataType::Int128)
312                        .to_boxed();
313                    }
314                    let s = Int128Chunked::from_chunks(name, chunks)
315                        .into_decimal_unchecked(Some(*precision), *scale)
316                        .into_series();
317                    Ok(s)
318                })
319            },
320            ArrowDataType::Null => Ok(new_null(name, &chunks)),
321            #[cfg(not(feature = "dtype-categorical"))]
322            ArrowDataType::Dictionary(_, _, _) => {
323                panic!("activate dtype-categorical to convert dictionary arrays")
324            },
325            #[cfg(feature = "dtype-categorical")]
326            ArrowDataType::Dictionary(key_type, value_type, _) => {
327                use arrow::datatypes::IntegerType;
328                // don't spuriously call this; triggers a read on mmapped data
329                let arr = if chunks.len() > 1 {
330                    concatenate_owned_unchecked(&chunks)?
331                } else {
332                    chunks[0].clone()
333                };
334
335                // If the value type is a string, they are converted to Categoricals or Enums
336                if matches!(
337                    value_type.as_ref(),
338                    ArrowDataType::Utf8
339                        | ArrowDataType::LargeUtf8
340                        | ArrowDataType::Utf8View
341                        | ArrowDataType::Null
342                ) {
343                    macro_rules! unpack_keys_values {
344                        ($dt:ty) => {{
345                            let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
346                            let keys = arr.keys();
347                            let keys = cast(keys, &ArrowDataType::UInt32).unwrap();
348                            let values = arr.values();
349                            let values = cast(&**values, &ArrowDataType::Utf8View)?;
350                            (keys, values)
351                        }};
352                    }
353
354                    use IntegerType as I;
355                    let (keys, values) = match key_type {
356                        I::Int8 => unpack_keys_values!(i8),
357                        I::UInt8 => unpack_keys_values!(u8),
358                        I::Int16 => unpack_keys_values!(i16),
359                        I::UInt16 => unpack_keys_values!(u16),
360                        I::Int32 => unpack_keys_values!(i32),
361                        I::UInt32 => unpack_keys_values!(u32),
362                        I::Int64 => unpack_keys_values!(i64),
363                        _ => polars_bail!(
364                            ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
365                        ),
366                    };
367
368                    let keys = keys.as_any().downcast_ref::<PrimitiveArray<u32>>().unwrap();
369                    let values = values.as_any().downcast_ref::<Utf8ViewArray>().unwrap();
370
371                    // Categoricals and Enums expect the RevMap values to not contain any nulls
372                    let (keys, values) =
373                        polars_compute::propagate_dictionary::propagate_dictionary_value_nulls(
374                            keys, values,
375                        );
376
377                    let mut ordering = CategoricalOrdering::default();
378                    if let Some(metadata) = md {
379                        if metadata.is_enum() {
380                            // SAFETY:
381                            // the invariants of an Arrow Dictionary guarantee the keys are in bounds
382                            return Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
383                                UInt32Chunked::with_chunk(name, keys),
384                                Arc::new(RevMapping::build_local(values)),
385                                true,
386                                CategoricalOrdering::Physical, // Enum always uses physical ordering
387                            )
388                            .into_series());
389                        } else if let Some(o) = metadata.categorical() {
390                            ordering = o;
391                        }
392                    }
393
394                    return Ok(CategoricalChunked::from_keys_and_values(
395                        name, &keys, &values, ordering,
396                    )
397                    .into_series());
398                }
399
400                macro_rules! unpack_keys_values {
401                    ($dt:ty) => {{
402                        let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
403                        let keys = arr.keys();
404                        let keys = polars_compute::cast::primitive_as_primitive::<
405                            $dt,
406                            <IdxType as PolarsNumericType>::Native,
407                        >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
408                        (arr.values(), keys)
409                    }};
410                }
411
412                use IntegerType as I;
413                let (values, keys) = match key_type {
414                    I::Int8 => unpack_keys_values!(i8),
415                    I::UInt8 => unpack_keys_values!(u8),
416                    I::Int16 => unpack_keys_values!(i16),
417                    I::UInt16 => unpack_keys_values!(u16),
418                    I::Int32 => unpack_keys_values!(i32),
419                    I::UInt32 => unpack_keys_values!(u32),
420                    I::Int64 => unpack_keys_values!(i64),
421                    _ => polars_bail!(
422                        ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
423                    ),
424                };
425
426                // Convert the dictionary to a flat array
427                let values = Series::_try_from_arrow_unchecked_with_md(
428                    name,
429                    vec![values.clone()],
430                    values.dtype(),
431                    None,
432                )?;
433                let values = values.take_unchecked(&IdxCa::from_chunks_and_dtype(
434                    PlSmallStr::EMPTY,
435                    vec![keys.to_boxed()],
436                    IDX_DTYPE,
437                ));
438
439                Ok(values)
440            },
441            #[cfg(feature = "object")]
442            ArrowDataType::Extension(ext)
443                if ext.name == EXTENSION_NAME && ext.metadata.is_some() =>
444            {
445                assert_eq!(chunks.len(), 1);
446                let arr = chunks[0]
447                    .as_any()
448                    .downcast_ref::<FixedSizeBinaryArray>()
449                    .unwrap();
450                // SAFETY:
451                // this is highly unsafe. it will dereference a raw ptr on the heap
452                // make sure the ptr is allocated and from this pid
453                // (the pid is checked before dereference)
454                let s = {
455                    let pe = PolarsExtension::new(arr.clone());
456                    let s = pe.get_series(&name);
457                    pe.take_and_forget();
458                    s
459                };
460                Ok(s)
461            },
462            #[cfg(feature = "dtype-struct")]
463            ArrowDataType::Struct(_) => {
464                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
465
466                unsafe {
467                    let mut ca =
468                        StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
469                    ca.propagate_nulls();
470                    Ok(ca.into_series())
471                }
472            },
473            ArrowDataType::FixedSizeBinary(_) => {
474                let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
475                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
476            },
477            ArrowDataType::Map(_, _) => map_arrays_to_series(name, chunks),
478            dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
479        }
480    }
481}
482
483fn map_arrays_to_series(name: PlSmallStr, chunks: Vec<ArrayRef>) -> PolarsResult<Series> {
484    let chunks = chunks
485        .iter()
486        .map(|arr| {
487            // we convert the map to the logical type: List<struct<key, value>>
488            let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
489            let inner = arr.field().clone();
490
491            // map has i32 offsets
492            let dtype = ListArray::<i32>::default_datatype(inner.dtype().clone());
493            Box::new(ListArray::<i32>::new(
494                dtype,
495                arr.offsets().clone(),
496                inner,
497                arr.validity().cloned(),
498            )) as ArrayRef
499        })
500        .collect::<Vec<_>>();
501    Series::try_from((name, chunks))
502}
503
504fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
505    arr.iter().map(|arr| f(&**arr)).collect()
506}
507
508/// Converts to physical types and bubbles up the correct [`DataType`].
509#[allow(clippy::only_used_in_recursion)]
510unsafe fn to_physical_and_dtype(
511    arrays: Vec<ArrayRef>,
512    md: Option<&Metadata>,
513) -> (Vec<ArrayRef>, DataType) {
514    match arrays[0].dtype() {
515        ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
516            let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
517            (chunks, DataType::String)
518        },
519        ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
520            let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
521            (chunks, DataType::Binary)
522        },
523        #[allow(unused_variables)]
524        dt @ ArrowDataType::Dictionary(_, _, _) => {
525            feature_gated!("dtype-categorical", {
526                let s = unsafe {
527                    let dt = dt.clone();
528                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
529                }
530                .unwrap();
531                (s.chunks().clone(), s.dtype().clone())
532            })
533        },
534        ArrowDataType::List(field) => {
535            let out = convert(&arrays, |arr| {
536                cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
537            });
538            to_physical_and_dtype(out, md)
539        },
540        #[cfg(feature = "dtype-array")]
541        ArrowDataType::FixedSizeList(field, size) => {
542            let values = arrays
543                .iter()
544                .map(|arr| {
545                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
546                    arr.values().clone()
547                })
548                .collect::<Vec<_>>();
549
550            let (converted_values, dtype) =
551                to_physical_and_dtype(values, field.metadata.as_deref());
552
553            let arrays = arrays
554                .iter()
555                .zip(converted_values)
556                .map(|(arr, values)| {
557                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
558
559                    let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
560                    Box::from(FixedSizeListArray::new(
561                        dtype,
562                        arr.len(),
563                        values,
564                        arr.validity().cloned(),
565                    )) as ArrayRef
566                })
567                .collect();
568            (arrays, DataType::Array(Box::new(dtype), *size))
569        },
570        ArrowDataType::LargeList(field) => {
571            let values = arrays
572                .iter()
573                .map(|arr| {
574                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
575                    arr.values().clone()
576                })
577                .collect::<Vec<_>>();
578
579            let (converted_values, dtype) =
580                to_physical_and_dtype(values, field.metadata.as_deref());
581
582            let arrays = arrays
583                .iter()
584                .zip(converted_values)
585                .map(|(arr, values)| {
586                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
587
588                    let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
589                    Box::from(ListArray::<i64>::new(
590                        dtype,
591                        arr.offsets().clone(),
592                        values,
593                        arr.validity().cloned(),
594                    )) as ArrayRef
595                })
596                .collect();
597            (arrays, DataType::List(Box::new(dtype)))
598        },
599        ArrowDataType::Struct(_fields) => {
600            feature_gated!("dtype-struct", {
601                let mut pl_fields = None;
602                let arrays = arrays
603                    .iter()
604                    .map(|arr| {
605                        let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
606                        let (values, dtypes): (Vec<_>, Vec<_>) = arr
607                            .values()
608                            .iter()
609                            .zip(_fields.iter())
610                            .map(|(value, field)| {
611                                let mut out = to_physical_and_dtype(
612                                    vec![value.clone()],
613                                    field.metadata.as_deref(),
614                                );
615                                (out.0.pop().unwrap(), out.1)
616                            })
617                            .unzip();
618
619                        let arrow_fields = values
620                            .iter()
621                            .zip(_fields.iter())
622                            .map(|(arr, field)| {
623                                ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
624                            })
625                            .collect();
626                        let arrow_array = Box::new(StructArray::new(
627                            ArrowDataType::Struct(arrow_fields),
628                            arr.len(),
629                            values,
630                            arr.validity().cloned(),
631                        )) as ArrayRef;
632
633                        if pl_fields.is_none() {
634                            pl_fields = Some(
635                                _fields
636                                    .iter()
637                                    .zip(dtypes)
638                                    .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
639                                    .collect_vec(),
640                            )
641                        }
642
643                        arrow_array
644                    })
645                    .collect_vec();
646
647                (arrays, DataType::Struct(pl_fields.unwrap()))
648            })
649        },
650        // Use Series architecture to convert nested logical types to physical.
651        dt @ (ArrowDataType::Duration(_)
652        | ArrowDataType::Time32(_)
653        | ArrowDataType::Time64(_)
654        | ArrowDataType::Timestamp(_, _)
655        | ArrowDataType::Date32
656        | ArrowDataType::Decimal(_, _)
657        | ArrowDataType::Date64) => {
658            let dt = dt.clone();
659            let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt).unwrap();
660            let dtype = s.dtype().clone();
661            (std::mem::take(s.chunks_mut()), dtype)
662        },
663        dt => {
664            let dtype = DataType::from_arrow(dt, true, md);
665            (arrays, dtype)
666        },
667    }
668}
669
670fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
671    let mut chunks_iter = chunks.iter();
672    let dtype: ArrowDataType = chunks_iter
673        .next()
674        .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
675        .dtype()
676        .clone();
677
678    for chunk in chunks_iter {
679        if chunk.dtype() != &dtype {
680            polars_bail!(
681                ComputeError: "cannot create series from multiple arrays with different types"
682            );
683        }
684    }
685    Ok(dtype)
686}
687
688impl Series {
689    pub fn try_new<T>(
690        name: PlSmallStr,
691        data: T,
692    ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
693    where
694        (PlSmallStr, T): TryInto<Self>,
695    {
696        // # TODO
697        // * Remove the TryFrom<tuple> impls in favor of this
698        <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
699    }
700}
701
702impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
703    type Error = PolarsError;
704
705    fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
706        let (name, chunks) = name_arr;
707
708        let dtype = check_types(&chunks)?;
709        // SAFETY:
710        // dtype is checked
711        unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
712    }
713}
714
715impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
716    type Error = PolarsError;
717
718    fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
719        let (name, arr) = name_arr;
720        Series::try_from((name, vec![arr]))
721    }
722}
723
724impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
725    type Error = PolarsError;
726
727    fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
728        let (field, chunks) = field_arr;
729
730        let dtype = check_types(&chunks)?;
731
732        // SAFETY:
733        // dtype is checked
734        unsafe {
735            Series::_try_from_arrow_unchecked_with_md(
736                field.name.clone(),
737                chunks,
738                &dtype,
739                field.metadata.as_deref(),
740            )
741        }
742    }
743}
744
745impl TryFrom<(&ArrowField, ArrayRef)> for Series {
746    type Error = PolarsError;
747
748    fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
749        let (field, arr) = field_arr;
750        Series::try_from((field, vec![arr]))
751    }
752}
753
754/// Used to convert a [`ChunkedArray`], `&dyn SeriesTrait` and [`Series`]
755/// into a [`Series`].
756/// # Safety
757///
758/// This trait is marked `unsafe` as the `is_series` return is used
759/// to transmute to `Series`. This must always return `false` except
760/// for `Series` structs.
761pub unsafe trait IntoSeries {
762    fn is_series() -> bool {
763        false
764    }
765
766    fn into_series(self) -> Series
767    where
768        Self: Sized;
769}
770
771impl<T> From<ChunkedArray<T>> for Series
772where
773    T: PolarsDataType,
774    ChunkedArray<T>: IntoSeries,
775{
776    fn from(ca: ChunkedArray<T>) -> Self {
777        ca.into_series()
778    }
779}
780
781#[cfg(feature = "dtype-date")]
782impl From<DateChunked> for Series {
783    fn from(a: DateChunked) -> Self {
784        a.into_series()
785    }
786}
787
788#[cfg(feature = "dtype-datetime")]
789impl From<DatetimeChunked> for Series {
790    fn from(a: DatetimeChunked) -> Self {
791        a.into_series()
792    }
793}
794
795#[cfg(feature = "dtype-duration")]
796impl From<DurationChunked> for Series {
797    fn from(a: DurationChunked) -> Self {
798        a.into_series()
799    }
800}
801
802#[cfg(feature = "dtype-time")]
803impl From<TimeChunked> for Series {
804    fn from(a: TimeChunked) -> Self {
805        a.into_series()
806    }
807}
808
809unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
810    fn into_series(self) -> Series {
811        Series(self)
812    }
813}
814
815unsafe impl IntoSeries for Series {
816    fn is_series() -> bool {
817        true
818    }
819
820    fn into_series(self) -> Series {
821        self
822    }
823}
824
825fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
826    let len = chunks.iter().map(|arr| arr.len()).sum();
827    Series::new_null(name, len)
828}
polars_core/series/from.rs

polars_core/series/
from.rs
