diff --git a/Apache.Arrow.Tests.slnf b/Apache.Arrow.Tests.slnf index 90bf9bda..6af33f35 100644 --- a/Apache.Arrow.Tests.slnf +++ b/Apache.Arrow.Tests.slnf @@ -8,6 +8,7 @@ "test\\Apache.Arrow.Flight.TestWeb\\Apache.Arrow.Flight.TestWeb.csproj", "test\\Apache.Arrow.Tests\\Apache.Arrow.Tests.csproj", "test\\Apache.Arrow.Scalars.Tests\\Apache.Arrow.Scalars.Tests.csproj", + "test\\Apache.Arrow.Compute.Tests\\Apache.Arrow.Compute.Tests.csproj", "test\\Apache.Arrow.Operations.Tests\\Apache.Arrow.Operations.Tests.csproj" ] } diff --git a/Apache.Arrow.sln b/Apache.Arrow.sln index 26bd326c..bf6e818c 100644 --- a/Apache.Arrow.sln +++ b/Apache.Arrow.sln @@ -38,6 +38,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Operations", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Operations.Tests", "test\Apache.Arrow.Operations.Tests\Apache.Arrow.Operations.Tests.csproj", "{73EBE132-AE05-4C32-9525-515F4768156B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Compute", "src\Apache.Arrow.Compute\Apache.Arrow.Compute.csproj", "{3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{0C88DD14-F956-CE84-757C-A364CCF449FC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Compute.Tests", "test\Apache.Arrow.Compute.Tests\Apache.Arrow.Compute.Tests.csproj", "{E183F970-90FA-49F1-B339-7827171A3B02}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Compute.Benchmarks", "test\Apache.Arrow.Compute.Benchmarks\Apache.Arrow.Compute.Benchmarks.csproj", "{C9657321-6DFA-411A-AD42-D51C51D0FAF0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -264,10 +274,69 @@ Global {73EBE132-AE05-4C32-9525-515F4768156B}.Release|x64.Build.0 = Release|Any CPU {73EBE132-AE05-4C32-9525-515F4768156B}.Release|x86.ActiveCfg = Release|Any CPU {73EBE132-AE05-4C32-9525-515F4768156B}.Release|x86.Build.0 = Release|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Debug|x64.ActiveCfg = Debug|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Debug|x64.Build.0 = Debug|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Debug|x86.ActiveCfg = Debug|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Debug|x86.Build.0 = Debug|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Release|Any CPU.Build.0 = Release|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Release|x64.ActiveCfg = Release|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Release|x64.Build.0 = Release|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Release|x86.ActiveCfg = Release|Any CPU + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0}.Release|x86.Build.0 = Release|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Debug|x64.ActiveCfg = Debug|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Debug|x64.Build.0 = Debug|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Debug|x86.ActiveCfg = Debug|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Debug|x86.Build.0 = Debug|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Release|Any CPU.Build.0 = Release|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Release|x64.ActiveCfg = Release|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Release|x64.Build.0 = Release|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Release|x86.ActiveCfg = Release|Any CPU + {E183F970-90FA-49F1-B339-7827171A3B02}.Release|x86.Build.0 = Release|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Debug|x64.ActiveCfg = Debug|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Debug|x64.Build.0 = Debug|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Debug|x86.ActiveCfg = Debug|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Debug|x86.Build.0 = Debug|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Release|Any CPU.Build.0 = Release|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Release|x64.ActiveCfg = Release|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Release|x64.Build.0 = Release|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Release|x86.ActiveCfg = Release|Any CPU + {C9657321-6DFA-411A-AD42-D51C51D0FAF0}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {BA6B2B0D-EAAE-4183-8A39-1B9CF571F71F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {9CCEC01B-E67A-4726-BE72-7B514F76163F} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {742DF47D-77C5-4B84-9E0C-69645F1161EA} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {D6443535-3740-4F6C-8001-F90EDAF4CF0C} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {058F9CFA-2A13-43B8-87D9-E69F63F9EFF0} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {2490AA1E-DDA4-4069-B065-79A4897B0582} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E4F74938-E8FF-4AC1-A495-FEE95FC1EFDF} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {B62E77D2-D0B0-4C0C-BA78-1C117DE4C299} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {5D7FF380-B7DF-4752-B415-7C08C70C9F06} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {DCC99EB1-4E60-4F0D-AEA9-C44A4C0C8B1D} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {2ADE087A-B424-4895-8CC5-10170D10BA62} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {7E66CBB4-D921-41E7-A98A-7C6DEA521696} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {E8264B7F-B680-4A55-939B-85DB628164BB} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {EE364C9A-CB88-4DCE-9209-6ACBB8E6934F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0C495233-010C-45F8-BAB2-D9CD0B7B9861} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {F3A8B7C6-D5E4-4321-9ABC-DEF012345678} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {B4799C3C-179F-470F-A8A4-8A75D172E3E1} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {73EBE132-AE05-4C32-9525-515F4768156B} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {3F9AEB1B-814A-4B82-8E0A-481AEA0CBDD0} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E183F970-90FA-49F1-B339-7827171A3B02} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {C9657321-6DFA-411A-AD42-D51C51D0FAF0} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FD0BB617-6031-4844-B99D-B331E335B572} EndGlobalSection diff --git a/Directory.Packages.props b/Directory.Packages.props index 80d3b53c..32a310f9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -40,6 +40,7 @@ + diff --git a/dev/release/verify_rc.ps1 b/dev/release/verify_rc.ps1 index de64d051..ec862cb0 100644 --- a/dev/release/verify_rc.ps1 +++ b/dev/release/verify_rc.ps1 @@ -227,6 +227,7 @@ function Test-Binary-Distribution { Reference-Package "Apache.Arrow.Flight.AspNetCore" @("Apache.Arrow.Flight.TestWeb") Reference-Package "Apache.Arrow.Operations" @("Apache.Arrow.Operations.Tests", "Apache.Arrow.Scalars.Tests") Reference-Package "Apache.Arrow.Scalars" @("Apache.Arrow.Scalars.Tests", "Apache.Arrow.Tests", "Apache.Arrow.Operations.Tests") + Reference-Package "Apache.Arrow.Compute" @("Apache.Arrow.Compute.Tests") # Move src directory to ensure we are only testing against built packages Rename-Item src src.backup diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index e2c22941..f07d6ce3 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -186,6 +186,7 @@ test_binary_distribution() { reference_package "Apache.Arrow.Flight.AspNetCore" "Apache.Arrow.Flight.TestWeb" reference_package "Apache.Arrow.Operations" "Apache.Arrow.Operations.Tests" "Apache.Arrow.Scalars.Tests" reference_package "Apache.Arrow.Scalars" "Apache.Arrow.Scalars.Tests" "Apache.Arrow.Tests" "Apache.Arrow.Operations.Tests" + reference_package "Apache.Arrow.Compute" "Apache.Arrow.Compute.Tests" # Move src directory to ensure we are only testing against built packages mv src src.backup diff --git a/src/Apache.Arrow.Compute/Aggregations.cs b/src/Apache.Arrow.Compute/Aggregations.cs new file mode 100644 index 00000000..26c5167e --- /dev/null +++ b/src/Apache.Arrow.Compute/Aggregations.cs @@ -0,0 +1,416 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +#if NET8_0_OR_GREATER +using System.Numerics; +using System.Numerics.Tensors; +#endif + +namespace Apache.Arrow.Compute +{ + /// + /// Aggregation kernels over (Sum/Min/Max/Mean). + /// + /// + /// + /// Null handling follows LINQ semantics: null entries are skipped and do not contribute to the + /// result. Sum of an empty or all-null array returns zero. Min, Max and + /// Mean throw when the array contains no non-null + /// elements, matching + /// and . + /// + /// + /// On net8.0 and later the kernels are generic over INumber<T> and, when the + /// array has no nulls, dispatch to TensorPrimitives for a SIMD-accelerated single + /// pass over the contiguous values buffer; when nulls are present they fall back to a correct, + /// validity-aware scalar loop. On netstandard2.0 and net462 (where generic math and + /// TensorPrimitives are unavailable) the kernels are provided as per-type overloads + /// (, , , + /// ) backed by scalar loops with the same null semantics. + /// + /// + public static class Aggregations + { + private const string NoElements = "Sequence contains no non-null elements."; + +#if NET8_0_OR_GREATER + /// Sums the non-null elements. Returns zero for an empty or all-null array. + public static T Sum(this PrimitiveArray array) + where T : unmanaged, INumber + { + if (array is null) throw new ArgumentNullException(nameof(array)); + + ReadOnlySpan values = array.Values; + + if (array.NullCount == 0) + { + return TensorPrimitives.Sum(values); + } + + T acc = T.Zero; + for (int i = 0; i < values.Length; i++) + { + if (array.IsValid(i)) + { + acc += values[i]; + } + } + return acc; + } + + /// Returns the smallest non-null element. + /// The array contains no non-null elements. + public static T Min(this PrimitiveArray array) + where T : unmanaged, INumber, IMinMaxValue + { + if (array is null) throw new ArgumentNullException(nameof(array)); + + ReadOnlySpan values = array.Values; + + if (values.Length == 0 || array.Length - array.NullCount == 0) + { + throw new InvalidOperationException(NoElements); + } + + if (array.NullCount == 0) + { + return TensorPrimitives.Min(values); + } + + T min = T.MaxValue; + for (int i = 0; i < values.Length; i++) + { + if (!array.IsValid(i)) continue; + if (values[i] < min) { min = values[i]; } + } + return min; + } + + /// Returns the largest non-null element. + /// The array contains no non-null elements. + public static T Max(this PrimitiveArray array) + where T : unmanaged, INumber, IMinMaxValue + { + if (array is null) throw new ArgumentNullException(nameof(array)); + + ReadOnlySpan values = array.Values; + + if (values.Length == 0 || array.Length - array.NullCount == 0) + { + throw new InvalidOperationException(NoElements); + } + + if (array.NullCount == 0) + { + return TensorPrimitives.Max(values); + } + + T max = T.MinValue; + for (int i = 0; i < values.Length; i++) + { + if (!array.IsValid(i)) continue; + if (values[i] > max) { max = values[i]; } + } + return max; + } + + /// Returns the arithmetic mean of the non-null elements as a . + /// The array contains no non-null elements. + public static double Mean(this PrimitiveArray array) + where T : unmanaged, INumber + { + if (array is null) throw new ArgumentNullException(nameof(array)); + + long count = array.Length - array.NullCount; + if (count == 0) + { + throw new InvalidOperationException(NoElements); + } + + T sum = array.Sum(); + return double.CreateChecked(sum) / count; + } +#else + // netstandard2.0 / net462 fallback: generic math and TensorPrimitives are unavailable, so the + // kernels are provided as per-type overloads backed by validity-aware scalar loops. The null + // semantics match the generic net8.0+ implementation above. + + #region Int32Array + + /// Sums the non-null elements. Returns zero for an empty or all-null array. + public static int Sum(this Int32Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + int acc = 0; + bool noNulls = array.NullCount == 0; + for (int i = 0; i < values.Length; i++) + { + if (noNulls || array.IsValid(i)) acc += values[i]; + } + return acc; + } + + /// Returns the smallest non-null element. + /// The array contains no non-null elements. + public static int Min(this Int32Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + int min = 0; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { min = values[i]; set = true; } + else if (values[i] < min) min = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return min; + } + + /// Returns the largest non-null element. + /// The array contains no non-null elements. + public static int Max(this Int32Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + int max = 0; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { max = values[i]; set = true; } + else if (values[i] > max) max = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return max; + } + + /// Returns the arithmetic mean of the non-null elements as a . + /// The array contains no non-null elements. + public static double Mean(this Int32Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + long count = array.Length - array.NullCount; + if (count == 0) throw new InvalidOperationException(NoElements); + return (double)array.Sum() / count; + } + + #endregion + + #region Int64Array + + /// Sums the non-null elements. Returns zero for an empty or all-null array. + public static long Sum(this Int64Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + long acc = 0; + bool noNulls = array.NullCount == 0; + for (int i = 0; i < values.Length; i++) + { + if (noNulls || array.IsValid(i)) acc += values[i]; + } + return acc; + } + + /// Returns the smallest non-null element. + /// The array contains no non-null elements. + public static long Min(this Int64Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + long min = 0; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { min = values[i]; set = true; } + else if (values[i] < min) min = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return min; + } + + /// Returns the largest non-null element. + /// The array contains no non-null elements. + public static long Max(this Int64Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + long max = 0; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { max = values[i]; set = true; } + else if (values[i] > max) max = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return max; + } + + /// Returns the arithmetic mean of the non-null elements as a . + /// The array contains no non-null elements. + public static double Mean(this Int64Array array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + long count = array.Length - array.NullCount; + if (count == 0) throw new InvalidOperationException(NoElements); + return (double)array.Sum() / count; + } + + #endregion + + #region FloatArray + + /// Sums the non-null elements. Returns zero for an empty or all-null array. + public static float Sum(this FloatArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + float acc = 0f; + bool noNulls = array.NullCount == 0; + for (int i = 0; i < values.Length; i++) + { + if (noNulls || array.IsValid(i)) acc += values[i]; + } + return acc; + } + + /// Returns the smallest non-null element. + /// The array contains no non-null elements. + public static float Min(this FloatArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + float min = 0f; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { min = values[i]; set = true; } + else if (values[i] < min) min = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return min; + } + + /// Returns the largest non-null element. + /// The array contains no non-null elements. + public static float Max(this FloatArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + float max = 0f; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { max = values[i]; set = true; } + else if (values[i] > max) max = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return max; + } + + /// Returns the arithmetic mean of the non-null elements as a . + /// The array contains no non-null elements. + public static double Mean(this FloatArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + long count = array.Length - array.NullCount; + if (count == 0) throw new InvalidOperationException(NoElements); + return (double)array.Sum() / count; + } + + #endregion + + #region DoubleArray + + /// Sums the non-null elements. Returns zero for an empty or all-null array. + public static double Sum(this DoubleArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + double acc = 0d; + bool noNulls = array.NullCount == 0; + for (int i = 0; i < values.Length; i++) + { + if (noNulls || array.IsValid(i)) acc += values[i]; + } + return acc; + } + + /// Returns the smallest non-null element. + /// The array contains no non-null elements. + public static double Min(this DoubleArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + double min = 0d; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { min = values[i]; set = true; } + else if (values[i] < min) min = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return min; + } + + /// Returns the largest non-null element. + /// The array contains no non-null elements. + public static double Max(this DoubleArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + ReadOnlySpan values = array.Values; + bool noNulls = array.NullCount == 0; + bool set = false; + double max = 0d; + for (int i = 0; i < values.Length; i++) + { + if (!noNulls && !array.IsValid(i)) continue; + if (!set) { max = values[i]; set = true; } + else if (values[i] > max) max = values[i]; + } + if (!set) throw new InvalidOperationException(NoElements); + return max; + } + + /// Returns the arithmetic mean of the non-null elements as a . + /// The array contains no non-null elements. + public static double Mean(this DoubleArray array) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + long count = array.Length - array.NullCount; + if (count == 0) throw new InvalidOperationException(NoElements); + return array.Sum() / count; + } + + #endregion +#endif + } +} diff --git a/src/Apache.Arrow.Compute/Apache.Arrow.Compute.csproj b/src/Apache.Arrow.Compute/Apache.Arrow.Compute.csproj new file mode 100644 index 00000000..989f7b50 --- /dev/null +++ b/src/Apache.Arrow.Compute/Apache.Arrow.Compute.csproj @@ -0,0 +1,19 @@ + + + + true + netstandard2.0;net8.0;net462 + true + Arrow compute kernels: SIMD-accelerated aggregations over PrimitiveArray<T>. On net8.0+ the kernels are generic (INumber<T>) and dispatch to System.Numerics.Tensors (TensorPrimitives) on the null-free fast path; on netstandard2.0/net462 they fall back to per-type scalar kernels. + + + + + + + + + + + + diff --git a/test/Apache.Arrow.Compute.Benchmarks/AggregationBenchmarks.cs b/test/Apache.Arrow.Compute.Benchmarks/AggregationBenchmarks.cs new file mode 100644 index 00000000..55dddf8a --- /dev/null +++ b/test/Apache.Arrow.Compute.Benchmarks/AggregationBenchmarks.cs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using BenchmarkDotNet.Attributes; + +namespace Apache.Arrow.Compute.Benchmarks +{ + /// + /// Compares the SIMD-accelerated aggregation kernels (which dispatch to TensorPrimitives on the + /// null-free fast path) against a straightforward managed scalar loop over the same Arrow array. + /// This is the evidence that the fast path earns its place. + /// + [MemoryDiagnoser] + public class AggregationBenchmarks + { + [Params(1024, 1_000_000)] + public int Size { get; set; } + + private DoubleArray _values = null!; + + [GlobalSetup] + public void Setup() + { + var builder = new DoubleArray.Builder(); + builder.Reserve(Size); + var rng = new Random(17); + for (int i = 0; i < Size; i++) + { + builder.Append(rng.NextDouble() * 1000.0); + } + _values = builder.Build(); + } + + // SIMD fast path (null-free => TensorPrimitives). + [Benchmark(Baseline = true)] + public double Sum_Kernel() => _values.Sum(); + + // Straightforward managed scalar loop over the same values buffer. + [Benchmark] + public double Sum_NaiveScalar() + { + ReadOnlySpan values = _values.Values; + double acc = 0d; + for (int i = 0; i < values.Length; i++) + { + acc += values[i]; + } + return acc; + } + + [Benchmark] + public double Min_Kernel() => _values.Min(); + + [Benchmark] + public double Min_NaiveScalar() + { + ReadOnlySpan values = _values.Values; + double min = values[0]; + for (int i = 1; i < values.Length; i++) + { + if (values[i] < min) min = values[i]; + } + return min; + } + + [Benchmark] + public double Mean_Kernel() => _values.Mean(); + } +} diff --git a/test/Apache.Arrow.Compute.Benchmarks/Apache.Arrow.Compute.Benchmarks.csproj b/test/Apache.Arrow.Compute.Benchmarks/Apache.Arrow.Compute.Benchmarks.csproj new file mode 100644 index 00000000..c8cf5599 --- /dev/null +++ b/test/Apache.Arrow.Compute.Benchmarks/Apache.Arrow.Compute.Benchmarks.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + + + + + + + + + + + + diff --git a/test/Apache.Arrow.Compute.Benchmarks/Program.cs b/test/Apache.Arrow.Compute.Benchmarks/Program.cs new file mode 100644 index 00000000..5c65a603 --- /dev/null +++ b/test/Apache.Arrow.Compute.Benchmarks/Program.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using BenchmarkDotNet.Running; + +namespace Apache.Arrow.Compute.Benchmarks +{ + public static class Program + { + public static void Main(string[] args) + { + BenchmarkSwitcher + .FromAssembly(typeof(Program).Assembly) + .Run(args); + } + } +} diff --git a/test/Apache.Arrow.Compute.Tests/AggregationsTests.cs b/test/Apache.Arrow.Compute.Tests/AggregationsTests.cs new file mode 100644 index 00000000..30f57abe --- /dev/null +++ b/test/Apache.Arrow.Compute.Tests/AggregationsTests.cs @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Linq; +using Apache.Arrow; +using Apache.Arrow.Compute; +using Xunit; + +namespace Apache.Arrow.Compute.Tests +{ + public class AggregationsTests + { + private static DoubleArray Doubles(params double?[] values) + { + var builder = new DoubleArray.Builder(); + foreach (double? v in values) + { + if (v.HasValue) builder.Append(v.Value); + else builder.AppendNull(); + } + return builder.Build(); + } + + private static Int32Array Ints(params int?[] values) + { + var builder = new Int32Array.Builder(); + foreach (int? v in values) + { + if (v.HasValue) builder.Append(v.Value); + else builder.AppendNull(); + } + return builder.Build(); + } + + [Fact] + public void Sum_Int32_NoNulls() + { + Assert.Equal(10, Ints(1, 2, 3, 4).Sum()); + } + + [Fact] + public void Sum_Int32_WithNulls() + { + Assert.Equal(4, Ints(1, null, 3).Sum()); + } + + [Fact] + public void Sum_Double_NoNulls() + { + Assert.Equal(6.5, Doubles(1.0, 2.0, 3.5).Sum(), 6); + } + + [Fact] + public void Min_Max_Double_NoNulls() + { + var a = Doubles(3.0, -1.0, 7.5, 2.0); + Assert.Equal(-1.0, a.Min(), 6); + Assert.Equal(7.5, a.Max(), 6); + } + + [Fact] + public void Min_Max_WithNulls_IgnoresNulls() + { + var a = Doubles(null, 5.0, null, 2.0, 9.0); + Assert.Equal(2.0, a.Min(), 6); + Assert.Equal(9.0, a.Max(), 6); + } + + [Fact] + public void Mean_Double_WithNulls_DividesByNonNullCount() + { + // (2 + 4) / 2 = 3, the null is excluded from both sum and count. + Assert.Equal(3.0, Doubles(2.0, null, 4.0).Mean(), 6); + } + + [Fact] + public void Mean_Int32_ReturnsDouble() + { + Assert.Equal(2.5, Ints(1, 2, 3, 4).Mean(), 6); + } + + [Fact] + public void SingleElement() + { + Assert.Equal(42.0, Doubles(42.0).Sum(), 6); + Assert.Equal(42.0, Doubles(42.0).Min(), 6); + Assert.Equal(42.0, Doubles(42.0).Max(), 6); + Assert.Equal(42.0, Doubles(42.0).Mean(), 6); + } + + [Fact] + public void Empty_Sum_IsZero_ButMinMaxMeanThrow() + { + var empty = Doubles(); + Assert.Equal(0.0, empty.Sum(), 6); + Assert.Throws(() => empty.Min()); + Assert.Throws(() => empty.Max()); + Assert.Throws(() => empty.Mean()); + } + + [Fact] + public void AllNull_MinMaxMeanThrow() + { + var allNull = Doubles(null, null, null); + Assert.Throws(() => allNull.Min()); + Assert.Throws(() => allNull.Max()); + Assert.Throws(() => allNull.Mean()); + } + + [Fact] + public void Large_FastPath_MatchesScalar() + { + const int n = 1_000_000; + var rng = new Random(17); + double[] data = Enumerable.Range(0, n).Select(_ => rng.NextDouble() * 100.0).ToArray(); + + var builder = new DoubleArray.Builder(); + builder.Append(data.AsSpan()); + DoubleArray array = builder.Build(); + + double scalar = 0.0; + for (int i = 0; i < n; i++) scalar += data[i]; + + // Fast (TensorPrimitives) path; allow small floating-point reorder tolerance. + Assert.Equal(scalar, array.Sum(), 3); + Assert.Equal(data.Min(), array.Min(), 9); + Assert.Equal(data.Max(), array.Max(), 9); + } + } +} diff --git a/test/Apache.Arrow.Compute.Tests/Apache.Arrow.Compute.Tests.csproj b/test/Apache.Arrow.Compute.Tests/Apache.Arrow.Compute.Tests.csproj new file mode 100644 index 00000000..5d3a7733 --- /dev/null +++ b/test/Apache.Arrow.Compute.Tests/Apache.Arrow.Compute.Tests.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + false + + + + + + + + + + + + + +